Bug Summary

File:uvm/uvm_fault.c
Warning:line 1047, column 3
Value stored to 'oanon' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name uvm_fault.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/uvm/uvm_fault.c
1/* $OpenBSD: uvm_fault.c,v 1.135 2023/09/05 05:08:26 guenther Exp $ */
2/* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */
3
4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
29 */
30
31/*
32 * uvm_fault.c: fault handler
33 */
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/percpu.h>
39#include <sys/proc.h>
40#include <sys/malloc.h>
41#include <sys/mman.h>
42#include <sys/tracepoint.h>
43
44#include <uvm/uvm.h>
45
46/*
47 *
48 * a word on page faults:
49 *
50 * types of page faults we handle:
51 *
52 * CASE 1: upper layer faults CASE 2: lower layer faults
53 *
54 * CASE 1A CASE 1B CASE 2A CASE 2B
55 * read/write1 write>1 read/write +-cow_write/zero
56 * | | | |
57 * +--|--+ +--|--+ +-----+ + | + | +-----+
58 * amap | V | | ---------> new | | | | ^ |
59 * +-----+ +-----+ +-----+ + | + | +--|--+
60 * | | |
61 * +-----+ +-----+ +--|--+ | +--|--+
62 * uobj | d/c | | d/c | | V | +----+ |
63 * +-----+ +-----+ +-----+ +-----+
64 *
65 * d/c = don't care
66 *
67 * case [0]: layerless fault
68 * no amap or uobj is present. this is an error.
69 *
70 * case [1]: upper layer fault [anon active]
71 * 1A: [read] or [write with anon->an_ref == 1]
72 * I/O takes place in upper level anon and uobj is not touched.
73 * 1B: [write with anon->an_ref > 1]
74 * new anon is alloc'd and data is copied off ["COW"]
75 *
76 * case [2]: lower layer fault [uobj]
77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
78 * I/O takes place directly in object.
79 * 2B: [write to copy_on_write] or [read on NULL uobj]
80 * data is "promoted" from uobj to a new anon.
81 * if uobj is null, then we zero fill.
82 *
83 * we follow the standard UVM locking protocol ordering:
84 *
85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
86 * we hold a PG_BUSY page if we unlock for I/O
87 *
88 *
89 * the code is structured as follows:
90 *
91 * - init the "IN" params in the ufi structure
92 * ReFault: (ERESTART returned to the loop in uvm_fault)
93 * - do lookups [locks maps], check protection, handle needs_copy
94 * - check for case 0 fault (error)
95 * - establish "range" of fault
96 * - if we have an amap lock it and extract the anons
97 * - if sequential advice deactivate pages behind us
98 * - at the same time check pmap for unmapped areas and anon for pages
99 * that we could map in (and do map it if found)
100 * - check object for resident pages that we could map in
101 * - if (case 2) goto Case2
102 * - >>> handle case 1
103 * - ensure source anon is resident in RAM
104 * - if case 1B alloc new anon and copy from source
105 * - map the correct page in
106 * Case2:
107 * - >>> handle case 2
108 * - ensure source page is resident (if uobj)
109 * - if case 2B alloc new anon and copy from source (could be zero
110 * fill if uobj == NULL)
111 * - map the correct page in
112 * - done!
113 *
114 * note on paging:
115 * if we have to do I/O we place a PG_BUSY page in the correct object,
116 * unlock everything, and do the I/O. when I/O is done we must reverify
117 * the state of the world before assuming that our data structures are
118 * valid. [because mappings could change while the map is unlocked]
119 *
120 * alternative 1: unbusy the page in question and restart the page fault
121 * from the top (ReFault). this is easy but does not take advantage
122 * of the information that we already have from our previous lookup,
123 * although it is possible that the "hints" in the vm_map will help here.
124 *
125 * alternative 2: the system already keeps track of a "version" number of
126 * a map. [i.e. every time you write-lock a map (e.g. to change a
127 * mapping) you bump the version number up by one...] so, we can save
128 * the version number of the map before we release the lock and start I/O.
129 * then when I/O is done we can relock and check the version numbers
130 * to see if anything changed. this might save us some over 1 because
131 * we don't have to unbusy the page and may be less compares(?).
132 *
133 * alternative 3: put in backpointers or a way to "hold" part of a map
134 * in place while I/O is in progress. this could be complex to
135 * implement (especially with structures like amap that can be referenced
136 * by multiple map entries, and figuring out what should wait could be
137 * complex as well...).
138 *
139 * we use alternative 2. given that we are multi-threaded now we may want
140 * to reconsider the choice.
141 */
142
143/*
144 * local data structures
145 */
146struct uvm_advice {
147 int nback;
148 int nforw;
149};
150
151/*
152 * page range array: set up in uvmfault_init().
153 */
154static struct uvm_advice uvmadvice[MADV_MASK0x7 + 1];
155
156#define UVM_MAXRANGE16 16 /* must be max() of nback+nforw+1 */
157
158/*
159 * private prototypes
160 */
161static void uvmfault_amapcopy(struct uvm_faultinfo *);
162static inline void uvmfault_anonflush(struct vm_anon **, int);
163void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t);
164void uvmfault_update_stats(struct uvm_faultinfo *);
165
166/*
167 * inline functions
168 */
169/*
170 * uvmfault_anonflush: try and deactivate pages in specified anons
171 *
172 * => does not have to deactivate page if it is busy
173 */
174static inline void
175uvmfault_anonflush(struct vm_anon **anons, int n)
176{
177 int lcv;
178 struct vm_page *pg;
179
180 for (lcv = 0; lcv < n; lcv++) {
181 if (anons[lcv] == NULL((void *)0))
182 continue;
183 KASSERT(rw_lock_held(anons[lcv]->an_lock))((rw_lock_held(anons[lcv]->an_lock)) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 183, "rw_lock_held(anons[lcv]->an_lock)"
));
184 pg = anons[lcv]->an_page;
185 if (pg && (pg->pg_flags & PG_BUSY0x00000001) == 0) {
186 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
187 if (pg->wire_count == 0) {
188 pmap_page_protect(pg, PROT_NONE0x00);
189 uvm_pagedeactivate(pg);
190 }
191 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
192 }
193 }
194}
195
196/*
197 * normal functions
198 */
199/*
200 * uvmfault_init: compute proper values for the uvmadvice[] array.
201 */
202void
203uvmfault_init(void)
204{
205 int npages;
206
207 npages = atop(16384)((16384) >> 12);
208 if (npages > 0) {
209 KASSERT(npages <= UVM_MAXRANGE / 2)((npages <= 16 / 2) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 209, "npages <= UVM_MAXRANGE / 2"));
210 uvmadvice[MADV_NORMAL0].nforw = npages;
211 uvmadvice[MADV_NORMAL0].nback = npages - 1;
212 }
213
214 npages = atop(32768)((32768) >> 12);
215 if (npages > 0) {
216 KASSERT(npages <= UVM_MAXRANGE / 2)((npages <= 16 / 2) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 216, "npages <= UVM_MAXRANGE / 2"));
217 uvmadvice[MADV_SEQUENTIAL2].nforw = npages - 1;
218 uvmadvice[MADV_SEQUENTIAL2].nback = npages;
219 }
220}
221
222/*
223 * uvmfault_amapcopy: clear "needs_copy" in a map.
224 *
225 * => called with VM data structures unlocked (usually, see below)
226 * => we get a write lock on the maps and clear needs_copy for a VA
227 * => if we are out of RAM we sleep (waiting for more)
228 */
229static void
230uvmfault_amapcopy(struct uvm_faultinfo *ufi)
231{
232 for (;;) {
233 /*
234 * no mapping? give up.
235 */
236 if (uvmfault_lookup(ufi, TRUE1) == FALSE0)
237 return;
238
239 /*
240 * copy if needed.
241 */
242 if (UVM_ET_ISNEEDSCOPY(ufi->entry)(((ufi->entry)->etype & 0x0008) != 0))
243 amap_copy(ufi->map, ufi->entry, M_NOWAIT0x0002,
244 UVM_ET_ISSTACK(ufi->entry)(((ufi->entry)->etype & 0x0040) != 0) ? FALSE0 : TRUE1,
245 ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
246
247 /*
248 * didn't work? must be out of RAM. unlock and sleep.
249 */
250 if (UVM_ET_ISNEEDSCOPY(ufi->entry)(((ufi->entry)->etype & 0x0008) != 0)) {
251 uvmfault_unlockmaps(ufi, TRUE1);
252 uvm_wait("fltamapcopy");
253 continue;
254 }
255
256 /*
257 * got it! unlock and return.
258 */
259 uvmfault_unlockmaps(ufi, TRUE1);
260 return;
261 }
262 /*NOTREACHED*/
263}
264
265/*
266 * uvmfault_anonget: get data in an anon into a non-busy, non-released
267 * page in that anon.
268 *
269 * => Map, amap and thus anon should be locked by caller.
270 * => If we fail, we unlock everything and error is returned.
271 * => If we are successful, return with everything still locked.
272 * => We do not move the page on the queues [gets moved later]. If we
273 * allocate a new page [we_own], it gets put on the queues. Either way,
274 * the result is that the page is on the queues at return time
275 */
276int
277uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
278 struct vm_anon *anon)
279{
280 struct vm_page *pg;
281 int error;
282
283 KASSERT(rw_lock_held(anon->an_lock))((rw_lock_held(anon->an_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 283, "rw_lock_held(anon->an_lock)"
));
284 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 284, "anon->an_lock == amap->am_lock"
));
285
286 /* Increment the counters.*/
287 counters_inc(uvmexp_counters, flt_anget);
288 if (anon->an_page) {
289 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_minflt++;
290 } else {
291 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_majflt++;
292 }
293 error = 0;
294
295 /*
296 * Loop until we get the anon data, or fail.
297 */
298 for (;;) {
299 boolean_t we_own, locked;
300 /*
301 * Note: 'we_own' will become true if we set PG_BUSY on a page.
302 */
303 we_own = FALSE0;
304 pg = anon->an_page;
305
306 /*
307 * Is page resident? Make sure it is not busy/released.
308 */
309 if (pg) {
310 KASSERT(pg->pg_flags & PQ_ANON)((pg->pg_flags & 0x00100000) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 310, "pg->pg_flags & PQ_ANON"
));
311 KASSERT(pg->uanon == anon)((pg->uanon == anon) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 311, "pg->uanon == anon"));
312
313 /*
314 * if the page is busy, we drop all the locks and
315 * try again.
316 */
317 if ((pg->pg_flags & (PG_BUSY0x00000001|PG_RELEASED0x00000020)) == 0)
318 return (VM_PAGER_OK0);
319 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_WANTED0x00000002);
320 counters_inc(uvmexp_counters, flt_pgwait);
321
322 /*
323 * The last unlock must be an atomic unlock and wait
324 * on the owner of page.
325 */
326 if (pg->uobject) {
327 /* Owner of page is UVM object. */
328 uvmfault_unlockall(ufi, amap, NULL((void *)0));
329 rwsleep_nsec(pg, pg->uobject->vmobjlock,
330 PVM4 | PNORELOCK0x200, "anonget1", INFSLP0xffffffffffffffffULL);
331 } else {
332 /* Owner of page is anon. */
333 uvmfault_unlockall(ufi, NULL((void *)0), NULL((void *)0));
334 rwsleep_nsec(pg, anon->an_lock, PVM4 | PNORELOCK0x200,
335 "anonget2", INFSLP0xffffffffffffffffULL);
336 }
337 } else {
338 /*
339 * No page, therefore allocate one.
340 */
341 pg = uvm_pagealloc(NULL((void *)0), 0, anon, 0);
342 if (pg == NULL((void *)0)) {
343 /* Out of memory. Wait a little. */
344 uvmfault_unlockall(ufi, amap, NULL((void *)0));
345 counters_inc(uvmexp_counters, flt_noram);
346 uvm_wait("flt_noram1");
347 } else {
348 /* PG_BUSY bit is set. */
349 we_own = TRUE1;
350 uvmfault_unlockall(ufi, amap, NULL((void *)0));
351
352 /*
353 * Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into
354 * the uvm_swap_get() function with all data
355 * structures unlocked. Note that it is OK
356 * to read an_swslot here, because we hold
357 * PG_BUSY on the page.
358 */
359 counters_inc(uvmexp_counters, pageins);
360 error = uvm_swap_get(pg, anon->an_swslot,
361 PGO_SYNCIO0x002);
362
363 /*
364 * We clean up after the I/O below in the
365 * 'we_own' case.
366 */
367 }
368 }
369
370 /*
371 * Re-lock the map and anon.
372 */
373 locked = uvmfault_relock(ufi);
374 if (locked || we_own) {
375 rw_enter(anon->an_lock, RW_WRITE0x0001UL);
376 }
377
378 /*
379 * If we own the page (i.e. we set PG_BUSY), then we need
380 * to clean up after the I/O. There are three cases to
381 * consider:
382 *
383 * 1) Page was released during I/O: free anon and ReFault.
384 * 2) I/O not OK. Free the page and cause the fault to fail.
385 * 3) I/O OK! Activate the page and sync with the non-we_own
386 * case (i.e. drop anon lock if not locked).
387 */
388 if (we_own) {
389 if (pg->pg_flags & PG_WANTED0x00000002) {
390 wakeup(pg);
391 }
392
393 /*
394 * if we were RELEASED during I/O, then our anon is
395 * no longer part of an amap. we need to free the
396 * anon and try again.
397 */
398 if (pg->pg_flags & PG_RELEASED0x00000020) {
399 KASSERT(anon->an_ref == 0)((anon->an_ref == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 399, "anon->an_ref == 0"));
400 /*
401 * Released while we had unlocked amap.
402 */
403 if (locked)
404 uvmfault_unlockall(ufi, NULL((void *)0), NULL((void *)0));
405 uvm_anon_release(anon); /* frees page for us */
406 counters_inc(uvmexp_counters, flt_pgrele);
407 return (VM_PAGER_REFAULT7); /* refault! */
408 }
409
410 if (error != VM_PAGER_OK0) {
411 KASSERT(error != VM_PAGER_PEND)((error != 3) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 411, "error != VM_PAGER_PEND"));
412
413 /* remove page from anon */
414 anon->an_page = NULL((void *)0);
415
416 /*
417 * Remove the swap slot from the anon and
418 * mark the anon as having no real slot.
419 * Do not free the swap slot, thus preventing
420 * it from being used again.
421 */
422 uvm_swap_markbad(anon->an_swslot, 1);
423 anon->an_swslot = SWSLOT_BAD(-1);
424
425 /*
426 * Note: page was never !PG_BUSY, so it
427 * cannot be mapped and thus no need to
428 * pmap_page_protect() it.
429 */
430 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
431 uvm_pagefree(pg);
432 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
433
434 if (locked) {
435 uvmfault_unlockall(ufi, NULL((void *)0), NULL((void *)0));
436 }
437 rw_exit(anon->an_lock);
438 return (VM_PAGER_ERROR4);
439 }
440
441 /*
442 * We have successfully read the page, activate it.
443 */
444 pmap_clear_modify(pg)pmap_clear_attrs(pg, 0x0000000000000040UL);
445 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
446 uvm_pageactivate(pg);
447 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
448 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags,
449 PG_WANTED0x00000002|PG_BUSY0x00000001|PG_FAKE0x00000040);
450 UVM_PAGE_OWN(pg, NULL);
451 }
452
453 /*
454 * We were not able to re-lock the map - restart the fault.
455 */
456 if (!locked) {
457 if (we_own) {
458 rw_exit(anon->an_lock);
459 }
460 return (VM_PAGER_REFAULT7);
461 }
462
463 /*
464 * Verify that no one has touched the amap and moved
465 * the anon on us.
466 */
467 if (ufi != NULL((void *)0) && amap_lookup(&ufi->entry->aref,
468 ufi->orig_rvaddr - ufi->entry->start) != anon) {
469
470 uvmfault_unlockall(ufi, amap, NULL((void *)0));
471 return (VM_PAGER_REFAULT7);
472 }
473
474 /*
475 * Retry..
476 */
477 counters_inc(uvmexp_counters, flt_anretry);
478 continue;
479
480 }
481 /*NOTREACHED*/
482}
483
484/*
485 * Update statistics after fault resolution.
486 * - maxrss
487 */
488void
489uvmfault_update_stats(struct uvm_faultinfo *ufi)
490{
491 struct vm_map *map;
492 struct proc *p;
493 vsize_t res;
494
495 map = ufi->orig_map;
496
497 /*
498 * If this is a nested pmap (eg, a virtual machine pmap managed
499 * by vmm(4) on amd64/i386), don't do any updating, just return.
500 *
501 * pmap_nested() on other archs is #defined to 0, so this is a
502 * no-op.
503 */
504 if (pmap_nested(map->pmap)((map->pmap)->pm_type != 1))
505 return;
506
507 /* Update the maxrss for the process. */
508 if (map->flags & VM_MAP_ISVMSPACE0x40) {
509 p = curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc;
510 KASSERT(p != NULL && &p->p_vmspace->vm_map == map)((p != ((void *)0) && &p->p_vmspace->vm_map
== map) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 510, "p != NULL && &p->p_vmspace->vm_map == map"
));
511
512 res = pmap_resident_count(map->pmap)((map->pmap)->pm_stats.resident_count);
513 /* Convert res from pages to kilobytes. */
514 res <<= (PAGE_SHIFT12 - 10);
515
516 if (p->p_ru.ru_maxrss < res)
517 p->p_ru.ru_maxrss = res;
518 }
519}
520
521/*
522 * F A U L T - m a i n e n t r y p o i n t
523 */
524
525/*
526 * uvm_fault: page fault handler
527 *
528 * => called from MD code to resolve a page fault
529 * => VM data structures usually should be unlocked. however, it is
530 * possible to call here with the main map locked if the caller
531 * gets a write lock, sets it recursive, and then calls us (c.f.
532 * uvm_map_pageable). this should be avoided because it keeps
533 * the map locked off during I/O.
534 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
535 */
536#define MASK(entry)((((entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01 | 0x02
| 0x04)) (UVM_ET_ISCOPYONWRITE(entry)(((entry)->etype & 0x0004) != 0) ? \
537 ~PROT_WRITE0x02 : PROT_MASK(0x01 | 0x02 | 0x04))
538struct uvm_faultctx {
539 /*
540 * the following members are set up by uvm_fault_check() and
541 * read-only after that.
542 */
543 vm_prot_t enter_prot;
544 vm_prot_t access_type;
545 vaddr_t startva;
546 int npages;
547 int centeridx;
548 boolean_t narrow;
549 boolean_t wired;
550 paddr_t pa_flags;
551};
552
553int uvm_fault_check(
554 struct uvm_faultinfo *, struct uvm_faultctx *,
555 struct vm_anon ***);
556
557int uvm_fault_upper(
558 struct uvm_faultinfo *, struct uvm_faultctx *,
559 struct vm_anon **, vm_fault_t);
560boolean_t uvm_fault_upper_lookup(
561 struct uvm_faultinfo *, const struct uvm_faultctx *,
562 struct vm_anon **, struct vm_page **);
563
564int uvm_fault_lower(
565 struct uvm_faultinfo *, struct uvm_faultctx *,
566 struct vm_page **, vm_fault_t);
567
568int
569uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type,
570 vm_prot_t access_type)
571{
572 struct uvm_faultinfo ufi;
573 struct uvm_faultctx flt;
574 boolean_t shadowed;
575 struct vm_anon *anons_store[UVM_MAXRANGE16], **anons;
576 struct vm_page *pages[UVM_MAXRANGE16];
577 int error;
578
579 counters_inc(uvmexp_counters, faults);
580 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL)do { extern struct dt_probe (dt_static_uvm_fault); struct dt_probe
*dtp = &(dt_static_uvm_fault); if (__builtin_expect(((dt_tracing
) != 0), 0) && __builtin_expect(((dtp->dtp_recording
) != 0), 0)) { struct dt_provider *dtpv = dtp->dtp_prov; dtpv
->dtpv_enter(dtpv, dtp, vaddr, fault_type, access_type, ((
void *)0)); } } while (0);
581
582 /*
583 * init the IN parameters in the ufi
584 */
585 ufi.orig_map = orig_map;
586 ufi.orig_rvaddr = trunc_page(vaddr)((vaddr) & ~((1 << 12) - 1));
587 ufi.orig_size = PAGE_SIZE(1 << 12); /* can't get any smaller than this */
588 if (fault_type == VM_FAULT_WIRE((vm_fault_t) 0x2))
589 flt.narrow = TRUE1; /* don't look for neighborhood
590 * pages on wire */
591 else
592 flt.narrow = FALSE0; /* normal fault */
593 flt.access_type = access_type;
594
595
596 error = ERESTART-1;
597 while (error == ERESTART-1) { /* ReFault: */
598 anons = anons_store;
599
600 error = uvm_fault_check(&ufi, &flt, &anons);
601 if (error != 0)
602 continue;
603
604 /* True if there is an anon at the faulting address */
605 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
606 if (shadowed == TRUE1) {
607 /* case 1: fault on an anon in our amap */
608 error = uvm_fault_upper(&ufi, &flt, anons, fault_type);
609 } else {
610 struct uvm_object *uobj = ufi.entry->object.uvm_obj;
611
612 /*
613 * if the desired page is not shadowed by the amap and
614 * we have a backing object, then we check to see if
615 * the backing object would prefer to handle the fault
616 * itself (rather than letting us do it with the usual
617 * pgo_get hook). the backing object signals this by
618 * providing a pgo_fault routine.
619 */
620 if (uobj != NULL((void *)0) && uobj->pgops->pgo_fault != NULL((void *)0)) {
621 KERNEL_LOCK()_kernel_lock();
622 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
623 error = uobj->pgops->pgo_fault(&ufi,
624 flt.startva, pages, flt.npages,
625 flt.centeridx, fault_type, flt.access_type,
626 PGO_LOCKED0x040);
627 KERNEL_UNLOCK()_kernel_unlock();
628
629 if (error == VM_PAGER_OK0)
630 error = 0;
631 else if (error == VM_PAGER_REFAULT7)
632 error = ERESTART-1;
633 else
634 error = EACCES13;
635 } else {
636 /* case 2: fault on backing obj or zero fill */
637 error = uvm_fault_lower(&ufi, &flt, pages,
638 fault_type);
639 }
640 }
641 }
642
643 return error;
644}
645
646/*
647 * uvm_fault_check: check prot, handle needs-copy, etc.
648 *
649 * 1. lookup entry.
650 * 2. check protection.
651 * 3. adjust fault condition (mainly for simulated fault).
652 * 4. handle needs-copy (lazy amap copy).
653 * 5. establish range of interest for neighbor fault (aka pre-fault).
654 * 6. look up anons (if amap exists).
655 * 7. flush pages (if MADV_SEQUENTIAL)
656 *
657 * => called with nothing locked.
658 * => if we fail (result != 0) we unlock everything.
659 * => initialize/adjust many members of flt.
660 */
661int
662uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
663 struct vm_anon ***ranons)
664{
665 struct vm_amap *amap;
666 struct uvm_object *uobj;
667 int nback, nforw;
668
669 /*
670 * lookup and lock the maps
671 */
672 if (uvmfault_lookup(ufi, FALSE0) == FALSE0) {
673 return EFAULT14;
674 }
675 /* locked: maps(read) */
676
677#ifdef DIAGNOSTIC1
678 if ((ufi->map->flags & VM_MAP_PAGEABLE0x01) == 0)
679 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)",
680 ufi->map, ufi->orig_rvaddr);
681#endif
682
683 /*
684 * check protection
685 */
686 if ((ufi->entry->protection & flt->access_type) != flt->access_type) {
687 uvmfault_unlockmaps(ufi, FALSE0);
688 return EACCES13;
689 }
690
691 /*
692 * "enter_prot" is the protection we want to enter the page in at.
693 * for certain pages (e.g. copy-on-write pages) this protection can
694 * be more strict than ufi->entry->protection. "wired" means either
695 * the entry is wired or we are fault-wiring the pg.
696 */
697
698 flt->enter_prot = ufi->entry->protection;
699 flt->pa_flags = UVM_ET_ISWC(ufi->entry)(((ufi->entry)->etype & 0x0080) != 0) ? PMAP_WC0x2 : 0;
700 flt->wired = VM_MAPENT_ISWIRED(ufi->entry)((ufi->entry)->wired_count != 0) || (flt->narrow == TRUE1);
701 if (flt->wired)
702 flt->access_type = flt->enter_prot; /* full access for wired */
703
704 /* handle "needs_copy" case. */
705 if (UVM_ET_ISNEEDSCOPY(ufi->entry)(((ufi->entry)->etype & 0x0008) != 0)) {
706 if ((flt->access_type & PROT_WRITE0x02) ||
707 (ufi->entry->object.uvm_obj == NULL((void *)0))) {
708 /* need to clear */
709 uvmfault_unlockmaps(ufi, FALSE0);
710 uvmfault_amapcopy(ufi);
711 counters_inc(uvmexp_counters, flt_amcopy);
712 return ERESTART-1;
713 } else {
714 /*
715 * ensure that we pmap_enter page R/O since
716 * needs_copy is still true
717 */
718 flt->enter_prot &= ~PROT_WRITE0x02;
719 }
720 }
721
722 /*
723 * identify the players
724 */
725 amap = ufi->entry->aref.ar_amap; /* upper layer */
726 uobj = ufi->entry->object.uvm_obj; /* lower layer */
727
728 /*
729 * check for a case 0 fault. if nothing backing the entry then
730 * error now.
731 */
732 if (amap == NULL((void *)0) && uobj == NULL((void *)0)) {
733 uvmfault_unlockmaps(ufi, FALSE0);
734 return EFAULT14;
735 }
736
737 /*
738 * for a case 2B fault waste no time on adjacent pages because
739 * they are likely already entered.
740 */
741 if (uobj != NULL((void *)0) && amap != NULL((void *)0) &&
742 (flt->access_type & PROT_WRITE0x02) != 0) {
743 /* wide fault (!narrow) */
744 flt->narrow = TRUE1;
745 }
746
747 /*
748 * establish range of interest based on advice from mapper
749 * and then clip to fit map entry. note that we only want
750 * to do this the first time through the fault. if we
751 * ReFault we will disable this by setting "narrow" to true.
752 */
753 if (flt->narrow == FALSE0) {
754
755 /* wide fault (!narrow) */
756 nback = min(uvmadvice[ufi->entry->advice].nback,
757 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT12);
758 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT12);
759 nforw = min(uvmadvice[ufi->entry->advice].nforw,
760 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT12) - 1);
761 /*
762 * note: "-1" because we don't want to count the
763 * faulting page as forw
764 */
765 flt->npages = nback + nforw + 1;
766 flt->centeridx = nback;
767
768 flt->narrow = TRUE1; /* ensure only once per-fault */
769 } else {
770 /* narrow fault! */
771 nback = nforw = 0;
772 flt->startva = ufi->orig_rvaddr;
773 flt->npages = 1;
774 flt->centeridx = 0;
775 }
776
777 /*
778 * if we've got an amap then lock it and extract current anons.
779 */
780 if (amap) {
781 amap_lock(amap)rw_enter_write((amap)->am_lock);
782 amap_lookups(&ufi->entry->aref,
783 flt->startva - ufi->entry->start, *ranons, flt->npages);
784 } else {
785 *ranons = NULL((void *)0); /* to be safe */
786 }
787
788 /*
789 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
790 * now and then forget about them (for the rest of the fault).
791 */
792 if (ufi->entry->advice == MADV_SEQUENTIAL2 && nback != 0) {
793 /* flush back-page anons? */
794 if (amap)
795 uvmfault_anonflush(*ranons, nback);
796
797 /*
798 * flush object?
799 */
800 if (uobj) {
801 voff_t uoff;
802
803 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset;
804 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
805 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff +
806 ((vsize_t)nback << PAGE_SHIFT12), PGO_DEACTIVATE0x004);
807 rw_exit(uobj->vmobjlock);
808 }
809
810 /* now forget about the backpages */
811 if (amap)
812 *ranons += nback;
813 flt->startva += ((vsize_t)nback << PAGE_SHIFT12);
814 flt->npages -= nback;
815 flt->centeridx = 0;
816 }
817
818 return 0;
819}
820
821/*
822 * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
823 *
824 * iterate range of interest:
825 * 1. check if h/w mapping exists. if yes, we don't care
826 * 2. check if anon exists. if not, page is lower.
827 * 3. if anon exists, enter h/w mapping for neighbors.
828 *
829 * => called with amap locked (if exists).
830 */
831boolean_t
832uvm_fault_upper_lookup(struct uvm_faultinfo *ufi,
833 const struct uvm_faultctx *flt, struct vm_anon **anons,
834 struct vm_page **pages)
835{
836 struct vm_amap *amap = ufi->entry->aref.ar_amap;
837 struct vm_anon *anon;
838 boolean_t shadowed;
839 vaddr_t currva;
840 paddr_t pa;
841 int lcv;
842
843 /* locked: maps(read), amap(if there) */
844 KASSERT(amap == NULL ||((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 845, "amap == NULL || rw_write_held(amap->am_lock)"))
845 rw_write_held(amap->am_lock))((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 845, "amap == NULL || rw_write_held(amap->am_lock)"));
846
847 /*
848 * map in the backpages and frontpages we found in the amap in hopes
849 * of preventing future faults. we also init the pages[] array as
850 * we go.
851 */
852 currva = flt->startva;
853 shadowed = FALSE0;
854 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE(1 << 12)) {
855 /*
856 * dont play with VAs that are already mapped
857 * except for center)
858 */
859 if (lcv != flt->centeridx &&
860 pmap_extract(ufi->orig_map->pmap, currva, &pa)) {
861 pages[lcv] = PGO_DONTCARE((struct vm_page *) -1L);
862 continue;
863 }
864
865 /*
866 * unmapped or center page. check if any anon at this level.
867 */
868 if (amap == NULL((void *)0) || anons[lcv] == NULL((void *)0)) {
869 pages[lcv] = NULL((void *)0);
870 continue;
871 }
872
873 /*
874 * check for present page and map if possible.
875 */
876 pages[lcv] = PGO_DONTCARE((struct vm_page *) -1L);
877 if (lcv == flt->centeridx) { /* save center for later! */
878 shadowed = TRUE1;
879 continue;
880 }
881 anon = anons[lcv];
882 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 882, "anon->an_lock == amap->am_lock"
));
883 if (anon->an_page &&
884 (anon->an_page->pg_flags & (PG_RELEASED0x00000020|PG_BUSY0x00000001)) == 0) {
885 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
886 uvm_pageactivate(anon->an_page); /* reactivate */
887 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
888 counters_inc(uvmexp_counters, flt_namap);
889
890 /*
891 * Since this isn't the page that's actually faulting,
892 * ignore pmap_enter() failures; it's not critical
893 * that we enter these right now.
894 */
895 (void) pmap_enter(ufi->orig_map->pmap, currva,
896 VM_PAGE_TO_PHYS(anon->an_page)((anon->an_page)->phys_addr) | flt->pa_flags,
897 (anon->an_ref > 1) ?
898 (flt->enter_prot & ~PROT_WRITE0x02) : flt->enter_prot,
899 PMAP_CANFAIL0x00000020 |
900 (VM_MAPENT_ISWIRED(ufi->entry)((ufi->entry)->wired_count != 0) ? PMAP_WIRED0x00000010 : 0));
901 }
902 }
903 if (flt->npages > 1)
904 pmap_update(ufi->orig_map->pmap);
905
906 return shadowed;
907}
908
909/*
910 * uvm_fault_upper: handle upper fault.
911 *
912 * 1. acquire anon lock.
913 * 2. get anon. let uvmfault_anonget do the dirty work.
914 * 3. if COW, promote data to new anon
915 * 4. enter h/w mapping
916 */
917int
918uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
919 struct vm_anon **anons, vm_fault_t fault_type)
920{
921 struct vm_amap *amap = ufi->entry->aref.ar_amap;
922 struct vm_anon *oanon, *anon = anons[flt->centeridx];
923 struct vm_page *pg = NULL((void *)0);
924 int error, ret;
925
926 /* locked: maps(read), amap, anon */
927 KASSERT(rw_write_held(amap->am_lock))((rw_write_held(amap->am_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 927, "rw_write_held(amap->am_lock)"
));
928 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 928, "anon->an_lock == amap->am_lock"
));
929
930 /*
931 * no matter if we have case 1A or case 1B we are going to need to
932 * have the anon's memory resident. ensure that now.
933 */
934 /*
935 * let uvmfault_anonget do the dirty work.
936 * if it fails (!OK) it will unlock everything for us.
937 * if it succeeds, locks are still valid and locked.
938 * also, if it is OK, then the anon's page is on the queues.
939 * if the page is on loan from a uvm_object, then anonget will
940 * lock that object for us if it does not fail.
941 */
942 error = uvmfault_anonget(ufi, amap, anon);
943 switch (error) {
944 case VM_PAGER_OK0:
945 break;
946
947 case VM_PAGER_REFAULT7:
948 return ERESTART-1;
949
950 case VM_PAGER_ERROR4:
951 /*
952 * An error occurred while trying to bring in the
953 * page -- this is the only error we return right
954 * now.
955 */
956 return EACCES13; /* XXX */
957 default:
958#ifdef DIAGNOSTIC1
959 panic("uvm_fault: uvmfault_anonget -> %d", error);
960#else
961 return EACCES13;
962#endif
963 }
964
965 KASSERT(rw_write_held(amap->am_lock))((rw_write_held(amap->am_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 965, "rw_write_held(amap->am_lock)"
));
966 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 966, "anon->an_lock == amap->am_lock"
));
967
968 /*
969 * if we are case 1B then we will need to allocate a new blank
970 * anon to transfer the data into. note that we have a lock
971 * on anon, so no one can busy or release the page until we are done.
972 * also note that the ref count can't drop to zero here because
973 * it is > 1 and we are only dropping one ref.
974 *
975 * in the (hopefully very rare) case that we are out of RAM we
976 * will unlock, wait for more RAM, and refault.
977 *
978 * if we are out of anon VM we wait for RAM to become available.
979 */
980
981 if ((flt->access_type & PROT_WRITE0x02) != 0 && anon->an_ref > 1) {
982 counters_inc(uvmexp_counters, flt_acow);
983 oanon = anon; /* oanon = old */
984 anon = uvm_analloc();
985 if (anon) {
986 anon->an_lock = amap->am_lock;
987 pg = uvm_pagealloc(NULL((void *)0), 0, anon, 0);
988 }
989
990 /* check for out of RAM */
991 if (anon == NULL((void *)0) || pg == NULL((void *)0)) {
992 uvmfault_unlockall(ufi, amap, NULL((void *)0));
993 if (anon == NULL((void *)0))
994 counters_inc(uvmexp_counters, flt_noanon);
995 else {
996 anon->an_lock = NULL((void *)0);
997 anon->an_ref--;
998 uvm_anfree(anon)uvm_anfree_list((anon), ((void *)0));
999 counters_inc(uvmexp_counters, flt_noram);
1000 }
1001
1002 if (uvm_swapisfull())
1003 return ENOMEM12;
1004
1005 /* out of RAM, wait for more */
1006 if (anon == NULL((void *)0))
1007 uvm_anwait();
1008 else
1009 uvm_wait("flt_noram3");
1010 return ERESTART-1;
1011 }
1012
1013 /* got all resources, replace anon with nanon */
1014 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */
1015 /* un-busy! new page */
1016 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001|PG_FAKE0x00000040);
1017 UVM_PAGE_OWN(pg, NULL);
1018 ret = amap_add(&ufi->entry->aref,
1019 ufi->orig_rvaddr - ufi->entry->start, anon, 1);
1020 KASSERT(ret == 0)((ret == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1020, "ret == 0"));
1021
1022 /* deref: can not drop to zero here by defn! */
1023 oanon->an_ref--;
1024
1025#if defined(MULTIPROCESSOR1) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW)
1026 /*
1027 * If there are multiple threads, either uvm or the
1028 * pmap has to make sure no threads see the old RO
1029 * mapping once any have seen the new RW mapping.
1030 * uvm does it by inserting the new mapping RO and
1031 * letting it fault again.
1032 * This is only a problem on MP systems.
1033 */
1034 if (P_HASSIBLING(curproc)((({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc)->p_p->ps_threadcnt > 1)) {
1035 flt->enter_prot &= ~PROT_WRITE0x02;
1036 flt->access_type &= ~PROT_WRITE0x02;
1037 }
1038#endif
1039
1040 /*
1041 * note: anon is _not_ locked, but we have the sole references
1042 * to in from amap.
1043 * thus, no one can get at it until we are done with it.
1044 */
1045 } else {
1046 counters_inc(uvmexp_counters, flt_anon);
1047 oanon = anon;
Value stored to 'oanon' is never read
1048 pg = anon->an_page;
1049 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1050 flt->enter_prot = flt->enter_prot & ~PROT_WRITE0x02;
1051 }
1052
1053 /*
1054 * now map the page in .
1055 */
1056 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1057 VM_PAGE_TO_PHYS(pg)((pg)->phys_addr) | flt->pa_flags, flt->enter_prot,
1058 flt->access_type | PMAP_CANFAIL0x00000020 | (flt->wired ? PMAP_WIRED0x00000010 : 0)) != 0) {
1059 /*
1060 * No need to undo what we did; we can simply think of
1061 * this as the pmap throwing away the mapping information.
1062 *
1063 * We do, however, have to go through the ReFault path,
1064 * as the map may change while we're asleep.
1065 */
1066 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1067 if (uvm_swapisfull()) {
1068 /* XXX instrumentation */
1069 return ENOMEM12;
1070 }
1071 /* XXX instrumentation */
1072 uvm_wait("flt_pmfail1");
1073 return ERESTART-1;
1074 }
1075
1076 /*
1077 * ... update the page queues.
1078 */
1079 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1080
1081 if (fault_type == VM_FAULT_WIRE((vm_fault_t) 0x2)) {
1082 uvm_pagewire(pg);
1083 /*
1084 * since the now-wired page cannot be paged out,
1085 * release its swap resources for others to use.
1086 * since an anon with no swap cannot be PG_CLEAN,
1087 * clear its clean flag now.
1088 */
1089 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1090 uvm_anon_dropswap(anon);
1091 } else {
1092 /* activate it */
1093 uvm_pageactivate(pg);
1094 }
1095
1096 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1097
1098 /*
1099 * done case 1! finish up by unlocking everything and returning success
1100 */
1101 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1102 pmap_update(ufi->orig_map->pmap);
1103 return 0;
1104}
1105
1106/*
1107 * uvm_fault_lower_lookup: look up on-memory uobj pages.
1108 *
1109 * 1. get on-memory pages.
1110 * 2. if failed, give up (get only center page later).
1111 * 3. if succeeded, enter h/w mapping of neighbor pages.
1112 */
1113
1114struct vm_page *
1115uvm_fault_lower_lookup(
1116 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1117 struct vm_page **pages)
1118{
1119 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1120 struct vm_page *uobjpage = NULL((void *)0);
1121 int lcv, gotpages;
1122 vaddr_t currva;
1123
1124 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
1125
1126 counters_inc(uvmexp_counters, flt_lget);
1127 gotpages = flt->npages;
1128 (void) uobj->pgops->pgo_get(uobj,
1129 ufi->entry->offset + (flt->startva - ufi->entry->start),
1130 pages, &gotpages, flt->centeridx,
1131 flt->access_type & MASK(ufi->entry)((((ufi->entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01
| 0x02 | 0x04)), ufi->entry->advice,
1132 PGO_LOCKED0x040);
1133
1134 /*
1135 * check for pages to map, if we got any
1136 */
1137 if (gotpages == 0) {
1138 return NULL((void *)0);
1139 }
1140
1141 currva = flt->startva;
1142 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE(1 << 12)) {
1143 if (pages[lcv] == NULL((void *)0) ||
1144 pages[lcv] == PGO_DONTCARE((struct vm_page *) -1L))
1145 continue;
1146
1147 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0)(((pages[lcv]->pg_flags & 0x00000020) == 0) ? (void)0 :
__assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 1147
, "(pages[lcv]->pg_flags & PG_RELEASED) == 0"));
1148
1149 /*
1150 * if center page is resident and not
1151 * PG_BUSY, then pgo_get made it PG_BUSY
1152 * for us and gave us a handle to it.
1153 * remember this page as "uobjpage."
1154 * (for later use).
1155 */
1156 if (lcv == flt->centeridx) {
1157 uobjpage = pages[lcv];
1158 continue;
1159 }
1160
1161 /*
1162 * note: calling pgo_get with locked data
1163 * structures returns us pages which are
1164 * neither busy nor released, so we don't
1165 * need to check for this. we can just
1166 * directly enter the page (after moving it
1167 * to the head of the active queue [useful?]).
1168 */
1169
1170 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1171 uvm_pageactivate(pages[lcv]); /* reactivate */
1172 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1173 counters_inc(uvmexp_counters, flt_nomap);
1174
1175 /*
1176 * Since this page isn't the page that's
1177 * actually faulting, ignore pmap_enter()
1178 * failures; it's not critical that we
1179 * enter these right now.
1180 */
1181 (void) pmap_enter(ufi->orig_map->pmap, currva,
1182 VM_PAGE_TO_PHYS(pages[lcv])((pages[lcv])->phys_addr) | flt->pa_flags,
1183 flt->enter_prot & MASK(ufi->entry)((((ufi->entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01
| 0x02 | 0x04)),
1184 PMAP_CANFAIL0x00000020 |
1185 (flt->wired ? PMAP_WIRED0x00000010 : 0));
1186
1187 /*
1188 * NOTE: page can't be PG_WANTED because
1189 * we've held the lock the whole time
1190 * we've had the handle.
1191 */
1192 atomic_clearbits_intx86_atomic_clearbits_u32(&pages[lcv]->pg_flags, PG_BUSY0x00000001);
1193 UVM_PAGE_OWN(pages[lcv], NULL);
1194 }
1195 pmap_update(ufi->orig_map->pmap);
1196
1197 return uobjpage;
1198}
1199
1200/*
1201 * uvm_fault_lower: handle lower fault.
1202 *
1203 */
1204int
1205uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1206 struct vm_page **pages, vm_fault_t fault_type)
1207{
1208 struct vm_amap *amap = ufi->entry->aref.ar_amap;
1209 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1210 boolean_t promote, locked;
1211 int result;
1212 struct vm_page *uobjpage, *pg = NULL((void *)0);
1213 struct vm_anon *anon = NULL((void *)0);
1214 voff_t uoff;
1215
1216 /*
1217 * now, if the desired page is not shadowed by the amap and we have
1218 * a backing object that does not have a special fault routine, then
1219 * we ask (with pgo_get) the object for resident pages that we care
1220 * about and attempt to map them in. we do not let pgo_get block
1221 * (PGO_LOCKED).
1222 */
1223 if (uobj == NULL((void *)0)) {
1224 /* zero fill; don't care neighbor pages */
1225 uobjpage = NULL((void *)0);
1226 } else {
1227 uobjpage = uvm_fault_lower_lookup(ufi, flt, pages);
1228 }
1229
1230 /*
1231 * note that at this point we are done with any front or back pages.
1232 * we are now going to focus on the center page (i.e. the one we've
1233 * faulted on). if we have faulted on the bottom (uobj)
1234 * layer [i.e. case 2] and the page was both present and available,
1235 * then we've got a pointer to it as "uobjpage" and we've already
1236 * made it BUSY.
1237 */
1238
1239 /*
1240 * locked:
1241 */
1242 KASSERT(amap == NULL ||((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1243, "amap == NULL || rw_write_held(amap->am_lock)"))
1243 rw_write_held(amap->am_lock))((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1243, "amap == NULL || rw_write_held(amap->am_lock)"));
1244 KASSERT(uobj == NULL ||((uobj == ((void *)0) || rw_write_held(uobj->vmobjlock)) ?
(void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1245, "uobj == NULL || rw_write_held(uobj->vmobjlock)"))
1245 rw_write_held(uobj->vmobjlock))((uobj == ((void *)0) || rw_write_held(uobj->vmobjlock)) ?
(void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1245, "uobj == NULL || rw_write_held(uobj->vmobjlock)"));
1246
1247 /*
1248 * note that uobjpage can not be PGO_DONTCARE at this point. we now
1249 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1250 * have a backing object, check and see if we are going to promote
1251 * the data up to an anon during the fault.
1252 */
1253 if (uobj == NULL((void *)0)) {
1254 uobjpage = PGO_DONTCARE((struct vm_page *) -1L);
1255 promote = TRUE1; /* always need anon here */
1256 } else {
1257 KASSERT(uobjpage != PGO_DONTCARE)((uobjpage != ((struct vm_page *) -1L)) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 1257, "uobjpage != PGO_DONTCARE"
));
1258 promote = (flt->access_type & PROT_WRITE0x02) &&
1259 UVM_ET_ISCOPYONWRITE(ufi->entry)(((ufi->entry)->etype & 0x0004) != 0);
1260 }
1261
1262 /*
1263 * if uobjpage is not null then we do not need to do I/O to get the
1264 * uobjpage.
1265 *
1266 * if uobjpage is null, then we need to ask the pager to
1267 * get the data for us. once we have the data, we need to reverify
1268 * the state the world. we are currently not holding any resources.
1269 */
1270 if (uobjpage) {
1271 /* update rusage counters */
1272 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_minflt++;
1273 } else {
1274 int gotpages;
1275
1276 /* update rusage counters */
1277 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_majflt++;
1278
1279 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1280
1281 counters_inc(uvmexp_counters, flt_get);
1282 gotpages = 1;
1283 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1284 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1285 0, flt->access_type & MASK(ufi->entry)((((ufi->entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01
| 0x02 | 0x04)), ufi->entry->advice,
1286 PGO_SYNCIO0x002);
1287
1288 /*
1289 * recover from I/O
1290 */
1291 if (result != VM_PAGER_OK0) {
1292 KASSERT(result != VM_PAGER_PEND)((result != 3) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1292, "result != VM_PAGER_PEND"));
1293
1294 if (result == VM_PAGER_AGAIN5) {
1295 tsleep_nsec(&nowake, PVM4, "fltagain2",
1296 MSEC_TO_NSEC(5));
1297 return ERESTART-1;
1298 }
1299
1300 if (!UVM_ET_ISNOFAULT(ufi->entry)(((ufi->entry)->etype & 0x0020) != 0))
1301 return (EIO5);
1302
1303 uobjpage = PGO_DONTCARE((struct vm_page *) -1L);
1304 uobj = NULL((void *)0);
1305 promote = TRUE1;
1306 }
1307
1308 /* re-verify the state of the world. */
1309 locked = uvmfault_relock(ufi);
1310 if (locked && amap != NULL((void *)0))
1311 amap_lock(amap)rw_enter_write((amap)->am_lock);
1312
1313 /* might be changed */
1314 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1315 uobj = uobjpage->uobject;
1316 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
1317 }
1318
1319 /*
1320 * Re-verify that amap slot is still free. if there is
1321 * a problem, we clean up.
1322 */
1323 if (locked && amap && amap_lookup(&ufi->entry->aref,
1324 ufi->orig_rvaddr - ufi->entry->start)) {
1325 if (locked)
1326 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1327 locked = FALSE0;
1328 }
1329
1330 /* didn't get the lock? release the page and retry. */
1331 if (locked == FALSE0 && uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1332 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1333 /* make sure it is in queues */
1334 uvm_pageactivate(uobjpage);
1335 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1336
1337 if (uobjpage->pg_flags & PG_WANTED0x00000002)
1338 /* still holding object lock */
1339 wakeup(uobjpage);
1340 atomic_clearbits_intx86_atomic_clearbits_u32(&uobjpage->pg_flags,
1341 PG_BUSY0x00000001|PG_WANTED0x00000002);
1342 UVM_PAGE_OWN(uobjpage, NULL);
1343 }
1344
1345 if (locked == FALSE0) {
1346 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L))
1347 rw_exit(uobj->vmobjlock);
1348 return ERESTART-1;
1349 }
1350
1351 /*
1352 * we have the data in uobjpage which is PG_BUSY
1353 */
1354 }
1355
1356 /*
1357 * notes:
1358 * - at this point uobjpage can not be NULL
1359 * - at this point uobjpage could be PG_WANTED (handle later)
1360 */
1361 if (promote == FALSE0) {
1362 /*
1363 * we are not promoting. if the mapping is COW ensure that we
1364 * don't give more access than we should (e.g. when doing a read
1365 * fault on a COPYONWRITE mapping we want to map the COW page in
1366 * R/O even though the entry protection could be R/W).
1367 *
1368 * set "pg" to the page we want to map in (uobjpage, usually)
1369 */
1370 counters_inc(uvmexp_counters, flt_obj);
1371 if (UVM_ET_ISCOPYONWRITE(ufi->entry)(((ufi->entry)->etype & 0x0004) != 0))
1372 flt->enter_prot &= ~PROT_WRITE0x02;
1373 pg = uobjpage; /* map in the actual object */
1374
1375 /* assert(uobjpage != PGO_DONTCARE) */
1376
1377 /*
1378 * we are faulting directly on the page.
1379 */
1380 } else {
1381 /*
1382 * if we are going to promote the data to an anon we
1383 * allocate a blank anon here and plug it into our amap.
1384 */
1385#ifdef DIAGNOSTIC1
1386 if (amap == NULL((void *)0))
1387 panic("uvm_fault: want to promote data, but no anon");
1388#endif
1389
1390 anon = uvm_analloc();
1391 if (anon) {
1392 /*
1393 * In `Fill in data...' below, if
1394 * uobjpage == PGO_DONTCARE, we want
1395 * a zero'd, dirty page, so have
1396 * uvm_pagealloc() do that for us.
1397 */
1398 anon->an_lock = amap->am_lock;
1399 pg = uvm_pagealloc(NULL((void *)0), 0, anon,
1400 (uobjpage == PGO_DONTCARE((struct vm_page *) -1L)) ? UVM_PGA_ZERO0x0002 : 0);
1401 }
1402
1403 /*
1404 * out of memory resources?
1405 */
1406 if (anon == NULL((void *)0) || pg == NULL((void *)0)) {
1407 /*
1408 * arg! must unbusy our page and fail or sleep.
1409 */
1410 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1411 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1412 uvm_pageactivate(uobjpage);
1413 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1414
1415 if (uobjpage->pg_flags & PG_WANTED0x00000002)
1416 wakeup(uobjpage);
1417 atomic_clearbits_intx86_atomic_clearbits_u32(&uobjpage->pg_flags,
1418 PG_BUSY0x00000001|PG_WANTED0x00000002);
1419 UVM_PAGE_OWN(uobjpage, NULL);
1420 }
1421
1422 /* unlock and fail ... */
1423 uvmfault_unlockall(ufi, amap, uobj);
1424 if (anon == NULL((void *)0))
1425 counters_inc(uvmexp_counters, flt_noanon);
1426 else {
1427 anon->an_lock = NULL((void *)0);
1428 anon->an_ref--;
1429 uvm_anfree(anon)uvm_anfree_list((anon), ((void *)0));
1430 counters_inc(uvmexp_counters, flt_noram);
1431 }
1432
1433 if (uvm_swapisfull())
1434 return (ENOMEM12);
1435
1436 /* out of RAM, wait for more */
1437 if (anon == NULL((void *)0))
1438 uvm_anwait();
1439 else
1440 uvm_wait("flt_noram5");
1441 return ERESTART-1;
1442 }
1443
1444 /*
1445 * fill in the data
1446 */
1447 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1448 counters_inc(uvmexp_counters, flt_prcopy);
1449 /* copy page [pg now dirty] */
1450 uvm_pagecopy(uobjpage, pg);
1451
1452 /*
1453 * promote to shared amap? make sure all sharing
1454 * procs see it
1455 */
1456 if ((amap_flags(amap)((amap)->am_flags) & AMAP_SHARED0x1) != 0) {
1457 pmap_page_protect(uobjpage, PROT_NONE0x00);
1458 }
1459#if defined(MULTIPROCESSOR1) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW)
1460 /*
1461 * Otherwise:
1462 * If there are multiple threads, either uvm or the
1463 * pmap has to make sure no threads see the old RO
1464 * mapping once any have seen the new RW mapping.
1465 * uvm does it here by forcing it to PROT_NONE before
1466 * inserting the new mapping.
1467 */
1468 else if (P_HASSIBLING(curproc)((({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc)->p_p->ps_threadcnt > 1)) {
1469 pmap_page_protect(uobjpage, PROT_NONE0x00);
1470 }
1471#endif
1472
1473 /* dispose of uobjpage. drop handle to uobj as well. */
1474 if (uobjpage->pg_flags & PG_WANTED0x00000002)
1475 wakeup(uobjpage);
1476 atomic_clearbits_intx86_atomic_clearbits_u32(&uobjpage->pg_flags,
1477 PG_BUSY0x00000001|PG_WANTED0x00000002);
1478 UVM_PAGE_OWN(uobjpage, NULL);
1479 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1480 uvm_pageactivate(uobjpage);
1481 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1482 rw_exit(uobj->vmobjlock);
1483 uobj = NULL((void *)0);
1484 } else {
1485 counters_inc(uvmexp_counters, flt_przero);
1486 /*
1487 * Page is zero'd and marked dirty by uvm_pagealloc()
1488 * above.
1489 */
1490 }
1491
1492 if (amap_add(&ufi->entry->aref,
1493 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) {
1494 uvmfault_unlockall(ufi, amap, uobj);
1495 uvm_anfree(anon)uvm_anfree_list((anon), ((void *)0));
1496 counters_inc(uvmexp_counters, flt_noamap);
1497
1498 if (uvm_swapisfull())
1499 return (ENOMEM12);
1500
1501 amap_populate(&ufi->entry->aref,
1502 ufi->orig_rvaddr - ufi->entry->start);
1503 return ERESTART-1;
1504 }
1505 }
1506
1507 /* note: pg is either the uobjpage or the new page in the new anon */
1508 /*
1509 * all resources are present. we can now map it in and free our
1510 * resources.
1511 */
1512 if (amap == NULL((void *)0))
1513 KASSERT(anon == NULL)((anon == ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1513, "anon == NULL"));
1514 else {
1515 KASSERT(rw_write_held(amap->am_lock))((rw_write_held(amap->am_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 1515, "rw_write_held(amap->am_lock)"
));
1516 KASSERT(anon == NULL || anon->an_lock == amap->am_lock)((anon == ((void *)0) || anon->an_lock == amap->am_lock
) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1516, "anon == NULL || anon->an_lock == amap->am_lock"
));
1517 }
1518 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1519 VM_PAGE_TO_PHYS(pg)((pg)->phys_addr) | flt->pa_flags, flt->enter_prot,
1520 flt->access_type | PMAP_CANFAIL0x00000020 | (flt->wired ? PMAP_WIRED0x00000010 : 0)) != 0) {
1521 /*
1522 * No need to undo what we did; we can simply think of
1523 * this as the pmap throwing away the mapping information.
1524 *
1525 * We do, however, have to go through the ReFault path,
1526 * as the map may change while we're asleep.
1527 */
1528 if (pg->pg_flags & PG_WANTED0x00000002)
1529 wakeup(pg);
1530
1531 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001|PG_FAKE0x00000040|PG_WANTED0x00000002);
1532 UVM_PAGE_OWN(pg, NULL);
1533 uvmfault_unlockall(ufi, amap, uobj);
1534 if (uvm_swapisfull()) {
1535 /* XXX instrumentation */
1536 return (ENOMEM12);
1537 }
1538 /* XXX instrumentation */
1539 uvm_wait("flt_pmfail2");
1540 return ERESTART-1;
1541 }
1542
1543 if (fault_type == VM_FAULT_WIRE((vm_fault_t) 0x2)) {
1544 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1545 uvm_pagewire(pg);
1546 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1547 if (pg->pg_flags & PQ_AOBJ0x00200000) {
1548 /*
1549 * since the now-wired page cannot be paged out,
1550 * release its swap resources for others to use.
1551 * since an aobj page with no swap cannot be clean,
1552 * mark it dirty now.
1553 *
1554 * use pg->uobject here. if the page is from a
1555 * tmpfs vnode, the pages are backed by its UAO and
1556 * not the vnode.
1557 */
1558 KASSERT(uobj != NULL)((uobj != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1558, "uobj != NULL"));
1559 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock)((uobj->vmobjlock == pg->uobject->vmobjlock) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 1559
, "uobj->vmobjlock == pg->uobject->vmobjlock"));
1560 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1561 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT12);
1562 }
1563 } else {
1564 /* activate it */
1565 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1566 uvm_pageactivate(pg);
1567 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1568 }
1569
1570 if (pg->pg_flags & PG_WANTED0x00000002)
1571 wakeup(pg);
1572
1573 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001|PG_FAKE0x00000040|PG_WANTED0x00000002);
1574 UVM_PAGE_OWN(pg, NULL);
1575 uvmfault_unlockall(ufi, amap, uobj);
1576 pmap_update(ufi->orig_map->pmap);
1577
1578 return (0);
1579}
1580
1581
1582/*
1583 * uvm_fault_wire: wire down a range of virtual addresses in a map.
1584 *
1585 * => map may be read-locked by caller, but MUST NOT be write-locked.
1586 * => if map is read-locked, any operations which may cause map to
1587 * be write-locked in uvm_fault() must be taken care of by
1588 * the caller. See uvm_map_pageable().
1589 */
1590int
1591uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type)
1592{
1593 vaddr_t va;
1594 int rv;
1595
1596 /*
1597 * now fault it in a page at a time. if the fault fails then we have
1598 * to undo what we have done. note that in uvm_fault PROT_NONE
1599 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1600 */
1601 for (va = start ; va < end ; va += PAGE_SIZE(1 << 12)) {
1602 rv = uvm_fault(map, va, VM_FAULT_WIRE((vm_fault_t) 0x2), access_type);
1603 if (rv) {
1604 if (va != start) {
1605 uvm_fault_unwire(map, start, va);
1606 }
1607 return (rv);
1608 }
1609 }
1610
1611 return (0);
1612}
1613
1614/*
1615 * uvm_fault_unwire(): unwire range of virtual space.
1616 */
1617void
1618uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end)
1619{
1620
1621 vm_map_lock_read(map)vm_map_lock_read_ln(map, "/usr/src/sys/uvm/uvm_fault.c", 1621
);
1622 uvm_fault_unwire_locked(map, start, end);
1623 vm_map_unlock_read(map)vm_map_unlock_read_ln(map, "/usr/src/sys/uvm/uvm_fault.c", 1623
);
1624}
1625
1626/*
1627 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
1628 *
1629 * => map must be at least read-locked.
1630 */
1631void
1632uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end)
1633{
1634 vm_map_entry_t entry, oentry = NULL((void *)0), next;
1635 pmap_t pmap = vm_map_pmap(map)((map)->pmap);
1636 vaddr_t va;
1637 paddr_t pa;
1638 struct vm_page *pg;
1639
1640 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0)(((map->flags & 0x02) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 1640, "(map->flags & VM_MAP_INTRSAFE) == 0"
));
1641 vm_map_assert_anylock(map)vm_map_assert_anylock_ln(map, "/usr/src/sys/uvm/uvm_fault.c",
1641);
1642
1643 /*
1644 * we assume that the area we are unwiring has actually been wired
1645 * in the first place. this means that we should be able to extract
1646 * the PAs from the pmap.
1647 */
1648
1649 /*
1650 * find the beginning map entry for the region.
1651 */
1652 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map))((start >= ((map)->min_offset) && end <= ((map
)->max_offset)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1652, "start >= vm_map_min(map) && end <= vm_map_max(map)"
));
1653 if (uvm_map_lookup_entry(map, start, &entry) == FALSE0)
1654 panic("uvm_fault_unwire_locked: address not in map");
1655
1656 for (va = start; va < end ; va += PAGE_SIZE(1 << 12)) {
1657 if (pmap_extract(pmap, va, &pa) == FALSE0)
1658 continue;
1659
1660 /*
1661 * find the map entry for the current address.
1662 */
1663 KASSERT(va >= entry->start)((va >= entry->start) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 1663, "va >= entry->start"
));
1664 while (entry && va >= entry->end) {
1665 next = RBT_NEXT(uvm_map_addr, entry)uvm_map_addr_RBT_NEXT(entry);
1666 entry = next;
1667 }
1668
1669 if (entry == NULL((void *)0))
1670 return;
1671 if (va < entry->start)
1672 continue;
1673
1674 /*
1675 * lock it.
1676 */
1677 if (entry != oentry) {
1678 if (oentry != NULL((void *)0)) {
1679 uvm_map_unlock_entry(oentry);
1680 }
1681 uvm_map_lock_entry(entry);
1682 oentry = entry;
1683 }
1684
1685 /*
1686 * if the entry is no longer wired, tell the pmap.
1687 */
1688 if (VM_MAPENT_ISWIRED(entry)((entry)->wired_count != 0) == 0)
1689 pmap_unwire(pmap, va);
1690
1691 pg = PHYS_TO_VM_PAGE(pa);
1692 if (pg) {
1693 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1694 uvm_pageunwire(pg);
1695 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1696 }
1697 }
1698
1699 if (oentry != NULL((void *)0)) {
1700 uvm_map_unlock_entry(oentry);
1701 }
1702}
1703
1704/*
1705 * uvmfault_unlockmaps: unlock the maps
1706 */
1707void
1708uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked)
1709{
1710 /*
1711 * ufi can be NULL when this isn't really a fault,
1712 * but merely paging in anon data.
1713 */
1714 if (ufi == NULL((void *)0)) {
1715 return;
1716 }
1717
1718 uvmfault_update_stats(ufi);
1719 if (write_locked) {
1720 vm_map_unlock(ufi->map)vm_map_unlock_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c",
1720);
1721 } else {
1722 vm_map_unlock_read(ufi->map)vm_map_unlock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1722);
1723 }
1724}
1725
1726/*
1727 * uvmfault_unlockall: unlock everything passed in.
1728 *
1729 * => maps must be read-locked (not write-locked).
1730 */
1731void
1732uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap,
1733 struct uvm_object *uobj)
1734{
1735 if (uobj)
1736 rw_exit(uobj->vmobjlock);
1737 if (amap != NULL((void *)0))
1738 amap_unlock(amap)rw_exit_write((amap)->am_lock);
1739 uvmfault_unlockmaps(ufi, FALSE0);
1740}
1741
1742/*
1743 * uvmfault_lookup: lookup a virtual address in a map
1744 *
1745 * => caller must provide a uvm_faultinfo structure with the IN
1746 * params properly filled in
1747 * => we will lookup the map entry (handling submaps) as we go
1748 * => if the lookup is a success we will return with the maps locked
1749 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only
1750 * get a read lock.
1751 * => note that submaps can only appear in the kernel and they are
1752 * required to use the same virtual addresses as the map they
1753 * are referenced by (thus address translation between the main
1754 * map and the submap is unnecessary).
1755 */
1756
1757boolean_t
1758uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock)
1759{
1760 vm_map_t tmpmap;
1761
1762 /*
1763 * init ufi values for lookup.
1764 */
1765 ufi->map = ufi->orig_map;
1766 ufi->size = ufi->orig_size;
1767
1768 /*
1769 * keep going down levels until we are done. note that there can
1770 * only be two levels so we won't loop very long.
1771 */
1772 while (1) {
1773 if (ufi->orig_rvaddr < ufi->map->min_offset ||
1774 ufi->orig_rvaddr >= ufi->map->max_offset)
1775 return FALSE0;
1776
1777 /* lock map */
1778 if (write_lock) {
1779 vm_map_lock(ufi->map)vm_map_lock_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c", 1779
);
1780 } else {
1781 vm_map_lock_read(ufi->map)vm_map_lock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1781);
1782 }
1783
1784 /* lookup */
1785 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr,
1786 &ufi->entry)) {
1787 uvmfault_unlockmaps(ufi, write_lock);
1788 return FALSE0;
1789 }
1790
1791 /* reduce size if necessary */
1792 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size)
1793 ufi->size = ufi->entry->end - ufi->orig_rvaddr;
1794
1795 /*
1796 * submap? replace map with the submap and lookup again.
1797 * note: VAs in submaps must match VAs in main map.
1798 */
1799 if (UVM_ET_ISSUBMAP(ufi->entry)(((ufi->entry)->etype & 0x0002) != 0)) {
1800 tmpmap = ufi->entry->object.sub_map;
1801 uvmfault_unlockmaps(ufi, write_lock);
1802 ufi->map = tmpmap;
1803 continue;
1804 }
1805
1806 /*
1807 * got it!
1808 */
1809 ufi->mapv = ufi->map->timestamp;
1810 return TRUE1;
1811
1812 } /* while loop */
1813
1814 /*NOTREACHED*/
1815}
1816
1817/*
1818 * uvmfault_relock: attempt to relock the same version of the map
1819 *
1820 * => fault data structures should be unlocked before calling.
1821 * => if a success (TRUE) maps will be locked after call.
1822 */
1823boolean_t
1824uvmfault_relock(struct uvm_faultinfo *ufi)
1825{
1826 /*
1827 * ufi can be NULL when this isn't really a fault,
1828 * but merely paging in anon data.
1829 */
1830 if (ufi == NULL((void *)0)) {
1831 return TRUE1;
1832 }
1833
1834 counters_inc(uvmexp_counters, flt_relck);
1835
1836 /*
1837 * relock map. fail if version mismatch (in which case nothing
1838 * gets locked).
1839 */
1840 vm_map_lock_read(ufi->map)vm_map_lock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1840);
1841 if (ufi->mapv != ufi->map->timestamp) {
1842 vm_map_unlock_read(ufi->map)vm_map_unlock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1842);
1843 return FALSE0;
1844 }
1845
1846 counters_inc(uvmexp_counters, flt_relckok);
1847 return TRUE1; /* got it! */
1848}