Bug Summary

File:uvm/uvm_fault.c
Warning:line 914, column 33
Array access (from variable 'anons') results in a null pointer dereference

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name uvm_fault.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/uvm/uvm_fault.c
1/* $OpenBSD: uvm_fault.c,v 1.124 2021/12/28 13:16:28 mpi Exp $ */
2/* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */
3
4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
29 */
30
31/*
32 * uvm_fault.c: fault handler
33 */
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/percpu.h>
39#include <sys/proc.h>
40#include <sys/malloc.h>
41#include <sys/mman.h>
42#include <sys/tracepoint.h>
43
44#include <uvm/uvm.h>
45
46/*
47 *
48 * a word on page faults:
49 *
50 * types of page faults we handle:
51 *
52 * CASE 1: upper layer faults CASE 2: lower layer faults
53 *
54 * CASE 1A CASE 1B CASE 2A CASE 2B
55 * read/write1 write>1 read/write +-cow_write/zero
56 * | | | |
57 * +--|--+ +--|--+ +-----+ + | + | +-----+
58 * amap | V | | ---------> new | | | | ^ |
59 * +-----+ +-----+ +-----+ + | + | +--|--+
60 * | | |
61 * +-----+ +-----+ +--|--+ | +--|--+
62 * uobj | d/c | | d/c | | V | +----+ |
63 * +-----+ +-----+ +-----+ +-----+
64 *
65 * d/c = don't care
66 *
67 * case [0]: layerless fault
68 * no amap or uobj is present. this is an error.
69 *
70 * case [1]: upper layer fault [anon active]
71 * 1A: [read] or [write with anon->an_ref == 1]
72 * I/O takes place in upper level anon and uobj is not touched.
73 * 1B: [write with anon->an_ref > 1]
74 * new anon is alloc'd and data is copied off ["COW"]
75 *
76 * case [2]: lower layer fault [uobj]
77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
78 * I/O takes place directly in object.
79 * 2B: [write to copy_on_write] or [read on NULL uobj]
80 * data is "promoted" from uobj to a new anon.
81 * if uobj is null, then we zero fill.
82 *
83 * we follow the standard UVM locking protocol ordering:
84 *
85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
86 * we hold a PG_BUSY page if we unlock for I/O
87 *
88 *
89 * the code is structured as follows:
90 *
91 * - init the "IN" params in the ufi structure
92 * ReFault: (ERESTART returned to the loop in uvm_fault)
93 * - do lookups [locks maps], check protection, handle needs_copy
94 * - check for case 0 fault (error)
95 * - establish "range" of fault
96 * - if we have an amap lock it and extract the anons
97 * - if sequential advice deactivate pages behind us
98 * - at the same time check pmap for unmapped areas and anon for pages
99 * that we could map in (and do map it if found)
100 * - check object for resident pages that we could map in
101 * - if (case 2) goto Case2
102 * - >>> handle case 1
103 * - ensure source anon is resident in RAM
104 * - if case 1B alloc new anon and copy from source
105 * - map the correct page in
106 * Case2:
107 * - >>> handle case 2
108 * - ensure source page is resident (if uobj)
109 * - if case 2B alloc new anon and copy from source (could be zero
110 * fill if uobj == NULL)
111 * - map the correct page in
112 * - done!
113 *
114 * note on paging:
115 * if we have to do I/O we place a PG_BUSY page in the correct object,
116 * unlock everything, and do the I/O. when I/O is done we must reverify
117 * the state of the world before assuming that our data structures are
118 * valid. [because mappings could change while the map is unlocked]
119 *
120 * alternative 1: unbusy the page in question and restart the page fault
121 * from the top (ReFault). this is easy but does not take advantage
122 * of the information that we already have from our previous lookup,
123 * although it is possible that the "hints" in the vm_map will help here.
124 *
125 * alternative 2: the system already keeps track of a "version" number of
126 * a map. [i.e. every time you write-lock a map (e.g. to change a
127 * mapping) you bump the version number up by one...] so, we can save
128 * the version number of the map before we release the lock and start I/O.
129 * then when I/O is done we can relock and check the version numbers
130 * to see if anything changed. this might save us some over 1 because
131 * we don't have to unbusy the page and may be less compares(?).
132 *
133 * alternative 3: put in backpointers or a way to "hold" part of a map
134 * in place while I/O is in progress. this could be complex to
135 * implement (especially with structures like amap that can be referenced
136 * by multiple map entries, and figuring out what should wait could be
137 * complex as well...).
138 *
139 * we use alternative 2. given that we are multi-threaded now we may want
140 * to reconsider the choice.
141 */
142
143/*
144 * local data structures
145 */
146struct uvm_advice {
147 int nback;
148 int nforw;
149};
150
151/*
152 * page range array: set up in uvmfault_init().
153 */
154static struct uvm_advice uvmadvice[MADV_MASK0x7 + 1];
155
156#define UVM_MAXRANGE16 16 /* must be max() of nback+nforw+1 */
157
158/*
159 * private prototypes
160 */
161static void uvmfault_amapcopy(struct uvm_faultinfo *);
162static inline void uvmfault_anonflush(struct vm_anon **, int);
163void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t);
164void uvmfault_update_stats(struct uvm_faultinfo *);
165
166/*
167 * inline functions
168 */
169/*
170 * uvmfault_anonflush: try and deactivate pages in specified anons
171 *
172 * => does not have to deactivate page if it is busy
173 */
174static inline void
175uvmfault_anonflush(struct vm_anon **anons, int n)
176{
177 int lcv;
178 struct vm_page *pg;
179
180 for (lcv = 0; lcv < n; lcv++) {
181 if (anons[lcv] == NULL((void *)0))
182 continue;
183 KASSERT(rw_lock_held(anons[lcv]->an_lock))((rw_lock_held(anons[lcv]->an_lock)) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 183, "rw_lock_held(anons[lcv]->an_lock)"
));
184 pg = anons[lcv]->an_page;
185 if (pg && (pg->pg_flags & PG_BUSY0x00000001) == 0) {
186 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
187 if (pg->wire_count == 0) {
188 pmap_page_protect(pg, PROT_NONE0x00);
189 uvm_pagedeactivate(pg);
190 }
191 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
192 }
193 }
194}
195
196/*
197 * normal functions
198 */
199/*
200 * uvmfault_init: compute proper values for the uvmadvice[] array.
201 */
202void
203uvmfault_init(void)
204{
205 int npages;
206
207 npages = atop(16384)((16384) >> 12);
208 if (npages > 0) {
209 KASSERT(npages <= UVM_MAXRANGE / 2)((npages <= 16 / 2) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 209, "npages <= UVM_MAXRANGE / 2"));
210 uvmadvice[MADV_NORMAL0].nforw = npages;
211 uvmadvice[MADV_NORMAL0].nback = npages - 1;
212 }
213
214 npages = atop(32768)((32768) >> 12);
215 if (npages > 0) {
216 KASSERT(npages <= UVM_MAXRANGE / 2)((npages <= 16 / 2) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 216, "npages <= UVM_MAXRANGE / 2"));
217 uvmadvice[MADV_SEQUENTIAL2].nforw = npages - 1;
218 uvmadvice[MADV_SEQUENTIAL2].nback = npages;
219 }
220}
221
222/*
223 * uvmfault_amapcopy: clear "needs_copy" in a map.
224 *
225 * => called with VM data structures unlocked (usually, see below)
226 * => we get a write lock on the maps and clear needs_copy for a VA
227 * => if we are out of RAM we sleep (waiting for more)
228 */
229static void
230uvmfault_amapcopy(struct uvm_faultinfo *ufi)
231{
232 for (;;) {
233 /*
234 * no mapping? give up.
235 */
236 if (uvmfault_lookup(ufi, TRUE1) == FALSE0)
237 return;
238
239 /*
240 * copy if needed.
241 */
242 if (UVM_ET_ISNEEDSCOPY(ufi->entry)(((ufi->entry)->etype & 0x0008) != 0))
243 amap_copy(ufi->map, ufi->entry, M_NOWAIT0x0002,
244 UVM_ET_ISSTACK(ufi->entry)(((ufi->entry)->etype & 0x0040) != 0) ? FALSE0 : TRUE1,
245 ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
246
247 /*
248 * didn't work? must be out of RAM. unlock and sleep.
249 */
250 if (UVM_ET_ISNEEDSCOPY(ufi->entry)(((ufi->entry)->etype & 0x0008) != 0)) {
251 uvmfault_unlockmaps(ufi, TRUE1);
252 uvm_wait("fltamapcopy");
253 continue;
254 }
255
256 /*
257 * got it! unlock and return.
258 */
259 uvmfault_unlockmaps(ufi, TRUE1);
260 return;
261 }
262 /*NOTREACHED*/
263}
264
265/*
266 * uvmfault_anonget: get data in an anon into a non-busy, non-released
267 * page in that anon.
268 *
269 * => Map, amap and thus anon should be locked by caller.
270 * => If we fail, we unlock everything and error is returned.
271 * => If we are successful, return with everything still locked.
272 * => We do not move the page on the queues [gets moved later]. If we
273 * allocate a new page [we_own], it gets put on the queues. Either way,
274 * the result is that the page is on the queues at return time
275 */
276int
277uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
278 struct vm_anon *anon)
279{
280 struct vm_page *pg;
281 int error;
282
283 KASSERT(rw_lock_held(anon->an_lock))((rw_lock_held(anon->an_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 283, "rw_lock_held(anon->an_lock)"
));
284 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 284, "anon->an_lock == amap->am_lock"
));
285
286 /* Increment the counters.*/
287 counters_inc(uvmexp_counters, flt_anget);
288 if (anon->an_page) {
289 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_minflt++;
290 } else {
291 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_majflt++;
292 }
293 error = 0;
294
295 /*
296 * Loop until we get the anon data, or fail.
297 */
298 for (;;) {
299 boolean_t we_own, locked;
300 /*
301 * Note: 'we_own' will become true if we set PG_BUSY on a page.
302 */
303 we_own = FALSE0;
304 pg = anon->an_page;
305
306 /*
307 * Is page resident? Make sure it is not busy/released.
308 */
309 if (pg) {
310 KASSERT(pg->pg_flags & PQ_ANON)((pg->pg_flags & 0x00100000) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 310, "pg->pg_flags & PQ_ANON"
));
311 KASSERT(pg->uanon == anon)((pg->uanon == anon) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 311, "pg->uanon == anon"));
312
313 /*
314 * if the page is busy, we drop all the locks and
315 * try again.
316 */
317 if ((pg->pg_flags & (PG_BUSY0x00000001|PG_RELEASED0x00000020)) == 0)
318 return (VM_PAGER_OK0);
319 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_WANTED0x00000002);
320 counters_inc(uvmexp_counters, flt_pgwait);
321
322 /*
323 * The last unlock must be an atomic unlock and wait
324 * on the owner of page.
325 */
326 if (pg->uobject) {
327 /* Owner of page is UVM object. */
328 uvmfault_unlockall(ufi, amap, NULL((void *)0));
329 rwsleep_nsec(pg, pg->uobject->vmobjlock,
330 PVM4 | PNORELOCK0x200, "anonget1", INFSLP0xffffffffffffffffULL);
331 } else {
332 /* Owner of page is anon. */
333 uvmfault_unlockall(ufi, NULL((void *)0), NULL((void *)0));
334 rwsleep_nsec(pg, anon->an_lock, PVM4 | PNORELOCK0x200,
335 "anonget2", INFSLP0xffffffffffffffffULL);
336 }
337 } else {
338 /*
339 * No page, therefore allocate one.
340 */
341 pg = uvm_pagealloc(NULL((void *)0), 0, anon, 0);
342 if (pg == NULL((void *)0)) {
343 /* Out of memory. Wait a little. */
344 uvmfault_unlockall(ufi, amap, NULL((void *)0));
345 counters_inc(uvmexp_counters, flt_noram);
346 uvm_wait("flt_noram1");
347 } else {
348 /* PG_BUSY bit is set. */
349 we_own = TRUE1;
350 uvmfault_unlockall(ufi, amap, NULL((void *)0));
351
352 /*
353 * Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into
354 * the uvm_swap_get() function with all data
355 * structures unlocked. Note that it is OK
356 * to read an_swslot here, because we hold
357 * PG_BUSY on the page.
358 */
359 counters_inc(uvmexp_counters, pageins);
360 error = uvm_swap_get(pg, anon->an_swslot,
361 PGO_SYNCIO0x002);
362
363 /*
364 * We clean up after the I/O below in the
365 * 'we_own' case.
366 */
367 }
368 }
369
370 /*
371 * Re-lock the map and anon.
372 */
373 locked = uvmfault_relock(ufi);
374 if (locked || we_own) {
375 rw_enter(anon->an_lock, RW_WRITE0x0001UL);
376 }
377
378 /*
379 * If we own the page (i.e. we set PG_BUSY), then we need
380 * to clean up after the I/O. There are three cases to
381 * consider:
382 *
383 * 1) Page was released during I/O: free anon and ReFault.
384 * 2) I/O not OK. Free the page and cause the fault to fail.
385 * 3) I/O OK! Activate the page and sync with the non-we_own
386 * case (i.e. drop anon lock if not locked).
387 */
388 if (we_own) {
389 if (pg->pg_flags & PG_WANTED0x00000002) {
390 wakeup(pg);
391 }
392 /* un-busy! */
393 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags,
394 PG_WANTED0x00000002|PG_BUSY0x00000001|PG_FAKE0x00000040);
395 UVM_PAGE_OWN(pg, NULL);
396
397 /*
398 * if we were RELEASED during I/O, then our anon is
399 * no longer part of an amap. we need to free the
400 * anon and try again.
401 */
402 if (pg->pg_flags & PG_RELEASED0x00000020) {
403 pmap_page_protect(pg, PROT_NONE0x00);
404 KASSERT(anon->an_ref == 0)((anon->an_ref == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 404, "anon->an_ref == 0"));
405 /*
406 * Released while we had unlocked amap.
407 */
408 if (locked)
409 uvmfault_unlockall(ufi, NULL((void *)0), NULL((void *)0));
410 uvm_anon_release(anon); /* frees page for us */
411 counters_inc(uvmexp_counters, flt_pgrele);
412 return (VM_PAGER_REFAULT7); /* refault! */
413 }
414
415 if (error != VM_PAGER_OK0) {
416 KASSERT(error != VM_PAGER_PEND)((error != 3) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 416, "error != VM_PAGER_PEND"));
417
418 /* remove page from anon */
419 anon->an_page = NULL((void *)0);
420
421 /*
422 * Remove the swap slot from the anon and
423 * mark the anon as having no real slot.
424 * Do not free the swap slot, thus preventing
425 * it from being used again.
426 */
427 uvm_swap_markbad(anon->an_swslot, 1);
428 anon->an_swslot = SWSLOT_BAD(-1);
429
430 /*
431 * Note: page was never !PG_BUSY, so it
432 * cannot be mapped and thus no need to
433 * pmap_page_protect() it.
434 */
435 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
436 uvm_pagefree(pg);
437 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
438
439 if (locked) {
440 uvmfault_unlockall(ufi, NULL((void *)0), NULL((void *)0));
441 }
442 rw_exit(anon->an_lock);
443 return (VM_PAGER_ERROR4);
444 }
445
446 /*
447 * We have successfully read the page, activate it.
448 */
449 pmap_clear_modify(pg)pmap_clear_attrs(pg, 0x0000000000000040UL);
450 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
451 uvm_pageactivate(pg);
452 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
453 }
454
455 /*
456 * We were not able to re-lock the map - restart the fault.
457 */
458 if (!locked) {
459 if (we_own) {
460 rw_exit(anon->an_lock);
461 }
462 return (VM_PAGER_REFAULT7);
463 }
464
465 /*
466 * Verify that no one has touched the amap and moved
467 * the anon on us.
468 */
469 if (ufi != NULL((void *)0) && amap_lookup(&ufi->entry->aref,
470 ufi->orig_rvaddr - ufi->entry->start) != anon) {
471
472 uvmfault_unlockall(ufi, amap, NULL((void *)0));
473 return (VM_PAGER_REFAULT7);
474 }
475
476 /*
477 * Retry..
478 */
479 counters_inc(uvmexp_counters, flt_anretry);
480 continue;
481
482 }
483 /*NOTREACHED*/
484}
485
486/*
487 * Update statistics after fault resolution.
488 * - maxrss
489 */
490void
491uvmfault_update_stats(struct uvm_faultinfo *ufi)
492{
493 struct vm_map *map;
494 struct proc *p;
495 vsize_t res;
496
497 map = ufi->orig_map;
498
499 /*
500 * If this is a nested pmap (eg, a virtual machine pmap managed
501 * by vmm(4) on amd64/i386), don't do any updating, just return.
502 *
503 * pmap_nested() on other archs is #defined to 0, so this is a
504 * no-op.
505 */
506 if (pmap_nested(map->pmap)((map->pmap)->pm_type != 1))
507 return;
508
509 /* Update the maxrss for the process. */
510 if (map->flags & VM_MAP_ISVMSPACE0x40) {
511 p = curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc;
512 KASSERT(p != NULL && &p->p_vmspace->vm_map == map)((p != ((void *)0) && &p->p_vmspace->vm_map
== map) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 512, "p != NULL && &p->p_vmspace->vm_map == map"
));
513
514 res = pmap_resident_count(map->pmap)((map->pmap)->pm_stats.resident_count);
515 /* Convert res from pages to kilobytes. */
516 res <<= (PAGE_SHIFT12 - 10);
517
518 if (p->p_ru.ru_maxrss < res)
519 p->p_ru.ru_maxrss = res;
520 }
521}
522
523/*
524 * F A U L T - m a i n e n t r y p o i n t
525 */
526
527/*
528 * uvm_fault: page fault handler
529 *
530 * => called from MD code to resolve a page fault
531 * => VM data structures usually should be unlocked. however, it is
532 * possible to call here with the main map locked if the caller
533 * gets a write lock, sets it recursive, and then calls us (c.f.
534 * uvm_map_pageable). this should be avoided because it keeps
535 * the map locked off during I/O.
536 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
537 */
538#define MASK(entry)((((entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01 | 0x02
| 0x04)) (UVM_ET_ISCOPYONWRITE(entry)(((entry)->etype & 0x0004) != 0) ? \
539 ~PROT_WRITE0x02 : PROT_MASK(0x01 | 0x02 | 0x04))
540struct uvm_faultctx {
541 /*
542 * the following members are set up by uvm_fault_check() and
543 * read-only after that.
544 */
545 vm_prot_t enter_prot;
546 vm_prot_t access_type;
547 vaddr_t startva;
548 int npages;
549 int centeridx;
550 boolean_t narrow;
551 boolean_t wired;
552 paddr_t pa_flags;
553};
554
555int uvm_fault_check(
556 struct uvm_faultinfo *, struct uvm_faultctx *,
557 struct vm_anon ***);
558
559int uvm_fault_upper(
560 struct uvm_faultinfo *, struct uvm_faultctx *,
561 struct vm_anon **, vm_fault_t);
562boolean_t uvm_fault_upper_lookup(
563 struct uvm_faultinfo *, const struct uvm_faultctx *,
564 struct vm_anon **, struct vm_page **);
565
566int uvm_fault_lower(
567 struct uvm_faultinfo *, struct uvm_faultctx *,
568 struct vm_page **, vm_fault_t);
569
570int
571uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type,
572 vm_prot_t access_type)
573{
574 struct uvm_faultinfo ufi;
575 struct uvm_faultctx flt;
576 boolean_t shadowed;
577 struct vm_anon *anons_store[UVM_MAXRANGE16], **anons;
578 struct vm_page *pages[UVM_MAXRANGE16];
579 int error;
580
581 counters_inc(uvmexp_counters, faults);
582 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL)do { extern struct dt_probe (dt_static_uvm_fault); struct dt_probe
*dtp = &(dt_static_uvm_fault); struct dt_provider *dtpv =
dtp->dtp_prov; if (__builtin_expect(((dt_tracing) != 0), 0
) && __builtin_expect(((dtp->dtp_recording) != 0),
0)) { dtpv->dtpv_enter(dtpv, dtp, vaddr, fault_type, access_type
, ((void *)0)); } } while (0);
4
←
Assuming 'dt_tracing' is equal to 0
→
5
←
Loop condition is false. Exiting loop
→
583
584 /*
585 * init the IN parameters in the ufi
586 */
587 ufi.orig_map = orig_map;
588 ufi.orig_rvaddr = trunc_page(vaddr)((vaddr) & ~((1 << 12) - 1));
589 ufi.orig_size = PAGE_SIZE(1 << 12); /* can't get any smaller than this */
590 if (fault_type
5.1
'fault_type' is equal to VM_FAULT_WIRE
 == VM_FAULT_WIRE((vm_fault_t) 0x2))
6
←
Taking true branch
→
591 flt.narrow = TRUE1; /* don't look for neighborhood
592 * pages on wire */
593 else
594 flt.narrow = FALSE0; /* normal fault */
595 flt.access_type = access_type;
596
597
598 error = ERESTART-1;
599 while (error == ERESTART-1) { /* ReFault: */
7
←
Loop condition is true. Entering loop body
→
600 anons = anons_store;
601
602 error = uvm_fault_check(&ufi, &flt, &anons);
8
←
Calling 'uvm_fault_check'
→
29
←
Returning from 'uvm_fault_check'
→
603 if (error
29.1
'error' is equal to 0
 != 0)
30
←
Taking false branch
→
604 continue;
605
606 /* True if there is an anon at the faulting address */
607 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
608 if (shadowed == TRUE1) {
31
←
Assuming 'shadowed' is equal to TRUE
→
32
←
Taking true branch
→
609 /* case 1: fault on an anon in our amap */
610 error = uvm_fault_upper(&ufi, &flt, anons, fault_type);
33
←
Passing null pointer value via 3rd parameter 'anons'
→
34
←
Calling 'uvm_fault_upper'
→
611 } else {
612 struct uvm_object *uobj = ufi.entry->object.uvm_obj;
613
614 /*
615 * if the desired page is not shadowed by the amap and
616 * we have a backing object, then we check to see if
617 * the backing object would prefer to handle the fault
618 * itself (rather than letting us do it with the usual
619 * pgo_get hook). the backing object signals this by
620 * providing a pgo_fault routine.
621 */
622 if (uobj != NULL((void *)0) && uobj->pgops->pgo_fault != NULL((void *)0)) {
623 KERNEL_LOCK()_kernel_lock();
624 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
625 error = uobj->pgops->pgo_fault(&ufi,
626 flt.startva, pages, flt.npages,
627 flt.centeridx, fault_type, flt.access_type,
628 PGO_LOCKED0x040);
629 KERNEL_UNLOCK()_kernel_unlock();
630
631 if (error == VM_PAGER_OK0)
632 error = 0;
633 else if (error == VM_PAGER_REFAULT7)
634 error = ERESTART-1;
635 else
636 error = EACCES13;
637 } else {
638 /* case 2: fault on backing obj or zero fill */
639 error = uvm_fault_lower(&ufi, &flt, pages,
640 fault_type);
641 }
642 }
643 }
644
645 return error;
646}
647
648/*
649 * uvm_fault_check: check prot, handle needs-copy, etc.
650 *
651 * 1. lookup entry.
652 * 2. check protection.
653 * 3. adjust fault condition (mainly for simulated fault).
654 * 4. handle needs-copy (lazy amap copy).
655 * 5. establish range of interest for neighbor fault (aka pre-fault).
656 * 6. look up anons (if amap exists).
657 * 7. flush pages (if MADV_SEQUENTIAL)
658 *
659 * => called with nothing locked.
660 * => if we fail (result != 0) we unlock everything.
661 * => initialize/adjust many members of flt.
662 */
663int
664uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
665 struct vm_anon ***ranons)
666{
667 struct vm_amap *amap;
668 struct uvm_object *uobj;
669 int nback, nforw;
670
671 /*
672 * lookup and lock the maps
673 */
674 if (uvmfault_lookup(ufi, FALSE0) == FALSE0) {
9
←
Taking false branch
→
675 return EFAULT14;
676 }
677 /* locked: maps(read) */
678
679#ifdef DIAGNOSTIC1
680 if ((ufi->map->flags & VM_MAP_PAGEABLE0x01) == 0)
10
←
Assuming the condition is false
→
11
←
Taking false branch
→
681 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)",
682 ufi->map, ufi->orig_rvaddr);
683#endif
684
685 /*
686 * check protection
687 */
688 if ((ufi->entry->protection & flt->access_type) != flt->access_type) {
12
←
Assuming the condition is false
→
13
←
Taking false branch
→
689 uvmfault_unlockmaps(ufi, FALSE0);
690 return EACCES13;
691 }
692
693 /*
694 * "enter_prot" is the protection we want to enter the page in at.
695 * for certain pages (e.g. copy-on-write pages) this protection can
696 * be more strict than ufi->entry->protection. "wired" means either
697 * the entry is wired or we are fault-wiring the pg.
698 */
699
700 flt->enter_prot = ufi->entry->protection;
701 flt->pa_flags = UVM_ET_ISWC(ufi->entry)(((ufi->entry)->etype & 0x0080) != 0) ? PMAP_WC0x2 : 0;
14
←
Assuming the condition is false
→
15
←
'?' condition is false
→
702 flt->wired = VM_MAPENT_ISWIRED(ufi->entry)((ufi->entry)->wired_count != 0) || (flt->narrow == TRUE1);
16
←
Assuming field 'wired_count' is equal to 0
→
703 if (flt->wired
16.1
Field 'wired' is 1
)
17
←
Taking true branch
→
704 flt->access_type = flt->enter_prot; /* full access for wired */
705
706 /* handle "needs_copy" case. */
707 if (UVM_ET_ISNEEDSCOPY(ufi->entry)(((ufi->entry)->etype & 0x0008) != 0)) {
18
←
Assuming the condition is true
→
19
←
Taking true branch
→
708 if ((flt->access_type & PROT_WRITE0x02) ||
20
←
Assuming the condition is false
→
22
←
Taking false branch
→
709 (ufi->entry->object.uvm_obj == NULL((void *)0))) {
21
←
Assuming field 'uvm_obj' is not equal to NULL
→
710 /* need to clear */
711 uvmfault_unlockmaps(ufi, FALSE0);
712 uvmfault_amapcopy(ufi);
713 counters_inc(uvmexp_counters, flt_amcopy);
714 return ERESTART-1;
715 } else {
716 /*
717 * ensure that we pmap_enter page R/O since
718 * needs_copy is still true
719 */
720 flt->enter_prot &= ~PROT_WRITE0x02;
721 }
722 }
723
724 /*
725 * identify the players
726 */
727 amap = ufi->entry->aref.ar_amap; /* upper layer */
728 uobj = ufi->entry->object.uvm_obj; /* lower layer */
729
730 /*
731 * check for a case 0 fault. if nothing backing the entry then
732 * error now.
733 */
734 if (amap == NULL((void *)0) && uobj
23.1
'uobj' is not equal to NULL
 == NULL((void *)0)) {
23
←
Assuming 'amap' is equal to NULL
→
24
←
Taking false branch
→
735 uvmfault_unlockmaps(ufi, FALSE0);
736 return EFAULT14;
737 }
738
739 /*
740 * establish range of interest based on advice from mapper
741 * and then clip to fit map entry. note that we only want
742 * to do this the first time through the fault. if we
743 * ReFault we will disable this by setting "narrow" to true.
744 */
745 if (flt->narrow
24.1
Field 'narrow' is not equal to FALSE
 == FALSE0) {
25
←
Taking false branch
→
746
747 /* wide fault (!narrow) */
748 nback = min(uvmadvice[ufi->entry->advice].nback,
749 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT12);
750 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT12);
751 nforw = min(uvmadvice[ufi->entry->advice].nforw,
752 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT12) - 1);
753 /*
754 * note: "-1" because we don't want to count the
755 * faulting page as forw
756 */
757 flt->npages = nback + nforw + 1;
758 flt->centeridx = nback;
759
760 flt->narrow = TRUE1; /* ensure only once per-fault */
761 } else {
762 /* narrow fault! */
763 nback = nforw = 0;
764 flt->startva = ufi->orig_rvaddr;
765 flt->npages = 1;
766 flt->centeridx = 0;
767 }
768
769 /*
770 * if we've got an amap then lock it and extract current anons.
771 */
772 if (amap
25.1
'amap' is null
) {
26
←
Taking false branch
→
773 amap_lock(amap)rw_enter_write((amap)->am_lock);
774 amap_lookups(&ufi->entry->aref,
775 flt->startva - ufi->entry->start, *ranons, flt->npages);
776 } else {
777 *ranons = NULL((void *)0); /* to be safe */
27
←
Null pointer value stored to 'anons'
→
778 }
779
780 /*
781 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
782 * now and then forget about them (for the rest of the fault).
783 */
784 if (ufi->entry->advice == MADV_SEQUENTIAL2 && nback != 0) {
28
←
Assuming field 'advice' is not equal to MADV_SEQUENTIAL
→
785 /* flush back-page anons? */
786 if (amap)
787 uvmfault_anonflush(*ranons, nback);
788
789 /*
790 * flush object?
791 */
792 if (uobj) {
793 voff_t uoff;
794
795 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset;
796 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
797 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff +
798 ((vsize_t)nback << PAGE_SHIFT12), PGO_DEACTIVATE0x004);
799 rw_exit(uobj->vmobjlock);
800 }
801
802 /* now forget about the backpages */
803 if (amap)
804 *ranons += nback;
805 flt->startva += ((vsize_t)nback << PAGE_SHIFT12);
806 flt->npages -= nback;
807 flt->centeridx = 0;
808 }
809
810 return 0;
811}
812
813/*
814 * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
815 *
816 * iterate range of interest:
817 * 1. check if h/w mapping exists. if yes, we don't care
818 * 2. check if anon exists. if not, page is lower.
819 * 3. if anon exists, enter h/w mapping for neighbors.
820 *
821 * => called with amap locked (if exists).
822 */
823boolean_t
824uvm_fault_upper_lookup(struct uvm_faultinfo *ufi,
825 const struct uvm_faultctx *flt, struct vm_anon **anons,
826 struct vm_page **pages)
827{
828 struct vm_amap *amap = ufi->entry->aref.ar_amap;
829 struct vm_anon *anon;
830 boolean_t shadowed;
831 vaddr_t currva;
832 paddr_t pa;
833 int lcv;
834
835 /* locked: maps(read), amap(if there) */
836 KASSERT(amap == NULL ||((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 837, "amap == NULL || rw_write_held(amap->am_lock)"))
837 rw_write_held(amap->am_lock))((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 837, "amap == NULL || rw_write_held(amap->am_lock)"));
838
839 /*
840 * map in the backpages and frontpages we found in the amap in hopes
841 * of preventing future faults. we also init the pages[] array as
842 * we go.
843 */
844 currva = flt->startva;
845 shadowed = FALSE0;
846 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE(1 << 12)) {
847 /*
848 * dont play with VAs that are already mapped
849 * except for center)
850 */
851 if (lcv != flt->centeridx &&
852 pmap_extract(ufi->orig_map->pmap, currva, &pa)) {
853 pages[lcv] = PGO_DONTCARE((struct vm_page *) -1L);
854 continue;
855 }
856
857 /*
858 * unmapped or center page. check if any anon at this level.
859 */
860 if (amap == NULL((void *)0) || anons[lcv] == NULL((void *)0)) {
861 pages[lcv] = NULL((void *)0);
862 continue;
863 }
864
865 /*
866 * check for present page and map if possible.
867 */
868 pages[lcv] = PGO_DONTCARE((struct vm_page *) -1L);
869 if (lcv == flt->centeridx) { /* save center for later! */
870 shadowed = TRUE1;
871 continue;
872 }
873 anon = anons[lcv];
874 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 874, "anon->an_lock == amap->am_lock"
));
875 if (anon->an_page &&
876 (anon->an_page->pg_flags & (PG_RELEASED0x00000020|PG_BUSY0x00000001)) == 0) {
877 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
878 uvm_pageactivate(anon->an_page); /* reactivate */
879 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
880 counters_inc(uvmexp_counters, flt_namap);
881
882 /*
883 * Since this isn't the page that's actually faulting,
884 * ignore pmap_enter() failures; it's not critical
885 * that we enter these right now.
886 */
887 (void) pmap_enter(ufi->orig_map->pmap, currva,
888 VM_PAGE_TO_PHYS(anon->an_page)((anon->an_page)->phys_addr) | flt->pa_flags,
889 (anon->an_ref > 1) ?
890 (flt->enter_prot & ~PROT_WRITE0x02) : flt->enter_prot,
891 PMAP_CANFAIL0x00000020 |
892 (VM_MAPENT_ISWIRED(ufi->entry)((ufi->entry)->wired_count != 0) ? PMAP_WIRED0x00000010 : 0));
893 }
894 }
895 if (flt->npages > 1)
896 pmap_update(ufi->orig_map->pmap);
897
898 return shadowed;
899}
900
901/*
902 * uvm_fault_upper: handle upper fault.
903 *
904 * 1. acquire anon lock.
905 * 2. get anon. let uvmfault_anonget do the dirty work.
906 * 3. if COW, promote data to new anon
907 * 4. enter h/w mapping
908 */
909int
910uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
911 struct vm_anon **anons, vm_fault_t fault_type)
912{
913 struct vm_amap *amap = ufi->entry->aref.ar_amap;
914 struct vm_anon *oanon, *anon = anons[flt->centeridx];
35
←
Array access (from variable 'anons') results in a null pointer dereference
915 struct vm_page *pg = NULL((void *)0);
916 int error, ret;
917
918 /* locked: maps(read), amap, anon */
919 KASSERT(rw_write_held(amap->am_lock))((rw_write_held(amap->am_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 919, "rw_write_held(amap->am_lock)"
));
920 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 920, "anon->an_lock == amap->am_lock"
));
921
922 /*
923 * no matter if we have case 1A or case 1B we are going to need to
924 * have the anon's memory resident. ensure that now.
925 */
926 /*
927 * let uvmfault_anonget do the dirty work.
928 * if it fails (!OK) it will unlock everything for us.
929 * if it succeeds, locks are still valid and locked.
930 * also, if it is OK, then the anon's page is on the queues.
931 * if the page is on loan from a uvm_object, then anonget will
932 * lock that object for us if it does not fail.
933 */
934 error = uvmfault_anonget(ufi, amap, anon);
935 switch (error) {
936 case VM_PAGER_OK0:
937 break;
938
939 case VM_PAGER_REFAULT7:
940 return ERESTART-1;
941
942 case VM_PAGER_ERROR4:
943 /*
944 * An error occurred while trying to bring in the
945 * page -- this is the only error we return right
946 * now.
947 */
948 return EACCES13; /* XXX */
949 default:
950#ifdef DIAGNOSTIC1
951 panic("uvm_fault: uvmfault_anonget -> %d", error);
952#else
953 return EACCES13;
954#endif
955 }
956
957 KASSERT(rw_write_held(amap->am_lock))((rw_write_held(amap->am_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 957, "rw_write_held(amap->am_lock)"
));
958 KASSERT(anon->an_lock == amap->am_lock)((anon->an_lock == amap->am_lock) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 958, "anon->an_lock == amap->am_lock"
));
959
960 /*
961 * if we are case 1B then we will need to allocate a new blank
962 * anon to transfer the data into. note that we have a lock
963 * on anon, so no one can busy or release the page until we are done.
964 * also note that the ref count can't drop to zero here because
965 * it is > 1 and we are only dropping one ref.
966 *
967 * in the (hopefully very rare) case that we are out of RAM we
968 * will unlock, wait for more RAM, and refault.
969 *
970 * if we are out of anon VM we wait for RAM to become available.
971 */
972
973 if ((flt->access_type & PROT_WRITE0x02) != 0 && anon->an_ref > 1) {
974 counters_inc(uvmexp_counters, flt_acow);
975 oanon = anon; /* oanon = old */
976 anon = uvm_analloc();
977 if (anon) {
978 anon->an_lock = amap->am_lock;
979 pg = uvm_pagealloc(NULL((void *)0), 0, anon, 0);
980 }
981
982 /* check for out of RAM */
983 if (anon == NULL((void *)0) || pg == NULL((void *)0)) {
984 uvmfault_unlockall(ufi, amap, NULL((void *)0));
985 if (anon == NULL((void *)0))
986 counters_inc(uvmexp_counters, flt_noanon);
987 else {
988 anon->an_lock = NULL((void *)0);
989 anon->an_ref--;
990 uvm_anfree(anon)uvm_anfree_list((anon), ((void *)0));
991 counters_inc(uvmexp_counters, flt_noram);
992 }
993
994 if (uvm_swapisfull())
995 return ENOMEM12;
996
997 /* out of RAM, wait for more */
998 if (anon == NULL((void *)0))
999 uvm_anwait();
1000 else
1001 uvm_wait("flt_noram3");
1002 return ERESTART-1;
1003 }
1004
1005 /* got all resources, replace anon with nanon */
1006 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */
1007 /* un-busy! new page */
1008 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001|PG_FAKE0x00000040);
1009 UVM_PAGE_OWN(pg, NULL);
1010 ret = amap_add(&ufi->entry->aref,
1011 ufi->orig_rvaddr - ufi->entry->start, anon, 1);
1012 KASSERT(ret == 0)((ret == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1012, "ret == 0"));
1013
1014 /* deref: can not drop to zero here by defn! */
1015 oanon->an_ref--;
1016
1017 /*
1018 * note: anon is _not_ locked, but we have the sole references
1019 * to in from amap.
1020 * thus, no one can get at it until we are done with it.
1021 */
1022 } else {
1023 counters_inc(uvmexp_counters, flt_anon);
1024 oanon = anon;
1025 pg = anon->an_page;
1026 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1027 flt->enter_prot = flt->enter_prot & ~PROT_WRITE0x02;
1028 }
1029
1030 /*
1031 * now map the page in .
1032 */
1033 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1034 VM_PAGE_TO_PHYS(pg)((pg)->phys_addr) | flt->pa_flags, flt->enter_prot,
1035 flt->access_type | PMAP_CANFAIL0x00000020 | (flt->wired ? PMAP_WIRED0x00000010 : 0)) != 0) {
1036 /*
1037 * No need to undo what we did; we can simply think of
1038 * this as the pmap throwing away the mapping information.
1039 *
1040 * We do, however, have to go through the ReFault path,
1041 * as the map may change while we're asleep.
1042 */
1043 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1044 if (uvm_swapisfull()) {
1045 /* XXX instrumentation */
1046 return ENOMEM12;
1047 }
1048 /* XXX instrumentation */
1049 uvm_wait("flt_pmfail1");
1050 return ERESTART-1;
1051 }
1052
1053 /*
1054 * ... update the page queues.
1055 */
1056 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1057
1058 if (fault_type == VM_FAULT_WIRE((vm_fault_t) 0x2)) {
1059 uvm_pagewire(pg);
1060 /*
1061 * since the now-wired page cannot be paged out,
1062 * release its swap resources for others to use.
1063 * since an anon with no swap cannot be PG_CLEAN,
1064 * clear its clean flag now.
1065 */
1066 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1067 uvm_anon_dropswap(anon);
1068 } else {
1069 /* activate it */
1070 uvm_pageactivate(pg);
1071 }
1072
1073 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1074
1075 /*
1076 * done case 1! finish up by unlocking everything and returning success
1077 */
1078 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1079 pmap_update(ufi->orig_map->pmap);
1080 return 0;
1081}
1082
1083/*
1084 * uvm_fault_lower_lookup: look up on-memory uobj pages.
1085 *
1086 * 1. get on-memory pages.
1087 * 2. if failed, give up (get only center page later).
1088 * 3. if succeeded, enter h/w mapping of neighbor pages.
1089 */
1090
1091struct vm_page *
1092uvm_fault_lower_lookup(
1093 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1094 struct vm_page **pages)
1095{
1096 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1097 struct vm_page *uobjpage = NULL((void *)0);
1098 int lcv, gotpages;
1099 vaddr_t currva;
1100
1101 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
1102
1103 counters_inc(uvmexp_counters, flt_lget);
1104 gotpages = flt->npages;
1105 (void) uobj->pgops->pgo_get(uobj,
1106 ufi->entry->offset + (flt->startva - ufi->entry->start),
1107 pages, &gotpages, flt->centeridx,
1108 flt->access_type & MASK(ufi->entry)((((ufi->entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01
| 0x02 | 0x04)), ufi->entry->advice,
1109 PGO_LOCKED0x040);
1110
1111 /*
1112 * check for pages to map, if we got any
1113 */
1114 if (gotpages == 0) {
1115 return NULL((void *)0);
1116 }
1117
1118 currva = flt->startva;
1119 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE(1 << 12)) {
1120 if (pages[lcv] == NULL((void *)0) ||
1121 pages[lcv] == PGO_DONTCARE((struct vm_page *) -1L))
1122 continue;
1123
1124 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0)(((pages[lcv]->pg_flags & 0x00000020) == 0) ? (void)0 :
__assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 1124
, "(pages[lcv]->pg_flags & PG_RELEASED) == 0"));
1125
1126 /*
1127 * if center page is resident and not
1128 * PG_BUSY, then pgo_get made it PG_BUSY
1129 * for us and gave us a handle to it.
1130 * remember this page as "uobjpage."
1131 * (for later use).
1132 */
1133 if (lcv == flt->centeridx) {
1134 uobjpage = pages[lcv];
1135 continue;
1136 }
1137
1138 /*
1139 * note: calling pgo_get with locked data
1140 * structures returns us pages which are
1141 * neither busy nor released, so we don't
1142 * need to check for this. we can just
1143 * directly enter the page (after moving it
1144 * to the head of the active queue [useful?]).
1145 */
1146
1147 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1148 uvm_pageactivate(pages[lcv]); /* reactivate */
1149 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1150 counters_inc(uvmexp_counters, flt_nomap);
1151
1152 /*
1153 * Since this page isn't the page that's
1154 * actually faulting, ignore pmap_enter()
1155 * failures; it's not critical that we
1156 * enter these right now.
1157 */
1158 (void) pmap_enter(ufi->orig_map->pmap, currva,
1159 VM_PAGE_TO_PHYS(pages[lcv])((pages[lcv])->phys_addr) | flt->pa_flags,
1160 flt->enter_prot & MASK(ufi->entry)((((ufi->entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01
| 0x02 | 0x04)),
1161 PMAP_CANFAIL0x00000020 |
1162 (flt->wired ? PMAP_WIRED0x00000010 : 0));
1163
1164 /*
1165 * NOTE: page can't be PG_WANTED because
1166 * we've held the lock the whole time
1167 * we've had the handle.
1168 */
1169 atomic_clearbits_intx86_atomic_clearbits_u32(&pages[lcv]->pg_flags, PG_BUSY0x00000001);
1170 UVM_PAGE_OWN(pages[lcv], NULL);
1171 }
1172 pmap_update(ufi->orig_map->pmap);
1173
1174 return uobjpage;
1175}
1176
1177/*
1178 * uvm_fault_lower: handle lower fault.
1179 *
1180 */
1181int
1182uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1183 struct vm_page **pages, vm_fault_t fault_type)
1184{
1185 struct vm_amap *amap = ufi->entry->aref.ar_amap;
1186 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1187 boolean_t promote, locked;
1188 int result;
1189 struct vm_page *uobjpage, *pg = NULL((void *)0);
1190 struct vm_anon *anon = NULL((void *)0);
1191 voff_t uoff;
1192
1193 /*
1194 * now, if the desired page is not shadowed by the amap and we have
1195 * a backing object that does not have a special fault routine, then
1196 * we ask (with pgo_get) the object for resident pages that we care
1197 * about and attempt to map them in. we do not let pgo_get block
1198 * (PGO_LOCKED).
1199 */
1200 if (uobj == NULL((void *)0)) {
1201 /* zero fill; don't care neighbor pages */
1202 uobjpage = NULL((void *)0);
1203 } else {
1204 uobjpage = uvm_fault_lower_lookup(ufi, flt, pages);
1205 }
1206
1207 /*
1208 * note that at this point we are done with any front or back pages.
1209 * we are now going to focus on the center page (i.e. the one we've
1210 * faulted on). if we have faulted on the bottom (uobj)
1211 * layer [i.e. case 2] and the page was both present and available,
1212 * then we've got a pointer to it as "uobjpage" and we've already
1213 * made it BUSY.
1214 */
1215
1216 /*
1217 * locked:
1218 */
1219 KASSERT(amap == NULL ||((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1220, "amap == NULL || rw_write_held(amap->am_lock)"))
1220 rw_write_held(amap->am_lock))((amap == ((void *)0) || rw_write_held(amap->am_lock)) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1220, "amap == NULL || rw_write_held(amap->am_lock)"));
1221 KASSERT(uobj == NULL ||((uobj == ((void *)0) || rw_write_held(uobj->vmobjlock)) ?
(void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1222, "uobj == NULL || rw_write_held(uobj->vmobjlock)"))
1222 rw_write_held(uobj->vmobjlock))((uobj == ((void *)0) || rw_write_held(uobj->vmobjlock)) ?
(void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1222, "uobj == NULL || rw_write_held(uobj->vmobjlock)"));
1223
1224 /*
1225 * note that uobjpage can not be PGO_DONTCARE at this point. we now
1226 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1227 * have a backing object, check and see if we are going to promote
1228 * the data up to an anon during the fault.
1229 */
1230 if (uobj == NULL((void *)0)) {
1231 uobjpage = PGO_DONTCARE((struct vm_page *) -1L);
1232 promote = TRUE1; /* always need anon here */
1233 } else {
1234 KASSERT(uobjpage != PGO_DONTCARE)((uobjpage != ((struct vm_page *) -1L)) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 1234, "uobjpage != PGO_DONTCARE"
));
1235 promote = (flt->access_type & PROT_WRITE0x02) &&
1236 UVM_ET_ISCOPYONWRITE(ufi->entry)(((ufi->entry)->etype & 0x0004) != 0);
1237 }
1238
1239 /*
1240 * if uobjpage is not null then we do not need to do I/O to get the
1241 * uobjpage.
1242 *
1243 * if uobjpage is null, then we need to ask the pager to
1244 * get the data for us. once we have the data, we need to reverify
1245 * the state the world. we are currently not holding any resources.
1246 */
1247 if (uobjpage) {
1248 /* update rusage counters */
1249 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_minflt++;
1250 } else {
1251 int gotpages;
1252
1253 /* update rusage counters */
1254 curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_ru.ru_majflt++;
1255
1256 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1257
1258 counters_inc(uvmexp_counters, flt_get);
1259 gotpages = 1;
1260 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1261 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1262 0, flt->access_type & MASK(ufi->entry)((((ufi->entry)->etype & 0x0004) != 0) ? ~0x02 : (0x01
| 0x02 | 0x04)), ufi->entry->advice,
1263 PGO_SYNCIO0x002);
1264
1265 /*
1266 * recover from I/O
1267 */
1268 if (result != VM_PAGER_OK0) {
1269 KASSERT(result != VM_PAGER_PEND)((result != 3) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1269, "result != VM_PAGER_PEND"));
1270
1271 if (result == VM_PAGER_AGAIN5) {
1272 tsleep_nsec(&nowake, PVM4, "fltagain2",
1273 MSEC_TO_NSEC(5));
1274 return ERESTART-1;
1275 }
1276
1277 if (!UVM_ET_ISNOFAULT(ufi->entry)(((ufi->entry)->etype & 0x0020) != 0))
1278 return (EIO5);
1279
1280 uobjpage = PGO_DONTCARE((struct vm_page *) -1L);
1281 uobj = NULL((void *)0);
1282 promote = TRUE1;
1283 }
1284
1285 /* re-verify the state of the world. */
1286 locked = uvmfault_relock(ufi);
1287 if (locked && amap != NULL((void *)0))
1288 amap_lock(amap)rw_enter_write((amap)->am_lock);
1289
1290 /* might be changed */
1291 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1292 uobj = uobjpage->uobject;
1293 rw_enter(uobj->vmobjlock, RW_WRITE0x0001UL);
1294 }
1295
1296 /*
1297 * Re-verify that amap slot is still free. if there is
1298 * a problem, we clean up.
1299 */
1300 if (locked && amap && amap_lookup(&ufi->entry->aref,
1301 ufi->orig_rvaddr - ufi->entry->start)) {
1302 if (locked)
1303 uvmfault_unlockall(ufi, amap, NULL((void *)0));
1304 locked = FALSE0;
1305 }
1306
1307 /* didn't get the lock? release the page and retry. */
1308 if (locked == FALSE0 && uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1309 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1310 /* make sure it is in queues */
1311 uvm_pageactivate(uobjpage);
1312 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1313
1314 if (uobjpage->pg_flags & PG_WANTED0x00000002)
1315 /* still holding object lock */
1316 wakeup(uobjpage);
1317 atomic_clearbits_intx86_atomic_clearbits_u32(&uobjpage->pg_flags,
1318 PG_BUSY0x00000001|PG_WANTED0x00000002);
1319 UVM_PAGE_OWN(uobjpage, NULL);
1320 }
1321
1322 if (locked == FALSE0) {
1323 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L))
1324 rw_exit(uobj->vmobjlock);
1325 return ERESTART-1;
1326 }
1327
1328 /*
1329 * we have the data in uobjpage which is PG_BUSY
1330 */
1331 }
1332
1333 /*
1334 * notes:
1335 * - at this point uobjpage can not be NULL
1336 * - at this point uobjpage could be PG_WANTED (handle later)
1337 */
1338 if (promote == FALSE0) {
1339 /*
1340 * we are not promoting. if the mapping is COW ensure that we
1341 * don't give more access than we should (e.g. when doing a read
1342 * fault on a COPYONWRITE mapping we want to map the COW page in
1343 * R/O even though the entry protection could be R/W).
1344 *
1345 * set "pg" to the page we want to map in (uobjpage, usually)
1346 */
1347 counters_inc(uvmexp_counters, flt_obj);
1348 if (UVM_ET_ISCOPYONWRITE(ufi->entry)(((ufi->entry)->etype & 0x0004) != 0))
1349 flt->enter_prot &= ~PROT_WRITE0x02;
1350 pg = uobjpage; /* map in the actual object */
1351
1352 /* assert(uobjpage != PGO_DONTCARE) */
1353
1354 /*
1355 * we are faulting directly on the page.
1356 */
1357 } else {
1358 /*
1359 * if we are going to promote the data to an anon we
1360 * allocate a blank anon here and plug it into our amap.
1361 */
1362#ifdef DIAGNOSTIC1
1363 if (amap == NULL((void *)0))
1364 panic("uvm_fault: want to promote data, but no anon");
1365#endif
1366
1367 anon = uvm_analloc();
1368 if (anon) {
1369 /*
1370 * In `Fill in data...' below, if
1371 * uobjpage == PGO_DONTCARE, we want
1372 * a zero'd, dirty page, so have
1373 * uvm_pagealloc() do that for us.
1374 */
1375 anon->an_lock = amap->am_lock;
1376 pg = uvm_pagealloc(NULL((void *)0), 0, anon,
1377 (uobjpage == PGO_DONTCARE((struct vm_page *) -1L)) ? UVM_PGA_ZERO0x0002 : 0);
1378 }
1379
1380 /*
1381 * out of memory resources?
1382 */
1383 if (anon == NULL((void *)0) || pg == NULL((void *)0)) {
1384 /*
1385 * arg! must unbusy our page and fail or sleep.
1386 */
1387 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1388 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1389 uvm_pageactivate(uobjpage);
1390 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1391
1392 if (uobjpage->pg_flags & PG_WANTED0x00000002)
1393 wakeup(uobjpage);
1394 atomic_clearbits_intx86_atomic_clearbits_u32(&uobjpage->pg_flags,
1395 PG_BUSY0x00000001|PG_WANTED0x00000002);
1396 UVM_PAGE_OWN(uobjpage, NULL);
1397 }
1398
1399 /* unlock and fail ... */
1400 uvmfault_unlockall(ufi, amap, uobj);
1401 if (anon == NULL((void *)0))
1402 counters_inc(uvmexp_counters, flt_noanon);
1403 else {
1404 anon->an_lock = NULL((void *)0);
1405 anon->an_ref--;
1406 uvm_anfree(anon)uvm_anfree_list((anon), ((void *)0));
1407 counters_inc(uvmexp_counters, flt_noram);
1408 }
1409
1410 if (uvm_swapisfull())
1411 return (ENOMEM12);
1412
1413 /* out of RAM, wait for more */
1414 if (anon == NULL((void *)0))
1415 uvm_anwait();
1416 else
1417 uvm_wait("flt_noram5");
1418 return ERESTART-1;
1419 }
1420
1421 /*
1422 * fill in the data
1423 */
1424 if (uobjpage != PGO_DONTCARE((struct vm_page *) -1L)) {
1425 counters_inc(uvmexp_counters, flt_prcopy);
1426 /* copy page [pg now dirty] */
1427 uvm_pagecopy(uobjpage, pg);
1428
1429 /*
1430 * promote to shared amap? make sure all sharing
1431 * procs see it
1432 */
1433 if ((amap_flags(amap)((amap)->am_flags) & AMAP_SHARED0x1) != 0) {
1434 pmap_page_protect(uobjpage, PROT_NONE0x00);
1435 }
1436
1437 /* dispose of uobjpage. drop handle to uobj as well. */
1438 if (uobjpage->pg_flags & PG_WANTED0x00000002)
1439 wakeup(uobjpage);
1440 atomic_clearbits_intx86_atomic_clearbits_u32(&uobjpage->pg_flags,
1441 PG_BUSY0x00000001|PG_WANTED0x00000002);
1442 UVM_PAGE_OWN(uobjpage, NULL);
1443 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1444 uvm_pageactivate(uobjpage);
1445 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1446 rw_exit(uobj->vmobjlock);
1447 uobj = NULL((void *)0);
1448 } else {
1449 counters_inc(uvmexp_counters, flt_przero);
1450 /*
1451 * Page is zero'd and marked dirty by uvm_pagealloc()
1452 * above.
1453 */
1454 }
1455
1456 if (amap_add(&ufi->entry->aref,
1457 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) {
1458 uvmfault_unlockall(ufi, amap, uobj);
1459 uvm_anfree(anon)uvm_anfree_list((anon), ((void *)0));
1460 counters_inc(uvmexp_counters, flt_noamap);
1461
1462 if (uvm_swapisfull())
1463 return (ENOMEM12);
1464
1465 amap_populate(&ufi->entry->aref,
1466 ufi->orig_rvaddr - ufi->entry->start);
1467 return ERESTART-1;
1468 }
1469 }
1470
1471 /* note: pg is either the uobjpage or the new page in the new anon */
1472 /*
1473 * all resources are present. we can now map it in and free our
1474 * resources.
1475 */
1476 if (amap == NULL((void *)0))
1477 KASSERT(anon == NULL)((anon == ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1477, "anon == NULL"));
1478 else {
1479 KASSERT(rw_write_held(amap->am_lock))((rw_write_held(amap->am_lock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 1479, "rw_write_held(amap->am_lock)"
));
1480 KASSERT(anon == NULL || anon->an_lock == amap->am_lock)((anon == ((void *)0) || anon->an_lock == amap->am_lock
) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1480, "anon == NULL || anon->an_lock == amap->am_lock"
));
1481 }
1482 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1483 VM_PAGE_TO_PHYS(pg)((pg)->phys_addr) | flt->pa_flags, flt->enter_prot,
1484 flt->access_type | PMAP_CANFAIL0x00000020 | (flt->wired ? PMAP_WIRED0x00000010 : 0)) != 0) {
1485 /*
1486 * No need to undo what we did; we can simply think of
1487 * this as the pmap throwing away the mapping information.
1488 *
1489 * We do, however, have to go through the ReFault path,
1490 * as the map may change while we're asleep.
1491 */
1492 if (pg->pg_flags & PG_WANTED0x00000002)
1493 wakeup(pg);
1494
1495 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001|PG_FAKE0x00000040|PG_WANTED0x00000002);
1496 UVM_PAGE_OWN(pg, NULL);
1497 uvmfault_unlockall(ufi, amap, uobj);
1498 if (uvm_swapisfull()) {
1499 /* XXX instrumentation */
1500 return (ENOMEM12);
1501 }
1502 /* XXX instrumentation */
1503 uvm_wait("flt_pmfail2");
1504 return ERESTART-1;
1505 }
1506
1507 if (fault_type == VM_FAULT_WIRE((vm_fault_t) 0x2)) {
1508 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1509 uvm_pagewire(pg);
1510 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1511 if (pg->pg_flags & PQ_AOBJ0x00200000) {
1512 /*
1513 * since the now-wired page cannot be paged out,
1514 * release its swap resources for others to use.
1515 * since an aobj page with no swap cannot be clean,
1516 * mark it dirty now.
1517 *
1518 * use pg->uobject here. if the page is from a
1519 * tmpfs vnode, the pages are backed by its UAO and
1520 * not the vnode.
1521 */
1522 KASSERT(uobj != NULL)((uobj != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1522, "uobj != NULL"));
1523 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock)((uobj->vmobjlock == pg->uobject->vmobjlock) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c", 1523
, "uobj->vmobjlock == pg->uobject->vmobjlock"));
1524 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1525 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT12);
1526 }
1527 } else {
1528 /* activate it */
1529 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1530 uvm_pageactivate(pg);
1531 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1532 }
1533
1534 if (pg->pg_flags & PG_WANTED0x00000002)
1535 wakeup(pg);
1536
1537 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001|PG_FAKE0x00000040|PG_WANTED0x00000002);
1538 UVM_PAGE_OWN(pg, NULL);
1539 uvmfault_unlockall(ufi, amap, uobj);
1540 pmap_update(ufi->orig_map->pmap);
1541
1542 return (0);
1543}
1544
1545
1546/*
1547 * uvm_fault_wire: wire down a range of virtual addresses in a map.
1548 *
1549 * => map may be read-locked by caller, but MUST NOT be write-locked.
1550 * => if map is read-locked, any operations which may cause map to
1551 * be write-locked in uvm_fault() must be taken care of by
1552 * the caller. See uvm_map_pageable().
1553 */
1554int
1555uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type)
1556{
1557 vaddr_t va;
1558 int rv;
1559
1560 /*
1561 * now fault it in a page at a time. if the fault fails then we have
1562 * to undo what we have done. note that in uvm_fault PROT_NONE
1563 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1564 */
1565 for (va = start ; va < end ; va += PAGE_SIZE(1 << 12)) {
1
Assuming 'va' is < 'end'
→
2
←
Loop condition is true. Entering loop body
→
1566 rv = uvm_fault(map, va, VM_FAULT_WIRE((vm_fault_t) 0x2), access_type);
3
←
Calling 'uvm_fault'
→
1567 if (rv) {
1568 if (va != start) {
1569 uvm_fault_unwire(map, start, va);
1570 }
1571 return (rv);
1572 }
1573 }
1574
1575 return (0);
1576}
1577
1578/*
1579 * uvm_fault_unwire(): unwire range of virtual space.
1580 */
1581void
1582uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end)
1583{
1584
1585 vm_map_lock_read(map)vm_map_lock_read_ln(map, "/usr/src/sys/uvm/uvm_fault.c", 1585
);
1586 uvm_fault_unwire_locked(map, start, end);
1587 vm_map_unlock_read(map)vm_map_unlock_read_ln(map, "/usr/src/sys/uvm/uvm_fault.c", 1587
);
1588}
1589
1590/*
1591 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
1592 *
1593 * => map must be at least read-locked.
1594 */
1595void
1596uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end)
1597{
1598 vm_map_entry_t entry, oentry = NULL((void *)0), next;
1599 pmap_t pmap = vm_map_pmap(map)((map)->pmap);
1600 vaddr_t va;
1601 paddr_t pa;
1602 struct vm_page *pg;
1603
1604 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0)(((map->flags & 0x02) == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 1604, "(map->flags & VM_MAP_INTRSAFE) == 0"
));
1605
1606 /*
1607 * we assume that the area we are unwiring has actually been wired
1608 * in the first place. this means that we should be able to extract
1609 * the PAs from the pmap.
1610 */
1611
1612 /*
1613 * find the beginning map entry for the region.
1614 */
1615 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map))((start >= ((map)->min_offset) && end <= ((map
)->max_offset)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1615, "start >= vm_map_min(map) && end <= vm_map_max(map)"
));
1616 if (uvm_map_lookup_entry(map, start, &entry) == FALSE0)
1617 panic("uvm_fault_unwire_locked: address not in map");
1618
1619 for (va = start; va < end ; va += PAGE_SIZE(1 << 12)) {
1620 if (pmap_extract(pmap, va, &pa) == FALSE0)
1621 continue;
1622
1623 /*
1624 * find the map entry for the current address.
1625 */
1626 KASSERT(va >= entry->start)((va >= entry->start) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_fault.c", 1626, "va >= entry->start"
));
1627 while (va >= entry->end) {
1628 next = RBT_NEXT(uvm_map_addr, entry)uvm_map_addr_RBT_NEXT(entry);
1629 KASSERT(next != NULL && next->start <= entry->end)((next != ((void *)0) && next->start <= entry->
end) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_fault.c"
, 1629, "next != NULL && next->start <= entry->end"
));
1630 entry = next;
1631 }
1632
1633 /*
1634 * lock it.
1635 */
1636 if (entry != oentry) {
1637 if (oentry != NULL((void *)0)) {
1638 uvm_map_unlock_entry(oentry);
1639 }
1640 uvm_map_lock_entry(entry);
1641 oentry = entry;
1642 }
1643
1644 /*
1645 * if the entry is no longer wired, tell the pmap.
1646 */
1647 if (VM_MAPENT_ISWIRED(entry)((entry)->wired_count != 0) == 0)
1648 pmap_unwire(pmap, va);
1649
1650 pg = PHYS_TO_VM_PAGE(pa);
1651 if (pg) {
1652 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1653 uvm_pageunwire(pg);
1654 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1655 }
1656 }
1657
1658 if (oentry != NULL((void *)0)) {
1659 uvm_map_unlock_entry(entry);
1660 }
1661}
1662
1663/*
1664 * uvmfault_unlockmaps: unlock the maps
1665 */
1666void
1667uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked)
1668{
1669 /*
1670 * ufi can be NULL when this isn't really a fault,
1671 * but merely paging in anon data.
1672 */
1673 if (ufi == NULL((void *)0)) {
1674 return;
1675 }
1676
1677 uvmfault_update_stats(ufi);
1678 if (write_locked) {
1679 vm_map_unlock(ufi->map)vm_map_unlock_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c",
1679);
1680 } else {
1681 vm_map_unlock_read(ufi->map)vm_map_unlock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1681);
1682 }
1683}
1684
1685/*
1686 * uvmfault_unlockall: unlock everything passed in.
1687 *
1688 * => maps must be read-locked (not write-locked).
1689 */
1690void
1691uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap,
1692 struct uvm_object *uobj)
1693{
1694 if (uobj)
1695 rw_exit(uobj->vmobjlock);
1696 if (amap != NULL((void *)0))
1697 amap_unlock(amap)rw_exit_write((amap)->am_lock);
1698 uvmfault_unlockmaps(ufi, FALSE0);
1699}
1700
1701/*
1702 * uvmfault_lookup: lookup a virtual address in a map
1703 *
1704 * => caller must provide a uvm_faultinfo structure with the IN
1705 * params properly filled in
1706 * => we will lookup the map entry (handling submaps) as we go
1707 * => if the lookup is a success we will return with the maps locked
1708 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only
1709 * get a read lock.
1710 * => note that submaps can only appear in the kernel and they are
1711 * required to use the same virtual addresses as the map they
1712 * are referenced by (thus address translation between the main
1713 * map and the submap is unnecessary).
1714 */
1715
1716boolean_t
1717uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock)
1718{
1719 vm_map_t tmpmap;
1720
1721 /*
1722 * init ufi values for lookup.
1723 */
1724 ufi->map = ufi->orig_map;
1725 ufi->size = ufi->orig_size;
1726
1727 /*
1728 * keep going down levels until we are done. note that there can
1729 * only be two levels so we won't loop very long.
1730 */
1731 while (1) {
1732 if (ufi->orig_rvaddr < ufi->map->min_offset ||
1733 ufi->orig_rvaddr >= ufi->map->max_offset)
1734 return FALSE0;
1735
1736 /* lock map */
1737 if (write_lock) {
1738 vm_map_lock(ufi->map)vm_map_lock_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c", 1738
);
1739 } else {
1740 vm_map_lock_read(ufi->map)vm_map_lock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1740);
1741 }
1742
1743 /* lookup */
1744 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr,
1745 &ufi->entry)) {
1746 uvmfault_unlockmaps(ufi, write_lock);
1747 return FALSE0;
1748 }
1749
1750 /* reduce size if necessary */
1751 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size)
1752 ufi->size = ufi->entry->end - ufi->orig_rvaddr;
1753
1754 /*
1755 * submap? replace map with the submap and lookup again.
1756 * note: VAs in submaps must match VAs in main map.
1757 */
1758 if (UVM_ET_ISSUBMAP(ufi->entry)(((ufi->entry)->etype & 0x0002) != 0)) {
1759 tmpmap = ufi->entry->object.sub_map;
1760 uvmfault_unlockmaps(ufi, write_lock);
1761 ufi->map = tmpmap;
1762 continue;
1763 }
1764
1765 /*
1766 * got it!
1767 */
1768 ufi->mapv = ufi->map->timestamp;
1769 return TRUE1;
1770
1771 } /* while loop */
1772
1773 /*NOTREACHED*/
1774}
1775
1776/*
1777 * uvmfault_relock: attempt to relock the same version of the map
1778 *
1779 * => fault data structures should be unlocked before calling.
1780 * => if a success (TRUE) maps will be locked after call.
1781 */
1782boolean_t
1783uvmfault_relock(struct uvm_faultinfo *ufi)
1784{
1785 /*
1786 * ufi can be NULL when this isn't really a fault,
1787 * but merely paging in anon data.
1788 */
1789 if (ufi == NULL((void *)0)) {
1790 return TRUE1;
1791 }
1792
1793 counters_inc(uvmexp_counters, flt_relck);
1794
1795 /*
1796 * relock map. fail if version mismatch (in which case nothing
1797 * gets locked).
1798 */
1799 vm_map_lock_read(ufi->map)vm_map_lock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1799);
1800 if (ufi->mapv != ufi->map->timestamp) {
1801 vm_map_unlock_read(ufi->map)vm_map_unlock_read_ln(ufi->map, "/usr/src/sys/uvm/uvm_fault.c"
, 1801);
1802 return FALSE0;
1803 }
1804
1805 counters_inc(uvmexp_counters, flt_relckok);
1806 return TRUE1; /* got it! */
1807}