Bug Summary

File:uvm/uvm_swap.c
Warning:line 1921, column 8
1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name uvm_swap.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/uvm/uvm_swap.c
1/* $OpenBSD: uvm_swap.c,v 1.168 2023/12/05 15:50:45 claudio Exp $ */
2/* $NetBSD: uvm_swap.c,v 1.40 2000/11/17 11:39:39 mrg Exp $ */
3
4/*
5 * Copyright (c) 1995, 1996, 1997 Matthew R. Green
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
22 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp
30 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp
31 */
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/buf.h>
36#include <sys/conf.h>
37#include <sys/proc.h>
38#include <sys/namei.h>
39#include <sys/disklabel.h>
40#include <sys/errno.h>
41#include <sys/kernel.h>
42#include <sys/malloc.h>
43#include <sys/vnode.h>
44#include <sys/fcntl.h>
45#include <sys/extent.h>
46#include <sys/blist.h>
47#include <sys/mount.h>
48#include <sys/mutex.h>
49#include <sys/pool.h>
50#include <sys/syscallargs.h>
51#include <sys/swap.h>
52#include <sys/disk.h>
53#include <sys/task.h>
54#include <sys/pledge.h>
55#if defined(NFSCLIENT1)
56#include <sys/socket.h>
57#include <netinet/in.h>
58#include <nfs/nfsproto.h>
59#include <nfs/nfsdiskless.h>
60#endif
61
62#include <uvm/uvm.h>
63#ifdef UVM_SWAP_ENCRYPT1
64#include <uvm/uvm_swap_encrypt.h>
65#endif
66
67#include <sys/specdev.h>
68
69#include "vnd.h"
70
71/*
72 * uvm_swap.c: manage configuration and i/o to swap space.
73 */
74
75/*
76 * swap space is managed in the following way:
77 *
78 * each swap partition or file is described by a "swapdev" structure.
79 * each "swapdev" structure contains a "swapent" structure which contains
80 * information that is passed up to the user (via system calls).
81 *
82 * each swap partition is assigned a "priority" (int) which controls
83 * swap partition usage.
84 *
85 * the system maintains a global data structure describing all swap
86 * partitions/files. there is a sorted LIST of "swappri" structures
87 * which describe "swapdev"'s at that priority. this LIST is headed
88 * by the "swap_priority" global var. each "swappri" contains a
89 * TAILQ of "swapdev" structures at that priority.
90 *
91 * locking:
92 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl
93 * system call and prevents the swap priority list from changing
94 * while we are in the middle of a system call (e.g. SWAP_STATS).
95 * - uvm_swap_data_lock (mutex): this lock protects all swap data
96 * structures including the priority list, the swapdev structures,
97 * and the swapmap arena.
98 *
99 * each swap device has the following info:
100 * - swap device in use (could be disabled, preventing future use)
101 * - swap enabled (allows new allocations on swap)
102 * - map info in /dev/drum
103 * - vnode pointer
104 * for swap files only:
105 * - block size
106 * - max byte count in buffer
107 * - buffer
108 * - credentials to use when doing i/o to file
109 *
110 * userland controls and configures swap with the swapctl(2) system call.
111 * the sys_swapctl performs the following operations:
112 * [1] SWAP_NSWAP: returns the number of swap devices currently configured
113 * [2] SWAP_STATS: given a pointer to an array of swapent structures
114 * (passed in via "arg") of a size passed in via "misc" ... we load
115 * the current swap config into the array.
116 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a
117 * priority in "misc", start swapping on it.
118 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device
119 * [5] SWAP_CTL: changes the priority of a swap device (new priority in
120 * "misc")
121 */
122
123/*
124 * swapdev: describes a single swap partition/file
125 *
126 * note the following should be true:
127 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks]
128 * swd_nblks <= swd_mapsize [because mapsize includes disklabel]
129 */
130struct swapdev {
131 struct swapent swd_se;
132#define swd_devswd_se.se_dev swd_se.se_dev /* device id */
133#define swd_flagsswd_se.se_flags swd_se.se_flags /* flags:inuse/enable/fake */
134#define swd_priorityswd_se.se_priority swd_se.se_priority /* our priority */
135#define swd_inuseswd_se.se_inuse swd_se.se_inuse /* blocks used */
136#define swd_nblksswd_se.se_nblks swd_se.se_nblks /* total blocks */
137 char *swd_path; /* saved pathname of device */
138 int swd_pathlen; /* length of pathname */
139 int swd_npages; /* #pages we can use */
140 int swd_npginuse; /* #pages in use */
141 int swd_npgbad; /* #pages bad */
142 int swd_drumoffset; /* page0 offset in drum */
143 int swd_drumsize; /* #pages in drum */
144 blist_t swd_blist; /* blist for this swapdev */
145 struct vnode *swd_vp; /* backing vnode */
146 TAILQ_ENTRY(swapdev)struct { struct swapdev *tqe_next; struct swapdev **tqe_prev;
} swd_next; /* priority tailq */
147
148 int swd_bsize; /* blocksize (bytes) */
149 int swd_maxactive; /* max active i/o reqs */
150 int swd_active; /* # of active i/o reqs */
151 struct bufq swd_bufq;
152 struct ucred *swd_cred; /* cred for file access */
153#ifdef UVM_SWAP_ENCRYPT1
154#define SWD_KEY_SHIFT7 7 /* One key per 0.5 MByte */
155#define SWD_KEY(x,y)&((x)->swd_keys[((y) - (x)->swd_drumoffset) >>
7]) &((x)->swd_keys[((y) - (x)->swd_drumoffset) >> SWD_KEY_SHIFT7])
156#define SWD_KEY_SIZE(x)(((x) + (1 << 7) - 1) >> 7) (((x) + (1 << SWD_KEY_SHIFT7) - 1) >> SWD_KEY_SHIFT7)
157
158#define SWD_DCRYPT_SHIFT5 5
159#define SWD_DCRYPT_BITS32 32
160#define SWD_DCRYPT_MASK(32 - 1) (SWD_DCRYPT_BITS32 - 1)
161#define SWD_DCRYPT_OFF(x)((x) >> 5) ((x) >> SWD_DCRYPT_SHIFT5)
162#define SWD_DCRYPT_BIT(x)((x) & (32 - 1)) ((x) & SWD_DCRYPT_MASK(32 - 1))
163#define SWD_DCRYPT_SIZE(x)((((x) + (32 - 1)) >> 5) * sizeof(u_int32_t)) (SWD_DCRYPT_OFF((x) + SWD_DCRYPT_MASK)(((x) + (32 - 1)) >> 5) * sizeof(u_int32_t))
164 u_int32_t *swd_decrypt; /* bitmap for decryption */
165 struct swap_key *swd_keys; /* keys for different parts */
166#endif
167};
168
169/*
170 * swap device priority entry; the list is kept sorted on `spi_priority'.
171 */
172struct swappri {
173 int spi_priority; /* priority */
174 TAILQ_HEAD(spi_swapdev, swapdev)struct spi_swapdev { struct swapdev *tqh_first; struct swapdev
**tqh_last; } spi_swapdev;
175 /* tailq of swapdevs at this priority */
176 LIST_ENTRY(swappri)struct { struct swappri *le_next; struct swappri **le_prev; } spi_swappri; /* global list of pri's */
177};
178
179/*
180 * The following two structures are used to keep track of data transfers
181 * on swap devices associated with regular files.
182 * NOTE: this code is more or less a copy of vnd.c; we use the same
183 * structure names here to ease porting..
184 */
185struct vndxfer {
186 struct buf *vx_bp; /* Pointer to parent buffer */
187 struct swapdev *vx_sdp;
188 int vx_error;
189 int vx_pending; /* # of pending aux buffers */
190 int vx_flags;
191#define VX_BUSY1 1
192#define VX_DEAD2 2
193};
194
195struct vndbuf {
196 struct buf vb_buf;
197 struct vndxfer *vb_vnx;
198 struct task vb_task;
199};
200
201/*
202 * We keep a of pool vndbuf's and vndxfer structures.
203 */
204struct pool vndxfer_pool;
205struct pool vndbuf_pool;
206
207
208/*
209 * local variables
210 */
211struct extent *swapmap; /* controls the mapping of /dev/drum */
212
213/* list of all active swap devices [by priority] */
214LIST_HEAD(swap_priority, swappri)struct swap_priority { struct swappri *lh_first; };
215struct swap_priority swap_priority; /* [S] */
216
217/* locks */
218struct mutex uvm_swap_data_lock = MUTEX_INITIALIZER(IPL_MPFLOOR){ ((void *)0), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9))), 0x0 };
219struct rwlock swap_syscall_lock = RWLOCK_INITIALIZER("swplk"){ 0, "swplk" };
220
221struct mutex oommtx = MUTEX_INITIALIZER(IPL_VM){ ((void *)0), ((((0xa)) > 0x0 && ((0xa)) < 0x9
) ? 0x9 : ((0xa))), 0x0 };
222struct vm_page *oompps[SWCLUSTPAGES((64 * 1024) >> 12)];
223int oom = 0;
224
225/*
226 * prototypes
227 */
228void swapdrum_add(struct swapdev *, int);
229struct swapdev *swapdrum_getsdp(int);
230
231struct swapdev *swaplist_find(struct vnode *, int);
232void swaplist_insert(struct swapdev *,
233 struct swappri *, int);
234void swaplist_trim(void);
235
236int swap_on(struct proc *, struct swapdev *);
237int swap_off(struct proc *, struct swapdev *);
238
239void sw_reg_strategy(struct swapdev *, struct buf *, int);
240void sw_reg_iodone(struct buf *);
241void sw_reg_iodone_internal(void *);
242void sw_reg_start(struct swapdev *);
243
244int uvm_swap_io(struct vm_page **, int, int, int);
245
246void swapmount(void);
247int uvm_swap_allocpages(struct vm_page **, int, int);
248
249#ifdef UVM_SWAP_ENCRYPT1
250/* for swap encrypt */
251void uvm_swap_markdecrypt(struct swapdev *, int, int, int);
252boolean_t uvm_swap_needdecrypt(struct swapdev *, int);
253void uvm_swap_initcrypt(struct swapdev *, int);
254#endif
255
256/*
257 * uvm_swap_init: init the swap system data structures and locks
258 *
259 * => called at boot time from init_main.c after the filesystems
260 * are brought up (which happens after uvm_init())
261 */
262void
263uvm_swap_init(void)
264{
265 int error;
266
267 /*
268 * first, init the swap list, its counter, and its lock.
269 * then get a handle on the vnode for /dev/drum by using
270 * the its dev_t number ("swapdev", from MD conf.c).
271 */
272 LIST_INIT(&swap_priority)do { ((&swap_priority)->lh_first) = ((void *)0); } while
(0);
273 uvmexp.nswapdev = 0;
274
275 if (!swapdev_vp && bdevvp(swapdev, &swapdev_vp))
276 panic("uvm_swap_init: can't get vnode for swap device");
277
278 /*
279 * create swap block extent to map /dev/drum. The extent spans
280 * 1 to INT_MAX allows 2 gigablocks of swap space. Note that
281 * block 0 is reserved (used to indicate an allocation failure,
282 * or no allocation).
283 */
284 swapmap = extent_create("swapmap", 1, INT_MAX0x7fffffff,
285 M_VMSWAP92, 0, 0, EX_NOWAIT0x0000);
286 if (swapmap == 0)
287 panic("uvm_swap_init: extent_create failed");
288
289 /* allocate pools for structures used for swapping to files. */
290 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, IPL_BIO0x3, 0,
291 "swp vnx", NULL((void *)0));
292 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, IPL_BIO0x3, 0,
293 "swp vnd", NULL((void *)0));
294
295 /* allocate pages for OOM situations. */
296 error = uvm_swap_allocpages(oompps, SWCLUSTPAGES((64 * 1024) >> 12), UVM_PLA_NOWAIT0x0002);
297 KASSERT(error == 0)((error == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c"
, 297, "error == 0"));
298
299 /* Setup the initial swap partition */
300 swapmount();
301}
302
303#ifdef UVM_SWAP_ENCRYPT1
304void
305uvm_swap_initcrypt_all(void)
306{
307 struct swapdev *sdp;
308 struct swappri *spp;
309 int npages;
310
311
312 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
313 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
314 if (sdp->swd_decrypt == NULL((void *)0)) {
315 npages = dbtob((uint64_t)sdp->swd_nblks)(((uint64_t)sdp->swd_se.se_nblks) << 9) >>
316 PAGE_SHIFT12;
317 uvm_swap_initcrypt(sdp, npages);
318 }
319 }
320 }
321}
322
323void
324uvm_swap_initcrypt(struct swapdev *sdp, int npages)
325{
326 /*
327 * keep information if a page needs to be decrypted when we get it
328 * from the swap device.
329 * We cannot chance a malloc later, if we are doing ASYNC puts,
330 * we may not call malloc with M_WAITOK. This consumes only
331 * 8KB memory for a 256MB swap partition.
332 */
333 sdp->swd_decrypt = malloc(SWD_DCRYPT_SIZE(npages)((((npages) + (32 - 1)) >> 5) * sizeof(u_int32_t)), M_VMSWAP92,
334 M_WAITOK0x0001|M_ZERO0x0008);
335 sdp->swd_keys = mallocarray(SWD_KEY_SIZE(npages)(((npages) + (1 << 7) - 1) >> 7),
336 sizeof(struct swap_key), M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008);
337}
338
339#endif /* UVM_SWAP_ENCRYPT */
340
341int
342uvm_swap_allocpages(struct vm_page **pps, int npages, int flags)
343{
344 struct pglist pgl;
345 int error, i;
346
347 KASSERT(npages <= SWCLUSTPAGES)((npages <= ((64 * 1024) >> 12)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 347, "npages <= SWCLUSTPAGES"
));
348
349 TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last
= &(&pgl)->tqh_first; } while (0);
350again:
351 error = uvm_pglistalloc(npages * PAGE_SIZE(1 << 12), dma_constraint.ucr_low,
352 dma_constraint.ucr_high, 0, 0, &pgl, npages, flags);
353 if (error && (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc == uvm.pagedaemon_proc)) {
354 mtx_enter(&oommtx);
355 if (oom) {
356 msleep_nsec(&oom, &oommtx, PVM4 | PNORELOCK0x200,
357 "oom", INFSLP0xffffffffffffffffULL);
358 goto again;
359 }
360 oom = 1;
361 for (i = 0; i < npages; i++) {
362 pps[i] = oompps[i];
363 atomic_setbits_intx86_atomic_setbits_u32(&pps[i]->pg_flags, PG_BUSY0x00000001);
364 }
365 mtx_leave(&oommtx);
366 return 0;
367 }
368 if (error)
369 return error;
370
371 for (i = 0; i < npages; i++) {
372 pps[i] = TAILQ_FIRST(&pgl)((&pgl)->tqh_first);
373 /* *sigh* */
374 atomic_setbits_intx86_atomic_setbits_u32(&pps[i]->pg_flags, PG_BUSY0x00000001);
375 TAILQ_REMOVE(&pgl, pps[i], pageq)do { if (((pps[i])->pageq.tqe_next) != ((void *)0)) (pps[i
])->pageq.tqe_next->pageq.tqe_prev = (pps[i])->pageq
.tqe_prev; else (&pgl)->tqh_last = (pps[i])->pageq.
tqe_prev; *(pps[i])->pageq.tqe_prev = (pps[i])->pageq.tqe_next
; ((pps[i])->pageq.tqe_prev) = ((void *)-1); ((pps[i])->
pageq.tqe_next) = ((void *)-1); } while (0);
376 }
377
378 return 0;
379}
380
381void
382uvm_swap_freepages(struct vm_page **pps, int npages)
383{
384 int i;
385
386 if (pps[0] == oompps[0]) {
387 for (i = 0; i < npages; i++)
388 uvm_pageclean(pps[i]);
389
390 mtx_enter(&oommtx);
391 KASSERT(oom == 1)((oom == 1) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c"
, 391, "oom == 1"));
392 oom = 0;
393 mtx_leave(&oommtx);
394 wakeup(&oom);
395 return;
396 }
397
398 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
399 for (i = 0; i < npages; i++)
400 uvm_pagefree(pps[i]);
401 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
402
403}
404
405#ifdef UVM_SWAP_ENCRYPT1
406/*
407 * Mark pages on the swap device for later decryption
408 */
409
410void
411uvm_swap_markdecrypt(struct swapdev *sdp, int startslot, int npages,
412 int decrypt)
413{
414 int pagestart, i;
415 int off, bit;
416
417 if (!sdp)
418 return;
419
420 pagestart = startslot - sdp->swd_drumoffset;
421 for (i = 0; i < npages; i++, pagestart++) {
422 off = SWD_DCRYPT_OFF(pagestart)((pagestart) >> 5);
423 bit = SWD_DCRYPT_BIT(pagestart)((pagestart) & (32 - 1));
424 if (decrypt)
425 /* pages read need decryption */
426 sdp->swd_decrypt[off] |= 1 << bit;
427 else
428 /* pages read do not need decryption */
429 sdp->swd_decrypt[off] &= ~(1 << bit);
430 }
431}
432
433/*
434 * Check if the page that we got from disk needs to be decrypted
435 */
436
437boolean_t
438uvm_swap_needdecrypt(struct swapdev *sdp, int off)
439{
440 if (!sdp)
441 return FALSE0;
442
443 off -= sdp->swd_drumoffset;
444 return sdp->swd_decrypt[SWD_DCRYPT_OFF(off)((off) >> 5)] & (1 << SWD_DCRYPT_BIT(off)((off) & (32 - 1))) ?
445 TRUE1 : FALSE0;
446}
447
448void
449uvm_swap_finicrypt_all(void)
450{
451 struct swapdev *sdp;
452 struct swappri *spp;
453 struct swap_key *key;
454 unsigned int nkeys;
455
456 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
457 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
458 if (sdp->swd_decrypt == NULL((void *)0))
459 continue;
460
461 nkeys = dbtob((uint64_t)sdp->swd_nblks)(((uint64_t)sdp->swd_se.se_nblks) << 9) >> PAGE_SHIFT12;
462 key = sdp->swd_keys + (SWD_KEY_SIZE(nkeys)(((nkeys) + (1 << 7) - 1) >> 7) - 1);
463 do {
464 if (key->refcount != 0)
465 swap_key_delete(key);
466 } while (key-- != sdp->swd_keys);
467 }
468 }
469}
470#endif /* UVM_SWAP_ENCRYPT */
471
472/*
473 * swaplist functions: functions that operate on the list of swap
474 * devices on the system.
475 */
476
477/*
478 * swaplist_insert: insert swap device "sdp" into the global list
479 *
480 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock
481 * => caller must provide a newly allocated swappri structure (we will
482 * FREE it if we don't need it... this it to prevent allocation
483 * blocking here while adding swap)
484 */
485void
486swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority)
487{
488 struct swappri *spp, *pspp;
489
490 KASSERT(rw_write_held(&swap_syscall_lock))((rw_write_held(&swap_syscall_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 490, "rw_write_held(&swap_syscall_lock)"
));
491 MUTEX_ASSERT_LOCKED(&uvm_swap_data_lock)do { if (((&uvm_swap_data_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&uvm_swap_data_lock), __func__); } while (0);
492
493 /*
494 * find entry at or after which to insert the new device.
495 */
496 pspp = NULL((void *)0);
497 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
498 if (priority <= spp->spi_priority)
499 break;
500 pspp = spp;
501 }
502
503 /*
504 * new priority?
505 */
506 if (spp == NULL((void *)0) || spp->spi_priority != priority) {
507 spp = newspp; /* use newspp! */
508
509 spp->spi_priority = priority;
510 TAILQ_INIT(&spp->spi_swapdev)do { (&spp->spi_swapdev)->tqh_first = ((void *)0); (
&spp->spi_swapdev)->tqh_last = &(&spp->spi_swapdev
)->tqh_first; } while (0);
511
512 if (pspp)
513 LIST_INSERT_AFTER(pspp, spp, spi_swappri)do { if (((spp)->spi_swappri.le_next = (pspp)->spi_swappri
.le_next) != ((void *)0)) (pspp)->spi_swappri.le_next->
spi_swappri.le_prev = &(spp)->spi_swappri.le_next; (pspp
)->spi_swappri.le_next = (spp); (spp)->spi_swappri.le_prev
= &(pspp)->spi_swappri.le_next; } while (0);
514 else
515 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri)do { if (((spp)->spi_swappri.le_next = (&swap_priority
)->lh_first) != ((void *)0)) (&swap_priority)->lh_first
->spi_swappri.le_prev = &(spp)->spi_swappri.le_next
; (&swap_priority)->lh_first = (spp); (spp)->spi_swappri
.le_prev = &(&swap_priority)->lh_first; } while (0
);
516 } else {
517 /* we don't need a new priority structure, free it */
518 free(newspp, M_VMSWAP92, sizeof(*newspp));
519 }
520
521 /*
522 * priority found (or created). now insert on the priority's
523 * tailq list and bump the total number of swapdevs.
524 */
525 sdp->swd_priorityswd_se.se_priority = priority;
526 TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next)do { (sdp)->swd_next.tqe_next = ((void *)0); (sdp)->swd_next
.tqe_prev = (&spp->spi_swapdev)->tqh_last; *(&spp
->spi_swapdev)->tqh_last = (sdp); (&spp->spi_swapdev
)->tqh_last = &(sdp)->swd_next.tqe_next; } while (0
);
527 uvmexp.nswapdev++;
528}
529
530/*
531 * swaplist_find: find and optionally remove a swap device from the
532 * global list.
533 *
534 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock
535 * => we return the swapdev we found (and removed)
536 */
537struct swapdev *
538swaplist_find(struct vnode *vp, boolean_t remove)
539{
540 struct swapdev *sdp;
541 struct swappri *spp;
542
543 KASSERT(rw_write_held(&swap_syscall_lock))((rw_write_held(&swap_syscall_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 543, "rw_write_held(&swap_syscall_lock)"
));
544 MUTEX_ASSERT_LOCKED(&uvm_swap_data_lock)do { if (((&uvm_swap_data_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&uvm_swap_data_lock), __func__); } while (0);
545
546 /*
547 * search the lists for the requested vp
548 */
549 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
550 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
551 if (sdp->swd_vp != vp)
552 continue;
553 if (remove) {
554 TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next)do { if (((sdp)->swd_next.tqe_next) != ((void *)0)) (sdp)->
swd_next.tqe_next->swd_next.tqe_prev = (sdp)->swd_next.
tqe_prev; else (&spp->spi_swapdev)->tqh_last = (sdp
)->swd_next.tqe_prev; *(sdp)->swd_next.tqe_prev = (sdp)
->swd_next.tqe_next; ((sdp)->swd_next.tqe_prev) = ((void
*)-1); ((sdp)->swd_next.tqe_next) = ((void *)-1); } while
(0);
555 uvmexp.nswapdev--;
556 }
557 return (sdp);
558 }
559 }
560 return (NULL((void *)0));
561}
562
563
564/*
565 * swaplist_trim: scan priority list for empty priority entries and kill
566 * them.
567 *
568 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock
569 */
570void
571swaplist_trim(void)
572{
573 struct swappri *spp, *nextspp;
574
575 KASSERT(rw_write_held(&swap_syscall_lock))((rw_write_held(&swap_syscall_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 575, "rw_write_held(&swap_syscall_lock)"
));
576 MUTEX_ASSERT_LOCKED(&uvm_swap_data_lock)do { if (((&uvm_swap_data_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&uvm_swap_data_lock), __func__); } while (0);
577
578 LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp)for ((spp) = ((&swap_priority)->lh_first); (spp) &&
((nextspp) = ((spp)->spi_swappri.le_next), 1); (spp) = (nextspp
)) {
579 if (!TAILQ_EMPTY(&spp->spi_swapdev)(((&spp->spi_swapdev)->tqh_first) == ((void *)0)))
580 continue;
581 LIST_REMOVE(spp, spi_swappri)do { if ((spp)->spi_swappri.le_next != ((void *)0)) (spp)->
spi_swappri.le_next->spi_swappri.le_prev = (spp)->spi_swappri
.le_prev; *(spp)->spi_swappri.le_prev = (spp)->spi_swappri
.le_next; ((spp)->spi_swappri.le_prev) = ((void *)-1); ((spp
)->spi_swappri.le_next) = ((void *)-1); } while (0);
582 free(spp, M_VMSWAP92, sizeof(*spp));
583 }
584}
585
586/*
587 * swapdrum_add: add a "swapdev"'s blocks into /dev/drum's area.
588 *
589 * => caller must hold swap_syscall_lock
590 * => uvm_swap_data_lock should be unlocked (we may sleep)
591 */
592void
593swapdrum_add(struct swapdev *sdp, int npages)
594{
595 u_long result;
596
597 if (extent_alloc(swapmap, npages, EX_NOALIGN, 0, EX_NOBOUNDARY,extent_alloc_subregion((swapmap), (swapmap)->ex_start, (swapmap
)->ex_end, (npages), (1), (0), (0), (0x0001), (&result
))
598 EX_WAITOK, &result)extent_alloc_subregion((swapmap), (swapmap)->ex_start, (swapmap
)->ex_end, (npages), (1), (0), (0), (0x0001), (&result
)))
599 panic("swapdrum_add");
600
601 sdp->swd_drumoffset = result;
602 sdp->swd_drumsize = npages;
603}
604
605/*
606 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back
607 * to the "swapdev" that maps that section of the drum.
608 *
609 * => each swapdev takes one big contig chunk of the drum
610 * => caller must hold uvm_swap_data_lock
611 */
612struct swapdev *
613swapdrum_getsdp(int pgno)
614{
615 struct swapdev *sdp;
616 struct swappri *spp;
617
618 MUTEX_ASSERT_LOCKED(&uvm_swap_data_lock)do { if (((&uvm_swap_data_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&uvm_swap_data_lock), __func__); } while (0);
619
620 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
621 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
622 if (pgno >= sdp->swd_drumoffset &&
623 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) {
624 return sdp;
625 }
626 }
627 }
628 return NULL((void *)0);
629}
630
631
632/*
633 * sys_swapctl: main entry point for swapctl(2) system call
634 * [with two helper functions: swap_on and swap_off]
635 */
636int
637sys_swapctl(struct proc *p, void *v, register_t *retval)
638{
639 struct sys_swapctl_args /* {
640 syscallarg(int) cmd;
641 syscallarg(void *) arg;
642 syscallarg(int) misc;
643 } */ *uap = (struct sys_swapctl_args *)v;
644 struct vnode *vp;
645 struct nameidata nd;
646 struct swappri *spp;
647 struct swapdev *sdp;
648 struct swapent *sep;
649 char userpath[MAXPATHLEN1024];
650 size_t len;
651 int count, error, misc;
652 int priority;
653
654 misc = SCARG(uap, misc)((uap)->misc.le.datum);
655
656 if ((error = pledge_swapctl(p, SCARG(uap, cmd)((uap)->cmd.le.datum))))
657 return error;
658
659 /*
660 * ensure serialized syscall access by grabbing the swap_syscall_lock
661 */
662 rw_enter_write(&swap_syscall_lock);
663
664 /*
665 * we handle the non-priv NSWAP and STATS request first.
666 *
667 * SWAP_NSWAP: return number of config'd swap devices
668 * [can also be obtained with uvmexp sysctl]
669 */
670 if (SCARG(uap, cmd)((uap)->cmd.le.datum) == SWAP_NSWAP3) {
671 *retval = uvmexp.nswapdev;
672 error = 0;
673 goto out;
674 }
675
676 /*
677 * SWAP_STATS: get stats on current # of configured swap devs
678 *
679 * note that the swap_priority list can't change as long
680 * as we are holding the swap_syscall_lock. we don't want
681 * to grab the uvm_swap_data_lock because we may fault&sleep during
682 * copyout() and we don't want to be holding that lock then!
683 */
684 if (SCARG(uap, cmd)((uap)->cmd.le.datum) == SWAP_STATS4) {
685 sep = (struct swapent *)SCARG(uap, arg)((uap)->arg.le.datum);
686 count = 0;
687
688 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
689 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
690 if (count >= misc)
691 continue;
692
693 sdp->swd_inuseswd_se.se_inuse =
694 btodb((u_int64_t)sdp->swd_npginuse <<(((u_int64_t)sdp->swd_npginuse << 12) >> 9)
695 PAGE_SHIFT)(((u_int64_t)sdp->swd_npginuse << 12) >> 9);
696 error = copyout(&sdp->swd_se, sep,
697 sizeof(struct swapent));
698 if (error)
699 goto out;
700
701 /* now copy out the path if necessary */
702 error = copyoutstr(sdp->swd_path,
703 sep->se_path, sizeof(sep->se_path), NULL((void *)0));
704 if (error)
705 goto out;
706
707 count++;
708 sep++;
709 }
710 }
711
712 *retval = count;
713 error = 0;
714 goto out;
715 }
716
717 /* all other requests require superuser privs. verify. */
718 if ((error = suser(p)))
719 goto out;
720
721 /*
722 * at this point we expect a path name in arg. we will
723 * use namei() to gain a vnode reference (vref), and lock
724 * the vnode (VOP_LOCK).
725 */
726 error = copyinstr(SCARG(uap, arg)((uap)->arg.le.datum), userpath, sizeof(userpath), &len);
727 if (error)
728 goto out;
729 disk_map(userpath, userpath, sizeof(userpath), DM_OPENBLCK0x2);
730 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, UIO_SYSSPACE, userpath, p)ndinitat(&nd, 0, 0x0040|0x0004, UIO_SYSSPACE, -100, userpath
, p);
731 if ((error = namei(&nd)))
732 goto out;
733 vp = nd.ni_vp;
734 /* note: "vp" is referenced and locked */
735
736 error = 0; /* assume no error */
737 switch(SCARG(uap, cmd)((uap)->cmd.le.datum)) {
738 case SWAP_DUMPDEV7:
739 if (vp->v_type != VBLK) {
740 error = ENOTBLK15;
741 break;
742 }
743 dumpdev = vp->v_rdevv_un.vu_specinfo->si_rdev;
744 break;
745 case SWAP_CTL5:
746 /*
747 * get new priority, remove old entry (if any) and then
748 * reinsert it in the correct place. finally, prune out
749 * any empty priority structures.
750 */
751 priority = SCARG(uap, misc)((uap)->misc.le.datum);
752 spp = malloc(sizeof *spp, M_VMSWAP92, M_WAITOK0x0001);
753 mtx_enter(&uvm_swap_data_lock);
754 if ((sdp = swaplist_find(vp, 1)) == NULL((void *)0)) {
755 error = ENOENT2;
756 } else {
757 swaplist_insert(sdp, spp, priority);
758 swaplist_trim();
759 }
760 mtx_leave(&uvm_swap_data_lock);
761 if (error)
762 free(spp, M_VMSWAP92, sizeof(*spp));
763 break;
764 case SWAP_ON1:
765 /*
766 * If the device is a regular file, make sure the filesystem
767 * can be used for swapping.
768 */
769 if (vp->v_type == VREG &&
770 (vp->v_mount->mnt_flag & MNT_SWAPPABLE0x00200000) == 0) {
771 error = ENOTSUP91;
772 break;
773 }
774
775 /*
776 * check for duplicates. if none found, then insert a
777 * dummy entry on the list to prevent someone else from
778 * trying to enable this device while we are working on
779 * it.
780 */
781
782 priority = SCARG(uap, misc)((uap)->misc.le.datum);
783 sdp = malloc(sizeof *sdp, M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008);
784 spp = malloc(sizeof *spp, M_VMSWAP92, M_WAITOK0x0001);
785 sdp->swd_flagsswd_se.se_flags = SWF_FAKE0x00000008; /* placeholder only */
786 sdp->swd_vp = vp;
787 sdp->swd_devswd_se.se_dev = (vp->v_type == VBLK) ? vp->v_rdevv_un.vu_specinfo->si_rdev : NODEV(dev_t)(-1);
788
789 /*
790 * XXX Is NFS elaboration necessary?
791 */
792 if (vp->v_type == VREG) {
793 sdp->swd_cred = crdup(p->p_ucred);
794 }
795
796 mtx_enter(&uvm_swap_data_lock);
797 if (swaplist_find(vp, 0) != NULL((void *)0)) {
798 error = EBUSY16;
799 mtx_leave(&uvm_swap_data_lock);
800 if (vp->v_type == VREG) {
801 crfree(sdp->swd_cred);
802 }
803 free(sdp, M_VMSWAP92, sizeof *sdp);
804 free(spp, M_VMSWAP92, sizeof *spp);
805 break;
806 }
807 swaplist_insert(sdp, spp, priority);
808 mtx_leave(&uvm_swap_data_lock);
809
810 sdp->swd_pathlen = len;
811 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP92, M_WAITOK0x0001);
812 strlcpy(sdp->swd_path, userpath, len);
813
814 /*
815 * we've now got a FAKE placeholder in the swap list.
816 * now attempt to enable swap on it. if we fail, undo
817 * what we've done and kill the fake entry we just inserted.
818 * if swap_on is a success, it will clear the SWF_FAKE flag
819 */
820
821 if ((error = swap_on(p, sdp)) != 0) {
822 mtx_enter(&uvm_swap_data_lock);
823 (void) swaplist_find(vp, 1); /* kill fake entry */
824 swaplist_trim();
825 mtx_leave(&uvm_swap_data_lock);
826 if (vp->v_type == VREG) {
827 crfree(sdp->swd_cred);
828 }
829 free(sdp->swd_path, M_VMSWAP92, sdp->swd_pathlen);
830 free(sdp, M_VMSWAP92, sizeof(*sdp));
831 break;
832 }
833 break;
834 case SWAP_OFF2:
835 mtx_enter(&uvm_swap_data_lock);
836 if ((sdp = swaplist_find(vp, 0)) == NULL((void *)0)) {
837 mtx_leave(&uvm_swap_data_lock);
838 error = ENXIO6;
839 break;
840 }
841
842 /*
843 * If a device isn't in use or enabled, we
844 * can't stop swapping from it (again).
845 */
846 if ((sdp->swd_flagsswd_se.se_flags & (SWF_INUSE0x00000001|SWF_ENABLE0x00000002)) == 0) {
847 mtx_leave(&uvm_swap_data_lock);
848 error = EBUSY16;
849 break;
850 }
851
852 /*
853 * do the real work.
854 */
855 error = swap_off(p, sdp);
856 break;
857 default:
858 error = EINVAL22;
859 }
860
861 /* done! release the ref gained by namei() and unlock. */
862 vput(vp);
863
864out:
865 rw_exit_write(&swap_syscall_lock);
866
867 return (error);
868}
869
870/*
871 * swap_on: attempt to enable a swapdev for swapping. note that the
872 * swapdev is already on the global list, but disabled (marked
873 * SWF_FAKE).
874 *
875 * => we avoid the start of the disk (to protect disk labels)
876 * => caller should leave uvm_swap_data_lock unlocked, we may lock it
877 * if needed.
878 */
879int
880swap_on(struct proc *p, struct swapdev *sdp)
881{
882 struct vnode *vp;
883 int error, npages, nblocks, size;
884 long addr;
885 struct vattr va;
886#if defined(NFSCLIENT1)
887 extern const struct vops nfs_vops;
888#endif /* defined(NFSCLIENT) */
889 dev_t dev;
890
891 /*
892 * we want to enable swapping on sdp. the swd_vp contains
893 * the vnode we want (locked and ref'd), and the swd_dev
894 * contains the dev_t of the file, if it a block device.
895 */
896
897 vp = sdp->swd_vp;
898 dev = sdp->swd_devswd_se.se_dev;
899
900#if NVND1 > 0
901 /* no swapping to vnds. */
902 if (bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_strategy == vndstrategy)
903 return (EOPNOTSUPP45);
904#endif
905
906 /*
907 * open the swap file (mostly useful for block device files to
908 * let device driver know what is up).
909 *
910 * we skip the open/close for root on swap because the root
911 * has already been opened when root was mounted (mountroot).
912 */
913 if (vp != rootvp) {
914 if ((error = VOP_OPEN(vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p)))
915 return (error);
916 }
917
918 /* XXX this only works for block devices */
919 /*
920 * we now need to determine the size of the swap area. for
921 * block specials we can call the d_psize function.
922 * for normal files, we must stat [get attrs].
923 *
924 * we put the result in nblks.
925 * for normal files, we also want the filesystem block size
926 * (which we get with statfs).
927 */
928 switch (vp->v_type) {
929 case VBLK:
930 if (bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_psize == 0 ||
931 (nblocks = (*bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_psize)(dev)) == -1) {
932 error = ENXIO6;
933 goto bad;
934 }
935 break;
936
937 case VREG:
938 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p)))
939 goto bad;
940 nblocks = (int)btodb(va.va_size)((va.va_size) >> 9);
941 if ((error =
942 VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)(*(vp->v_mount)->mnt_op->vfs_statfs)(vp->v_mount,
&vp->v_mount->mnt_stat, p)) != 0)
943 goto bad;
944
945 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize;
946 /*
947 * limit the max # of outstanding I/O requests we issue
948 * at any one time. take it easy on NFS servers.
949 */
950#if defined(NFSCLIENT1)
951 if (vp->v_op == &nfs_vops)
952 sdp->swd_maxactive = 2; /* XXX */
953 else
954#endif /* defined(NFSCLIENT) */
955 sdp->swd_maxactive = 8; /* XXX */
956 bufq_init(&sdp->swd_bufq, BUFQ_FIFO0);
957 break;
958
959 default:
960 error = ENXIO6;
961 goto bad;
962 }
963
964 /*
965 * save nblocks in a safe place and convert to pages.
966 */
967
968 sdp->swd_nblksswd_se.se_nblks = nblocks;
969 npages = dbtob((u_int64_t)nblocks)(((u_int64_t)nblocks) << 9) >> PAGE_SHIFT12;
970
971 /*
972 * for block special files, we want to make sure that leave
973 * the disklabel and bootblocks alone, so we arrange to skip
974 * over them (arbitrarily choosing to skip PAGE_SIZE bytes).
975 * note that because of this the "size" can be less than the
976 * actual number of blocks on the device.
977 */
978 if (vp->v_type == VBLK) {
979 /* we use pages 1 to (size - 1) [inclusive] */
980 size = npages - 1;
981 addr = 1;
982 } else {
983 /* we use pages 0 to (size - 1) [inclusive] */
984 size = npages;
985 addr = 0;
986 }
987
988 /*
989 * make sure we have enough blocks for a reasonable sized swap
990 * area. we want at least one page.
991 */
992
993 if (size < 1) {
994 error = EINVAL22;
995 goto bad;
996 }
997
998 /*
999 * now we need to allocate a blist to manage this swap device
1000 */
1001 sdp->swd_blist = blist_create(npages);
1002 /* mark all expect the `saved' region free. */
1003 blist_free(sdp->swd_blist, addr, size);
1004
1005#ifdef HIBERNATE1
1006 /*
1007 * Lock down the last region of primary disk swap, in case
1008 * hibernate needs to place a signature there.
1009 */
1010 if (dev == swdevt[0].sw_dev && vp->v_type == VBLK && size > 3 ) {
1011 if (blist_fill(sdp->swd_blist, npages - 1, 1) != 1)
1012 panic("hibernate reserve");
1013 }
1014#endif
1015
1016 /* add a ref to vp to reflect usage as a swap device. */
1017 vref(vp);
1018
1019#ifdef UVM_SWAP_ENCRYPT1
1020 if (uvm_doswapencrypt)
1021 uvm_swap_initcrypt(sdp, npages);
1022#endif
1023 /* now add the new swapdev to the drum and enable. */
1024 swapdrum_add(sdp, npages);
1025 sdp->swd_npages = size;
1026 mtx_enter(&uvm_swap_data_lock);
1027 sdp->swd_flagsswd_se.se_flags &= ~SWF_FAKE0x00000008; /* going live */
1028 sdp->swd_flagsswd_se.se_flags |= (SWF_INUSE0x00000001|SWF_ENABLE0x00000002);
1029 uvmexp.swpages += size;
1030 mtx_leave(&uvm_swap_data_lock);
1031 return (0);
1032
1033 /*
1034 * failure: clean up and return error.
1035 */
1036
1037bad:
1038 if (vp != rootvp)
1039 (void)VOP_CLOSE(vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p);
1040 return (error);
1041}
1042
1043/*
1044 * swap_off: stop swapping on swapdev
1045 *
1046 * => swap data should be locked, we will unlock.
1047 */
1048int
1049swap_off(struct proc *p, struct swapdev *sdp)
1050{
1051 int npages = sdp->swd_npages;
1052 int error = 0;
1053
1054 KASSERT(rw_write_held(&swap_syscall_lock))((rw_write_held(&swap_syscall_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 1054, "rw_write_held(&swap_syscall_lock)"
));
1055 MUTEX_ASSERT_LOCKED(&uvm_swap_data_lock)do { if (((&uvm_swap_data_lock)->mtx_owner != ({struct
cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})) && !(panicstr || db_active)) panic("mutex %p not held in %s"
, (&uvm_swap_data_lock), __func__); } while (0);
1056
1057 /* disable the swap area being removed */
1058 sdp->swd_flagsswd_se.se_flags &= ~SWF_ENABLE0x00000002;
1059 mtx_leave(&uvm_swap_data_lock);
1060
1061 /*
1062 * the idea is to find all the pages that are paged out to this
1063 * device, and page them all in. in uvm, swap-backed pageable
1064 * memory can take two forms: aobjs and anons. call the
1065 * swapoff hook for each subsystem to bring in pages.
1066 */
1067
1068 if (uao_swap_off(sdp->swd_drumoffset,
1069 sdp->swd_drumoffset + sdp->swd_drumsize) ||
1070 amap_swap_off(sdp->swd_drumoffset,
1071 sdp->swd_drumoffset + sdp->swd_drumsize)) {
1072 error = ENOMEM12;
1073 } else if (sdp->swd_npginuse > sdp->swd_npgbad) {
1074 error = EBUSY16;
1075 }
1076
1077 if (error) {
1078 mtx_enter(&uvm_swap_data_lock);
1079 sdp->swd_flagsswd_se.se_flags |= SWF_ENABLE0x00000002;
1080 mtx_leave(&uvm_swap_data_lock);
1081 return error;
1082 }
1083
1084 /*
1085 * done with the vnode and saved creds.
1086 * drop our ref on the vnode before calling VOP_CLOSE()
1087 * so that spec_close() can tell if this is the last close.
1088 */
1089 if (sdp->swd_vp->v_type == VREG) {
1090 crfree(sdp->swd_cred);
1091 }
1092 vrele(sdp->swd_vp);
1093 if (sdp->swd_vp != rootvp) {
1094 (void) VOP_CLOSE(sdp->swd_vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p);
1095 }
1096
1097 mtx_enter(&uvm_swap_data_lock);
1098 uvmexp.swpages -= npages;
1099
1100 if (swaplist_find(sdp->swd_vp, 1) == NULL((void *)0))
1101 panic("swap_off: swapdev not in list");
1102 swaplist_trim();
1103 mtx_leave(&uvm_swap_data_lock);
1104
1105 /*
1106 * free all resources!
1107 */
1108 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize,
1109 EX_WAITOK0x0001);
1110 blist_destroy(sdp->swd_blist);
1111 /* free sdp->swd_path ? */
1112 free(sdp, M_VMSWAP92, sizeof(*sdp));
1113 return (0);
1114}
1115
1116/*
1117 * /dev/drum interface and i/o functions
1118 */
1119
1120/*
1121 * swstrategy: perform I/O on the drum
1122 *
1123 * => we must map the i/o request from the drum to the correct swapdev.
1124 */
1125void
1126swstrategy(struct buf *bp)
1127{
1128 struct swapdev *sdp;
1129 int s, pageno, bn;
1130
1131 /*
1132 * convert block number to swapdev. note that swapdev can't
1133 * be yanked out from under us because we are holding resources
1134 * in it (i.e. the blocks we are doing I/O on).
1135 */
1136 pageno = dbtob((u_int64_t)bp->b_blkno)(((u_int64_t)bp->b_blkno) << 9) >> PAGE_SHIFT12;
1137 mtx_enter(&uvm_swap_data_lock);
1138 sdp = swapdrum_getsdp(pageno);
1139 mtx_leave(&uvm_swap_data_lock);
1140 if (sdp == NULL((void *)0)) {
1141 bp->b_error = EINVAL22;
1142 bp->b_flags |= B_ERROR0x00000400;
1143 s = splbio()splraise(0x3);
1144 biodone(bp);
1145 splx(s)spllower(s);
1146 return;
1147 }
1148
1149 /* convert drum page number to block number on this swapdev. */
1150 pageno -= sdp->swd_drumoffset; /* page # on swapdev */
1151 bn = btodb((u_int64_t)pageno << PAGE_SHIFT)(((u_int64_t)pageno << 12) >> 9); /* convert to diskblock */
1152
1153 /*
1154 * for block devices we finish up here.
1155 * for regular files we have to do more work which we delegate
1156 * to sw_reg_strategy().
1157 */
1158 switch (sdp->swd_vp->v_type) {
1159 default:
1160 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type);
1161 case VBLK:
1162 /*
1163 * must convert "bp" from an I/O on /dev/drum to an I/O
1164 * on the swapdev (sdp).
1165 */
1166 s = splbio()splraise(0x3);
1167 buf_replacevnode(bp, sdp->swd_vp);
1168
1169 bp->b_blkno = bn;
1170 splx(s)spllower(s);
1171 VOP_STRATEGY(bp->b_vp, bp);
1172 return;
1173 case VREG:
1174 /* delegate to sw_reg_strategy function. */
1175 sw_reg_strategy(sdp, bp, bn);
1176 return;
1177 }
1178 /* NOTREACHED */
1179}
1180
1181/*
1182 * sw_reg_strategy: handle swap i/o to regular files
1183 */
1184void
1185sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn)
1186{
1187 struct vnode *vp;
1188 struct vndxfer *vnx;
1189 daddr_t nbn;
1190 caddr_t addr;
1191 off_t byteoff;
1192 int s, off, nra, error, sz, resid;
1193
1194 /*
1195 * allocate a vndxfer head for this transfer and point it to
1196 * our buffer.
1197 */
1198 vnx = pool_get(&vndxfer_pool, PR_WAITOK0x0001);
1199 vnx->vx_flags = VX_BUSY1;
1200 vnx->vx_error = 0;
1201 vnx->vx_pending = 0;
1202 vnx->vx_bp = bp;
1203 vnx->vx_sdp = sdp;
1204
1205 /*
1206 * setup for main loop where we read filesystem blocks into
1207 * our buffer.
1208 */
1209 error = 0;
1210 bp->b_resid = bp->b_bcount; /* nothing transferred yet! */
1211 addr = bp->b_data; /* current position in buffer */
1212 byteoff = dbtob((u_int64_t)bn)(((u_int64_t)bn) << 9);
1213
1214 for (resid = bp->b_resid; resid; resid -= sz) {
1215 struct vndbuf *nbp;
1216 /*
1217 * translate byteoffset into block number. return values:
1218 * vp = vnode of underlying device
1219 * nbn = new block number (on underlying vnode dev)
1220 * nra = num blocks we can read-ahead (excludes requested
1221 * block)
1222 */
1223 nra = 0;
1224 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize,
1225 &vp, &nbn, &nra);
1226
1227 if (error == 0 && nbn == -1) {
1228 /*
1229 * this used to just set error, but that doesn't
1230 * do the right thing. Instead, it causes random
1231 * memory errors. The panic() should remain until
1232 * this condition doesn't destabilize the system.
1233 */
1234#if 1
1235 panic("sw_reg_strategy: swap to sparse file");
1236#else
1237 error = EIO5; /* failure */
1238#endif
1239 }
1240
1241 /*
1242 * punt if there was an error or a hole in the file.
1243 * we must wait for any i/o ops we have already started
1244 * to finish before returning.
1245 *
1246 * XXX we could deal with holes here but it would be
1247 * a hassle (in the write case).
1248 */
1249 if (error) {
1250 s = splbio()splraise(0x3);
1251 vnx->vx_error = error; /* pass error up */
1252 goto out;
1253 }
1254
1255 /*
1256 * compute the size ("sz") of this transfer (in bytes).
1257 */
1258 off = byteoff % sdp->swd_bsize;
1259 sz = (1 + nra) * sdp->swd_bsize - off;
1260 if (sz > resid)
1261 sz = resid;
1262
1263 /*
1264 * now get a buf structure. note that the vb_buf is
1265 * at the front of the nbp structure so that you can
1266 * cast pointers between the two structure easily.
1267 */
1268 nbp = pool_get(&vndbuf_pool, PR_WAITOK0x0001);
1269 nbp->vb_buf.b_flags = bp->b_flags | B_CALL0x00000040;
1270 nbp->vb_buf.b_bcount = sz;
1271 nbp->vb_buf.b_bufsize = sz;
1272 nbp->vb_buf.b_error = 0;
1273 nbp->vb_buf.b_data = addr;
1274 nbp->vb_buf.b_bq = NULL((void *)0);
1275 nbp->vb_buf.b_blkno = nbn + btodb(off)((off) >> 9);
1276 nbp->vb_buf.b_proc = bp->b_proc;
1277 nbp->vb_buf.b_iodone = sw_reg_iodone;
1278 nbp->vb_buf.b_vp = NULLVP((struct vnode *)((void *)0));
1279 nbp->vb_buf.b_vnbufs.le_next = NOLIST((struct buf *)0x87654321);
1280 LIST_INIT(&nbp->vb_buf.b_dep)do { ((&nbp->vb_buf.b_dep)->lh_first) = ((void *)0)
; } while (0);
1281
1282 /*
1283 * set b_dirtyoff/end and b_validoff/end. this is
1284 * required by the NFS client code (otherwise it will
1285 * just discard our I/O request).
1286 */
1287 if (bp->b_dirtyend == 0) {
1288 nbp->vb_buf.b_dirtyoff = 0;
1289 nbp->vb_buf.b_dirtyend = sz;
1290 } else {
1291 nbp->vb_buf.b_dirtyoff =
1292 max(0, bp->b_dirtyoff - (bp->b_bcount-resid));
1293 nbp->vb_buf.b_dirtyend =
1294 min(sz,
1295 max(0, bp->b_dirtyend - (bp->b_bcount-resid)));
1296 }
1297 if (bp->b_validend == 0) {
1298 nbp->vb_buf.b_validoff = 0;
1299 nbp->vb_buf.b_validend = sz;
1300 } else {
1301 nbp->vb_buf.b_validoff =
1302 max(0, bp->b_validoff - (bp->b_bcount-resid));
1303 nbp->vb_buf.b_validend =
1304 min(sz,
1305 max(0, bp->b_validend - (bp->b_bcount-resid)));
1306 }
1307
1308 /* patch it back to the vnx */
1309 nbp->vb_vnx = vnx;
1310 task_set(&nbp->vb_task, sw_reg_iodone_internal, nbp);
1311
1312 s = splbio()splraise(0x3);
1313 if (vnx->vx_error != 0) {
1314 pool_put(&vndbuf_pool, nbp);
1315 goto out;
1316 }
1317 vnx->vx_pending++;
1318
1319 /* assoc new buffer with underlying vnode */
1320 bgetvp(vp, &nbp->vb_buf);
1321
1322 /* start I/O if we are not over our limit */
1323 bufq_queue(&sdp->swd_bufq, &nbp->vb_buf);
1324 sw_reg_start(sdp);
1325 splx(s)spllower(s);
1326
1327 /*
1328 * advance to the next I/O
1329 */
1330 byteoff += sz;
1331 addr += sz;
1332 }
1333
1334 s = splbio()splraise(0x3);
1335
1336out: /* Arrive here at splbio */
1337 vnx->vx_flags &= ~VX_BUSY1;
1338 if (vnx->vx_pending == 0) {
1339 if (vnx->vx_error != 0) {
1340 bp->b_error = vnx->vx_error;
1341 bp->b_flags |= B_ERROR0x00000400;
1342 }
1343 pool_put(&vndxfer_pool, vnx);
1344 biodone(bp);
1345 }
1346 splx(s)spllower(s);
1347}
1348
1349/* sw_reg_start: start an I/O request on the requested swapdev. */
1350void
1351sw_reg_start(struct swapdev *sdp)
1352{
1353 struct buf *bp;
1354
1355 /* XXX: recursion control */
1356 if ((sdp->swd_flagsswd_se.se_flags & SWF_BUSY0x00000004) != 0)
1357 return;
1358
1359 sdp->swd_flagsswd_se.se_flags |= SWF_BUSY0x00000004;
1360
1361 while (sdp->swd_active < sdp->swd_maxactive) {
1362 bp = bufq_dequeue(&sdp->swd_bufq);
1363 if (bp == NULL((void *)0))
1364 break;
1365
1366 sdp->swd_active++;
1367
1368 if ((bp->b_flags & B_READ0x00008000) == 0)
1369 bp->b_vp->v_numoutput++;
1370
1371 VOP_STRATEGY(bp->b_vp, bp);
1372 }
1373 sdp->swd_flagsswd_se.se_flags &= ~SWF_BUSY0x00000004;
1374}
1375
1376/*
1377 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup
1378 *
1379 * => note that we can recover the vndbuf struct by casting the buf ptr
1380 *
1381 * XXX:
1382 * We only put this onto a taskq here, because of the maxactive game since
1383 * it basically requires us to call back into VOP_STRATEGY() (where we must
1384 * be able to sleep) via sw_reg_start().
1385 */
1386void
1387sw_reg_iodone(struct buf *bp)
1388{
1389 struct vndbuf *vbp = (struct vndbuf *)bp;
1390 task_add(systq, &vbp->vb_task);
1391}
1392
1393void
1394sw_reg_iodone_internal(void *xvbp)
1395{
1396 struct vndbuf *vbp = xvbp;
1397 struct vndxfer *vnx = vbp->vb_vnx;
1398 struct buf *pbp = vnx->vx_bp; /* parent buffer */
1399 struct swapdev *sdp = vnx->vx_sdp;
1400 int resid, s;
1401
1402 s = splbio()splraise(0x3);
1403
1404 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid;
1405 pbp->b_resid -= resid;
1406 vnx->vx_pending--;
1407
1408 /* pass error upward */
1409 if (vbp->vb_buf.b_error)
1410 vnx->vx_error = vbp->vb_buf.b_error;
1411
1412 /* disassociate this buffer from the vnode (if any). */
1413 if (vbp->vb_buf.b_vp != NULL((void *)0)) {
1414 brelvp(&vbp->vb_buf);
1415 }
1416
1417 /* kill vbp structure */
1418 pool_put(&vndbuf_pool, vbp);
1419
1420 /*
1421 * wrap up this transaction if it has run to completion or, in
1422 * case of an error, when all auxiliary buffers have returned.
1423 */
1424 if (vnx->vx_error != 0) {
1425 /* pass error upward */
1426 pbp->b_flags |= B_ERROR0x00000400;
1427 pbp->b_error = vnx->vx_error;
1428 if ((vnx->vx_flags & VX_BUSY1) == 0 && vnx->vx_pending == 0) {
1429 pool_put(&vndxfer_pool, vnx);
1430 biodone(pbp);
1431 }
1432 } else if (pbp->b_resid == 0) {
1433 KASSERT(vnx->vx_pending == 0)((vnx->vx_pending == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_swap.c", 1433, "vnx->vx_pending == 0"
));
1434 if ((vnx->vx_flags & VX_BUSY1) == 0) {
1435 pool_put(&vndxfer_pool, vnx);
1436 biodone(pbp);
1437 }
1438 }
1439
1440 /*
1441 * done! start next swapdev I/O if one is pending
1442 */
1443 sdp->swd_active--;
1444 sw_reg_start(sdp);
1445 splx(s)spllower(s);
1446}
1447
1448
1449/*
1450 * uvm_swap_alloc: allocate space on swap
1451 *
1452 * => allocation is done "round robin" down the priority list, as we
1453 * allocate in a priority we "rotate" the tail queue.
1454 * => space can be freed with uvm_swap_free
1455 * => we return the page slot number in /dev/drum (0 == invalid slot)
1456 * => we lock uvm_swap_data_lock
1457 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM
1458 */
1459int
1460uvm_swap_alloc(int *nslots, boolean_t lessok)
1461{
1462 struct swapdev *sdp;
1463 struct swappri *spp;
1464
1465 /*
1466 * no swap devices configured yet? definite failure.
1467 */
1468 if (uvmexp.nswapdev < 1)
1469 return 0;
1470
1471 /*
1472 * lock data lock, convert slots into blocks, and enter loop
1473 */
1474 KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c"
, 1474, "_kernel_lock_held()"));
1475 mtx_enter(&uvm_swap_data_lock);
1476
1477ReTry: /* XXXMRG */
1478 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
1479 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
1480 swblk_t result;
1481
1482 /* if it's not enabled, then we can't swap from it */
1483 if ((sdp->swd_flagsswd_se.se_flags & SWF_ENABLE0x00000002) == 0)
1484 continue;
1485 if (sdp->swd_npginuse + *nslots > sdp->swd_npages)
1486 continue;
1487 result = blist_alloc(sdp->swd_blist, *nslots);
1488 if (result == SWAPBLK_NONE((swblk_t)-1)) {
1489 continue;
1490 }
1491 KASSERT(result < sdp->swd_drumsize)((result < sdp->swd_drumsize) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_swap.c", 1491, "result < sdp->swd_drumsize"
));
1492
1493 /*
1494 * successful allocation! now rotate the tailq.
1495 */
1496 TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next)do { if (((sdp)->swd_next.tqe_next) != ((void *)0)) (sdp)->
swd_next.tqe_next->swd_next.tqe_prev = (sdp)->swd_next.
tqe_prev; else (&spp->spi_swapdev)->tqh_last = (sdp
)->swd_next.tqe_prev; *(sdp)->swd_next.tqe_prev = (sdp)
->swd_next.tqe_next; ((sdp)->swd_next.tqe_prev) = ((void
*)-1); ((sdp)->swd_next.tqe_next) = ((void *)-1); } while
(0);
1497 TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next)do { (sdp)->swd_next.tqe_next = ((void *)0); (sdp)->swd_next
.tqe_prev = (&spp->spi_swapdev)->tqh_last; *(&spp
->spi_swapdev)->tqh_last = (sdp); (&spp->spi_swapdev
)->tqh_last = &(sdp)->swd_next.tqe_next; } while (0
);
1498 sdp->swd_npginuse += *nslots;
1499 uvmexp.swpginuse += *nslots;
1500 mtx_leave(&uvm_swap_data_lock);
1501 /* done! return drum slot number */
1502 return result + sdp->swd_drumoffset;
1503 }
1504 }
1505
1506 /* XXXMRG: BEGIN HACK */
1507 if (*nslots > 1 && lessok) {
1508 *nslots = 1;
1509 /* XXXMRG: ugh! blist should support this for us */
1510 goto ReTry;
1511 }
1512 /* XXXMRG: END HACK */
1513
1514 mtx_leave(&uvm_swap_data_lock);
1515 return 0; /* failed */
1516}
1517
1518/*
1519 * uvm_swapisfilled: return true if the amount of free space in swap is
1520 * smaller than the size of a cluster.
1521 *
1522 * As long as some swap slots are being used by pages currently in memory,
1523 * it is possible to reuse them. Even if the swap space has been completly
1524 * filled we do not consider it full.
1525 */
1526int
1527uvm_swapisfilled(void)
1528{
1529 int result;
1530
1531 mtx_enter(&uvm_swap_data_lock);
1532 KASSERT(uvmexp.swpginuse <= uvmexp.swpages)((uvmexp.swpginuse <= uvmexp.swpages) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 1532, "uvmexp.swpginuse <= uvmexp.swpages"
));
1533 result = (uvmexp.swpginuse + SWCLUSTPAGES((64 * 1024) >> 12)) >= uvmexp.swpages;
1534 mtx_leave(&uvm_swap_data_lock);
1535
1536 return result;
1537}
1538
1539/*
1540 * uvm_swapisfull: return true if the amount of pages only in swap
1541 * accounts for more than 99% of the total swap space.
1542 *
1543 */
1544int
1545uvm_swapisfull(void)
1546{
1547 int result;
1548
1549 mtx_enter(&uvm_swap_data_lock);
1550 KASSERT(uvmexp.swpgonly <= uvmexp.swpages)((uvmexp.swpgonly <= uvmexp.swpages) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 1550, "uvmexp.swpgonly <= uvmexp.swpages"
));
1551 result = (uvmexp.swpgonly >= ((long)uvmexp.swpages * 99 / 100));
1552 mtx_leave(&uvm_swap_data_lock);
1553
1554 return result;
1555}
1556
1557/*
1558 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors
1559 *
1560 * => we lock uvm_swap_data_lock
1561 */
1562void
1563uvm_swap_markbad(int startslot, int nslots)
1564{
1565 struct swapdev *sdp;
1566
1567 mtx_enter(&uvm_swap_data_lock);
1568 sdp = swapdrum_getsdp(startslot);
1569 if (sdp != NULL((void *)0)) {
1570 /*
1571 * we just keep track of how many pages have been marked bad
1572 * in this device, to make everything add up in swap_off().
1573 * we assume here that the range of slots will all be within
1574 * one swap device.
1575 */
1576 sdp->swd_npgbad += nslots;
1577 }
1578 mtx_leave(&uvm_swap_data_lock);
1579}
1580
1581/*
1582 * uvm_swap_free: free swap slots
1583 *
1584 * => this can be all or part of an allocation made by uvm_swap_alloc
1585 * => we lock uvm_swap_data_lock
1586 */
1587void
1588uvm_swap_free(int startslot, int nslots)
1589{
1590 struct swapdev *sdp;
1591
1592 /*
1593 * ignore attempts to free the "bad" slot.
1594 */
1595
1596 if (startslot == SWSLOT_BAD(-1)) {
1597 return;
1598 }
1599
1600 /*
1601 * convert drum slot offset back to sdp, free the blocks
1602 * in the extent, and return. must hold pri lock to do
1603 * lookup and access the extent.
1604 */
1605 KERNEL_LOCK()_kernel_lock();
1606 mtx_enter(&uvm_swap_data_lock);
1607 sdp = swapdrum_getsdp(startslot);
1608 KASSERT(uvmexp.nswapdev >= 1)((uvmexp.nswapdev >= 1) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_swap.c", 1608, "uvmexp.nswapdev >= 1"
));
1609 KASSERT(sdp != NULL)((sdp != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c"
, 1609, "sdp != NULL"));
1610 KASSERT(sdp->swd_npginuse >= nslots)((sdp->swd_npginuse >= nslots) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_swap.c", 1610, "sdp->swd_npginuse >= nslots"
));
1611 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots);
1612 sdp->swd_npginuse -= nslots;
1613 uvmexp.swpginuse -= nslots;
1614 mtx_leave(&uvm_swap_data_lock);
1615
1616#ifdef UVM_SWAP_ENCRYPT1
1617 {
1618 int i;
1619 if (swap_encrypt_initialized) {
1620 /* Dereference keys */
1621 for (i = 0; i < nslots; i++)
1622 if (uvm_swap_needdecrypt(sdp, startslot + i)) {
1623 struct swap_key *key;
1624
1625 key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset
) >> 7]);
1626 if (key->refcount != 0)
1627 SWAP_KEY_PUT(sdp, key)do { (key)->refcount--; if ((key)->refcount == 0) { swap_key_delete
(key); } } while(0);;
1628 }
1629
1630 /* Mark range as not decrypt */
1631 uvm_swap_markdecrypt(sdp, startslot, nslots, 0);
1632 }
1633 }
1634#endif /* UVM_SWAP_ENCRYPT */
1635 KERNEL_UNLOCK()_kernel_unlock();
1636}
1637
1638/*
1639 * uvm_swap_put: put any number of pages into a contig place on swap
1640 *
1641 * => can be sync or async
1642 */
1643int
1644uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags)
1645{
1646 int result;
1647
1648 result = uvm_swap_io(ppsp, swslot, npages, B_WRITE0x00000000 |
1649 ((flags & PGO_SYNCIO0x002) ? 0 : B_ASYNC0x00000004));
1650
1651 return (result);
1652}
1653
1654/*
1655 * uvm_swap_get: get a single page from swap
1656 *
1657 * => usually a sync op (from fault)
1658 */
1659int
1660uvm_swap_get(struct vm_page *page, int swslot, int flags)
1661{
1662 int result;
1663
1664 atomic_inc_int(&uvmexp.nswget)_atomic_inc_int(&uvmexp.nswget);
1665 KASSERT(flags & PGO_SYNCIO)((flags & 0x002) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c"
, 1665, "flags & PGO_SYNCIO"));
1666 if (swslot == SWSLOT_BAD(-1)) {
1667 return VM_PAGER_ERROR4;
1668 }
1669
1670 KERNEL_LOCK()_kernel_lock();
1671 result = uvm_swap_io(&page, swslot, 1, B_READ0x00008000);
1672 KERNEL_UNLOCK()_kernel_unlock();
1673
1674 if (result == VM_PAGER_OK0 || result == VM_PAGER_PEND3) {
1675 /*
1676 * this page is no longer only in swap.
1677 */
1678 atomic_dec_int(&uvmexp.swpgonly)_atomic_dec_int(&uvmexp.swpgonly);
1679 }
1680 return (result);
1681}
1682
1683/*
1684 * uvm_swap_io: do an i/o operation to swap
1685 */
1686
1687int
1688uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags)
1689{
1690 daddr_t startblk;
1691 struct buf *bp;
1692 vaddr_t kva;
1693 int result, s, mapinflags, pflag, bounce = 0, i;
1694 boolean_t write, async;
1695 vaddr_t bouncekva;
1696 struct vm_page *tpps[SWCLUSTPAGES((64 * 1024) >> 12)];
1697 int pdaemon = (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc == uvm.pagedaemon_proc);
1698#ifdef UVM_SWAP_ENCRYPT1
1699 struct swapdev *sdp;
1
'sdp' declared without an initial value
→
1700 int encrypt = 0;
1701#endif
1702
1703 KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c"
, 1703, "_kernel_lock_held()"));
2
←
Assuming the condition is true
→
3
←
'?' condition is true
→
1704
1705 write = (flags & B_READ0x00008000) == 0;
4
←
Assuming the condition is false
→
1706 async = (flags & B_ASYNC0x00000004) != 0;
5
←
Assuming the condition is true
→
1707
1708 /* convert starting drum slot to block number */
1709 startblk = btodb((u_int64_t)startslot << PAGE_SHIFT)(((u_int64_t)startslot << 12) >> 9);
1710
1711 pflag = (async
5.1
'async' is 1
 || pdaemon) ? PR_NOWAIT0x0002 : PR_WAITOK0x0001;
1712 bp = pool_get(&bufpool, pflag | PR_ZERO0x0008);
1713 if (bp == NULL((void *)0))
6
←
Assuming 'bp' is not equal to NULL
→
1714 return (VM_PAGER_AGAIN5);
1715
1716 /*
1717 * map the pages into the kernel (XXX: currently required
1718 * by buffer system).
1719 */
1720 mapinflags = !write
7.1
'write' is 0
 ? UVMPAGER_MAPIN_READ0x02 : UVMPAGER_MAPIN_WRITE0x00;
7
←
Taking false branch
→
8
←
'?' condition is true
→
1721 if (!async
8.1
'async' is 1
)
9
←
Taking false branch
→
1722 mapinflags |= UVMPAGER_MAPIN_WAITOK0x01;
1723 kva = uvm_pagermapin(pps, npages, mapinflags);
1724 if (kva == 0) {
10
←
Assuming 'kva' is not equal to 0
→
11
←
Taking false branch
→
1725 pool_put(&bufpool, bp);
1726 return (VM_PAGER_AGAIN5);
1727 }
1728
1729#ifdef UVM_SWAP_ENCRYPT1
1730 if (write
11.1
'write' is 0
) {
1731 /*
1732 * Check if we need to do swap encryption on old pages.
1733 * Later we need a different scheme, that swap encrypts
1734 * all pages of a process that had at least one page swap
1735 * encrypted. Then we might not need to copy all pages
1736 * in the cluster, and avoid the memory overheard in
1737 * swapping.
1738 */
1739 if (uvm_doswapencrypt)
1740 encrypt = 1;
1741 }
1742
1743 if (swap_encrypt_initialized || encrypt
12.1
'encrypt' is 0
) {
12
←
Assuming 'swap_encrypt_initialized' is 0
→
1744 /*
1745 * we need to know the swap device that we are swapping to/from
1746 * to see if the pages need to be marked for decryption or
1747 * actually need to be decrypted.
1748 * XXX - does this information stay the same over the whole
1749 * execution of this function?
1750 */
1751 mtx_enter(&uvm_swap_data_lock);
1752 sdp = swapdrum_getsdp(startslot);
1753 mtx_leave(&uvm_swap_data_lock);
1754 }
1755
1756 /*
1757 * Check that we are dma capable for read (write always bounces
1758 * through the swapencrypt anyway...
1759 */
1760 if (write
12.2
'write' is 0
 && encrypt) {
1761 bounce = 1; /* bounce through swapencrypt always */
1762 } else {
1763#else
1764 {
1765#endif
1766
1767 for (i = 0; i < npages; i++) {
13
←
Assuming 'i' is < 'npages'
→
18
←
Assuming 'i' is >= 'npages'
→
19
←
Loop condition is false. Execution continues on line 1776
→
1768 if (VM_PAGE_TO_PHYS(pps[i])((pps[i])->phys_addr) < dma_constraint.ucr_low ||
14
←
Loop condition is true. Entering loop body
→
15
←
Assuming field 'phys_addr' is >= field 'ucr_low'
→
17
←
Taking false branch
→
1769 VM_PAGE_TO_PHYS(pps[i])((pps[i])->phys_addr) > dma_constraint.ucr_high) {
16
←
Assuming field 'phys_addr' is <= field 'ucr_high'
→
1770 bounce = 1;
1771 break;
1772 }
1773 }
1774 }
1775
1776 if (bounce
19.1
'bounce' is 0
) {
1777 int swmapflags, plaflags;
1778
1779 /* We always need write access. */
1780 swmapflags = UVMPAGER_MAPIN_READ0x02;
1781 plaflags = UVM_PLA_NOWAIT0x0002;
1782 if (!async) {
1783 swmapflags |= UVMPAGER_MAPIN_WAITOK0x01;
1784 plaflags = UVM_PLA_WAITOK0x0001;
1785 }
1786 if (uvm_swap_allocpages(tpps, npages, plaflags)) {
1787 pool_put(&bufpool, bp);
1788 uvm_pagermapout(kva, npages);
1789 return (VM_PAGER_AGAIN5);
1790 }
1791
1792 bouncekva = uvm_pagermapin(tpps, npages, swmapflags);
1793 if (bouncekva == 0) {
1794 pool_put(&bufpool, bp);
1795 uvm_pagermapout(kva, npages);
1796 uvm_swap_freepages(tpps, npages);
1797 return (VM_PAGER_AGAIN5);
1798 }
1799 }
1800
1801 /* encrypt to swap */
1802 if (write
19.2
'write' is 0
 && bounce) {
1803 int i, opages;
1804 caddr_t src, dst;
1805 u_int64_t block;
1806
1807 src = (caddr_t) kva;
1808 dst = (caddr_t) bouncekva;
1809 block = startblk;
1810 for (i = 0; i < npages; i++) {
1811#ifdef UVM_SWAP_ENCRYPT1
1812 struct swap_key *key;
1813
1814 if (encrypt) {
1815 key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset
) >> 7]);
1816 SWAP_KEY_GET(sdp, key)do { if ((key)->refcount == 0) { swap_key_create(key); } (
key)->refcount++; } while(0);; /* add reference */
1817
1818 swap_encrypt(key, src, dst, block, PAGE_SIZE(1 << 12));
1819 block += btodb(PAGE_SIZE)(((1 << 12)) >> 9);
1820 } else {
1821#else
1822 {
1823#endif /* UVM_SWAP_ENCRYPT */
1824 memcpy(dst, src, PAGE_SIZE)__builtin_memcpy((dst), (src), ((1 << 12)));
1825 }
1826 /* this just tells async callbacks to free */
1827 atomic_setbits_intx86_atomic_setbits_u32(&tpps[i]->pg_flags, PQ_ENCRYPT0x00400000);
1828 src += PAGE_SIZE(1 << 12);
1829 dst += PAGE_SIZE(1 << 12);
1830 }
1831
1832 uvm_pagermapout(kva, npages);
1833
1834 /* dispose of pages we dont use anymore */
1835 opages = npages;
1836 uvm_pager_dropcluster(NULL((void *)0), NULL((void *)0), pps, &opages,
1837 PGO_PDFREECLUST0x080);
1838
1839 kva = bouncekva;
1840 }
1841
1842 /*
1843 * prevent ASYNC reads.
1844 * uvm_swap_io is only called from uvm_swap_get, uvm_swap_get
1845 * assumes that all gets are SYNCIO. Just make sure here.
1846 * XXXARTUBC - might not be true anymore.
1847 */
1848 if (!write
19.3
'write' is 0
) {
20
←
Taking true branch
→
1849 flags &= ~B_ASYNC0x00000004;
1850 async = 0;
1851 }
1852
1853 /*
1854 * fill in the bp. we currently route our i/o through
1855 * /dev/drum's vnode [swapdev_vp].
1856 */
1857 bp->b_flags = B_BUSY0x00000010 | B_NOCACHE0x00001000 | B_RAW0x00004000 | (flags & (B_READ0x00008000|B_ASYNC0x00000004));
1858 bp->b_proc = &proc0; /* XXX */
1859 bp->b_vnbufs.le_next = NOLIST((struct buf *)0x87654321);
1860 if (bounce
20.1
'bounce' is 0
)
21
←
Taking false branch
→
1861 bp->b_data = (caddr_t)bouncekva;
1862 else
1863 bp->b_data = (caddr_t)kva;
1864 bp->b_bq = NULL((void *)0);
1865 bp->b_blkno = startblk;
1866 LIST_INIT(&bp->b_dep)do { ((&bp->b_dep)->lh_first) = ((void *)0); } while
(0);
22
←
Loop condition is false. Exiting loop
→
1867 s = splbio()splraise(0x3);
1868 bp->b_vp = NULL((void *)0);
1869 buf_replacevnode(bp, swapdev_vp);
1870 splx(s)spllower(s);
1871 bp->b_bufsize = bp->b_bcount = (long)npages << PAGE_SHIFT12;
1872
1873 /*
1874 * for pageouts we must set "dirtyoff" [NFS client code needs it].
1875 * and we bump v_numoutput (counter of number of active outputs).
1876 */
1877 if (write
22.1
'write' is 0
) {
23
←
Taking false branch
→
1878 bp->b_dirtyoff = 0;
1879 bp->b_dirtyend = npages << PAGE_SHIFT12;
1880#ifdef UVM_SWAP_ENCRYPT1
1881 /* mark the pages in the drum for decryption */
1882 if (swap_encrypt_initialized)
1883 uvm_swap_markdecrypt(sdp, startslot, npages, encrypt);
1884#endif
1885 s = splbio()splraise(0x3);
1886 swapdev_vp->v_numoutput++;
1887 splx(s)spllower(s);
1888 }
1889
1890 /* for async ops we must set up the iodone handler. */
1891 if (async
23.1
'async' is 0
) {
24
←
Taking false branch
→
1892 bp->b_flags |= B_CALL0x00000040 | (pdaemon ? B_PDAEMON0x00200000 : 0);
1893 bp->b_iodone = uvm_aio_biodone;
1894 }
1895
1896 /* now we start the I/O, and if async, return. */
1897 VOP_STRATEGY(bp->b_vp, bp);
1898 if (async
24.1
'async' is 0
)
25
←
Taking false branch
→
1899 return (VM_PAGER_PEND3);
1900
1901 /* must be sync i/o. wait for it to finish */
1902 (void) biowait(bp);
1903 result = (bp->b_flags & B_ERROR0x00000400) ? VM_PAGER_ERROR4 : VM_PAGER_OK0;
26
←
Assuming the condition is false
→
27
←
'?' condition is false
→
1904
1905 /* decrypt swap */
1906 if (!write
27.1
'write' is 0
 && !(bp->b_flags & B_ERROR0x00000400)) {
28
←
Taking true branch
→
1907 int i;
1908 caddr_t data = (caddr_t)kva;
1909 caddr_t dst = (caddr_t)kva;
1910 u_int64_t block = startblk;
1911
1912 if (bounce
28.1
'bounce' is 0
)
29
←
Taking false branch
→
1913 data = (caddr_t)bouncekva;
1914
1915 for (i = 0; i < npages; i++) {
30
←
Loop condition is true. Entering loop body
→
1916#ifdef UVM_SWAP_ENCRYPT1
1917 struct swap_key *key;
1918
1919 /* Check if we need to decrypt */
1920 if (swap_encrypt_initialized &&
31
←
Assuming 'swap_encrypt_initialized' is not equal to 0
→
1921 uvm_swap_needdecrypt(sdp, startslot + i)) {
32
←
1st function call argument is an uninitialized value
1922 key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset
) >> 7]);
1923 if (key->refcount == 0) {
1924 result = VM_PAGER_ERROR4;
1925 break;
1926 }
1927 swap_decrypt(key, data, dst, block, PAGE_SIZE(1 << 12));
1928 } else if (bounce) {
1929#else
1930 if (bounce) {
1931#endif
1932 memcpy(dst, data, PAGE_SIZE)__builtin_memcpy((dst), (data), ((1 << 12)));
1933 }
1934 data += PAGE_SIZE(1 << 12);
1935 dst += PAGE_SIZE(1 << 12);
1936 block += btodb(PAGE_SIZE)(((1 << 12)) >> 9);
1937 }
1938 if (bounce)
1939 uvm_pagermapout(bouncekva, npages);
1940 }
1941 /* kill the pager mapping */
1942 uvm_pagermapout(kva, npages);
1943
1944 /* Not anymore needed, free after encryption/bouncing */
1945 if (!write && bounce)
1946 uvm_swap_freepages(tpps, npages);
1947
1948 /* now dispose of the buf */
1949 s = splbio()splraise(0x3);
1950 if (bp->b_vp)
1951 brelvp(bp);
1952
1953 if (write && bp->b_vp)
1954 vwakeup(bp->b_vp);
1955 pool_put(&bufpool, bp);
1956 splx(s)spllower(s);
1957
1958 /* finally return. */
1959 return (result);
1960}
1961
1962void
1963swapmount(void)
1964{
1965 struct swapdev *sdp;
1966 struct swappri *spp;
1967 struct vnode *vp;
1968 dev_t swap_dev = swdevt[0].sw_dev;
1969 char *nam;
1970 char path[MNAMELEN90 + 1];
1971
1972 if (swap_dev == NODEV(dev_t)(-1))
1973 return;
1974
1975 rw_enter_write(&swap_syscall_lock);
1976
1977#if defined(NFSCLIENT1)
1978 if (swap_dev == NETDEV(dev_t)(-2)) {
1979 extern struct nfs_diskless nfs_diskless;
1980
1981 snprintf(path, sizeof(path), "%s",
1982 nfs_diskless.nd_swap.ndm_host);
1983 vp = nfs_diskless.sw_vp;
1984 goto gotit;
1985 } else
1986#endif
1987 if (bdevvp(swap_dev, &vp)) {
1988 rw_exit_write(&swap_syscall_lock);
1989 return;
1990 }
1991
1992 /* Construct a potential path to swap */
1993 if ((nam = findblkname(major(swap_dev)(((unsigned)(swap_dev) >> 8) & 0xff))))
1994 snprintf(path, sizeof(path), "/dev/%s%d%c", nam,
1995 DISKUNIT(swap_dev)(((unsigned)((swap_dev) & 0xff) | (((swap_dev) & 0xffff0000
) >> 8)) / 16), 'a' + DISKPART(swap_dev)(((unsigned)((swap_dev) & 0xff) | (((swap_dev) & 0xffff0000
) >> 8)) % 16));
1996 else
1997 snprintf(path, sizeof(path), "blkdev0x%x",
1998 swap_dev);
1999
2000#if defined(NFSCLIENT1)
2001gotit:
2002#endif
2003 sdp = malloc(sizeof(*sdp), M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008);
2004 spp = malloc(sizeof(*spp), M_VMSWAP92, M_WAITOK0x0001);
2005
2006 sdp->swd_flagsswd_se.se_flags = SWF_FAKE0x00000008;
2007 sdp->swd_devswd_se.se_dev = swap_dev;
2008
2009 sdp->swd_pathlen = strlen(path) + 1;
2010 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP92, M_WAITOK0x0001 | M_ZERO0x0008);
2011 strlcpy(sdp->swd_path, path, sdp->swd_pathlen);
2012
2013 sdp->swd_vp = vp;
2014
2015 mtx_enter(&uvm_swap_data_lock);
2016 swaplist_insert(sdp, spp, 0);
2017 mtx_leave(&uvm_swap_data_lock);
2018
2019 if (swap_on(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc, sdp)) {
2020 mtx_enter(&uvm_swap_data_lock);
2021 swaplist_find(vp, 1);
2022 swaplist_trim();
2023 vput(sdp->swd_vp);
2024 mtx_leave(&uvm_swap_data_lock);
2025 rw_exit_write(&swap_syscall_lock);
2026 free(sdp->swd_path, M_VMSWAP92, sdp->swd_pathlen);
2027 free(sdp, M_VMSWAP92, sizeof(*sdp));
2028 return;
2029 }
2030 rw_exit_write(&swap_syscall_lock);
2031}
2032
2033#ifdef HIBERNATE1
2034int
2035uvm_hibswap(dev_t dev, u_long *sp, u_long *ep)
2036{
2037 struct swapdev *sdp, *swd = NULL((void *)0);
2038 struct swappri *spp;
2039
2040 /* no swap devices configured yet? */
2041 if (uvmexp.nswapdev < 1 || dev != swdevt[0].sw_dev)
2042 return (1);
2043
2044 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
2045 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
2046 if (sdp->swd_devswd_se.se_dev == dev)
2047 swd = sdp;
2048 }
2049 }
2050
2051 if (swd == NULL((void *)0) || (swd->swd_flagsswd_se.se_flags & SWF_ENABLE0x00000002) == 0)
2052 return (1);
2053
2054 blist_gapfind(swd->swd_blist, sp, ep);
2055
2056 if (*ep - *sp == 0)
2057 /* no gap found */
2058 return (1);
2059
2060 /*
2061 * blist_gapfind returns the gap as [sp,ep[ ,
2062 * whereas [sp,ep] is expected from uvm_hibswap().
2063 */
2064 *ep -= 1;
2065
2066 return (0);
2067}
2068#endif /* HIBERNATE */
2069
2070#ifdef DDB1
2071void
2072swap_print_all(int (*pr)(const char *, ...))
2073{
2074 struct swappri *spp;
2075 struct swapdev *sdp;
2076
2077 LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void
*)0); (spp) = ((spp)->spi_swappri.le_next)) {
2078 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp)
!= ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) {
2079#ifdef HIBERNATE1
2080 u_long bgap = 0, egap = 0;
2081#endif
2082
2083 pr("swap %p path \"%s\" flags 0x%x\n", sdp,
2084 sdp->swd_path, sdp->swd_flagsswd_se.se_flags);
2085
2086 blist_print(sdp->swd_blist);
2087
2088#ifdef HIBERNATE1
2089 if (!uvm_hibswap(sdp->swd_devswd_se.se_dev, &bgap, &egap))
2090 pr("hibernate gap: [0x%lx, 0x%lx] size=%lu\n",
2091 bgap, egap, (egap - bgap + 1));
2092 else
2093 pr("hibernate gap: not found\n");
2094#endif
2095 }
2096 }
2097}
2098#endif /* DDB */