File: | uvm/uvm_swap.c |
Warning: | line 1844, column 8 1st function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* $OpenBSD: uvm_swap.c,v 1.152 2021/12/12 09:14:59 visa Exp $ */ | |||
2 | /* $NetBSD: uvm_swap.c,v 1.40 2000/11/17 11:39:39 mrg Exp $ */ | |||
3 | ||||
4 | /* | |||
5 | * Copyright (c) 1995, 1996, 1997 Matthew R. Green | |||
6 | * All rights reserved. | |||
7 | * | |||
8 | * Redistribution and use in source and binary forms, with or without | |||
9 | * modification, are permitted provided that the following conditions | |||
10 | * are met: | |||
11 | * 1. Redistributions of source code must retain the above copyright | |||
12 | * notice, this list of conditions and the following disclaimer. | |||
13 | * 2. Redistributions in binary form must reproduce the above copyright | |||
14 | * notice, this list of conditions and the following disclaimer in the | |||
15 | * documentation and/or other materials provided with the distribution. | |||
16 | * | |||
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | |||
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |||
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | |||
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, | |||
22 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |||
23 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED | |||
24 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||
25 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |||
26 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |||
27 | * SUCH DAMAGE. | |||
28 | * | |||
29 | * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp | |||
30 | * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp | |||
31 | */ | |||
32 | ||||
33 | #include <sys/param.h> | |||
34 | #include <sys/systm.h> | |||
35 | #include <sys/buf.h> | |||
36 | #include <sys/conf.h> | |||
37 | #include <sys/proc.h> | |||
38 | #include <sys/namei.h> | |||
39 | #include <sys/disklabel.h> | |||
40 | #include <sys/errno.h> | |||
41 | #include <sys/kernel.h> | |||
42 | #include <sys/malloc.h> | |||
43 | #include <sys/vnode.h> | |||
44 | #include <sys/fcntl.h> | |||
45 | #include <sys/extent.h> | |||
46 | #include <sys/mount.h> | |||
47 | #include <sys/pool.h> | |||
48 | #include <sys/syscallargs.h> | |||
49 | #include <sys/swap.h> | |||
50 | #include <sys/disk.h> | |||
51 | #include <sys/task.h> | |||
52 | #include <sys/pledge.h> | |||
53 | #if defined(NFSCLIENT1) | |||
54 | #include <sys/socket.h> | |||
55 | #include <sys/domain.h> | |||
56 | #include <netinet/in.h> | |||
57 | #include <nfs/nfsproto.h> | |||
58 | #include <nfs/nfsdiskless.h> | |||
59 | #endif | |||
60 | ||||
61 | #include <uvm/uvm.h> | |||
62 | #ifdef UVM_SWAP_ENCRYPT1 | |||
63 | #include <uvm/uvm_swap_encrypt.h> | |||
64 | #endif | |||
65 | ||||
66 | #include <sys/specdev.h> | |||
67 | ||||
68 | #include "vnd.h" | |||
69 | ||||
70 | /* | |||
71 | * uvm_swap.c: manage configuration and i/o to swap space. | |||
72 | */ | |||
73 | ||||
74 | /* | |||
75 | * swap space is managed in the following way: | |||
76 | * | |||
77 | * each swap partition or file is described by a "swapdev" structure. | |||
78 | * each "swapdev" structure contains a "swapent" structure which contains | |||
79 | * information that is passed up to the user (via system calls). | |||
80 | * | |||
81 | * each swap partition is assigned a "priority" (int) which controls | |||
82 | * swap partition usage. | |||
83 | * | |||
84 | * the system maintains a global data structure describing all swap | |||
85 | * partitions/files. there is a sorted LIST of "swappri" structures | |||
86 | * which describe "swapdev"'s at that priority. this LIST is headed | |||
87 | * by the "swap_priority" global var. each "swappri" contains a | |||
88 | * TAILQ of "swapdev" structures at that priority. | |||
89 | * | |||
90 | * locking: | |||
91 | * - swap_syscall_lock (sleep lock): this lock serializes the swapctl | |||
92 | * system call and prevents the swap priority list from changing | |||
93 | * while we are in the middle of a system call (e.g. SWAP_STATS). | |||
94 | * | |||
95 | * each swap device has the following info: | |||
96 | * - swap device in use (could be disabled, preventing future use) | |||
97 | * - swap enabled (allows new allocations on swap) | |||
98 | * - map info in /dev/drum | |||
99 | * - vnode pointer | |||
100 | * for swap files only: | |||
101 | * - block size | |||
102 | * - max byte count in buffer | |||
103 | * - buffer | |||
104 | * - credentials to use when doing i/o to file | |||
105 | * | |||
106 | * userland controls and configures swap with the swapctl(2) system call. | |||
107 | * the sys_swapctl performs the following operations: | |||
108 | * [1] SWAP_NSWAP: returns the number of swap devices currently configured | |||
109 | * [2] SWAP_STATS: given a pointer to an array of swapent structures | |||
110 | * (passed in via "arg") of a size passed in via "misc" ... we load | |||
111 | * the current swap config into the array. | |||
112 | * [3] SWAP_ON: given a pathname in arg (could be device or file) and a | |||
113 | * priority in "misc", start swapping on it. | |||
114 | * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device | |||
115 | * [5] SWAP_CTL: changes the priority of a swap device (new priority in | |||
116 | * "misc") | |||
117 | */ | |||
118 | ||||
119 | /* | |||
120 | * swapdev: describes a single swap partition/file | |||
121 | * | |||
122 | * note the following should be true: | |||
123 | * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] | |||
124 | * swd_nblks <= swd_mapsize [because mapsize includes disklabel] | |||
125 | */ | |||
126 | struct swapdev { | |||
127 | struct swapent swd_se; | |||
128 | #define swd_devswd_se.se_dev swd_se.se_dev /* device id */ | |||
129 | #define swd_flagsswd_se.se_flags swd_se.se_flags /* flags:inuse/enable/fake */ | |||
130 | #define swd_priorityswd_se.se_priority swd_se.se_priority /* our priority */ | |||
131 | #define swd_inuseswd_se.se_inuse swd_se.se_inuse /* blocks used */ | |||
132 | #define swd_nblksswd_se.se_nblks swd_se.se_nblks /* total blocks */ | |||
133 | char *swd_path; /* saved pathname of device */ | |||
134 | int swd_pathlen; /* length of pathname */ | |||
135 | int swd_npages; /* #pages we can use */ | |||
136 | int swd_npginuse; /* #pages in use */ | |||
137 | int swd_npgbad; /* #pages bad */ | |||
138 | int swd_drumoffset; /* page0 offset in drum */ | |||
139 | int swd_drumsize; /* #pages in drum */ | |||
140 | struct extent *swd_ex; /* extent for this swapdev */ | |||
141 | char swd_exname[12]; /* name of extent above */ | |||
142 | struct vnode *swd_vp; /* backing vnode */ | |||
143 | TAILQ_ENTRY(swapdev)struct { struct swapdev *tqe_next; struct swapdev **tqe_prev; } swd_next; /* priority tailq */ | |||
144 | ||||
145 | int swd_bsize; /* blocksize (bytes) */ | |||
146 | int swd_maxactive; /* max active i/o reqs */ | |||
147 | int swd_active; /* # of active i/o reqs */ | |||
148 | struct bufq swd_bufq; | |||
149 | struct ucred *swd_cred; /* cred for file access */ | |||
150 | #ifdef UVM_SWAP_ENCRYPT1 | |||
151 | #define SWD_KEY_SHIFT7 7 /* One key per 0.5 MByte */ | |||
152 | #define SWD_KEY(x,y)&((x)->swd_keys[((y) - (x)->swd_drumoffset) >> 7]) &((x)->swd_keys[((y) - (x)->swd_drumoffset) >> SWD_KEY_SHIFT7]) | |||
153 | #define SWD_KEY_SIZE(x)(((x) + (1 << 7) - 1) >> 7) (((x) + (1 << SWD_KEY_SHIFT7) - 1) >> SWD_KEY_SHIFT7) | |||
154 | ||||
155 | #define SWD_DCRYPT_SHIFT5 5 | |||
156 | #define SWD_DCRYPT_BITS32 32 | |||
157 | #define SWD_DCRYPT_MASK(32 - 1) (SWD_DCRYPT_BITS32 - 1) | |||
158 | #define SWD_DCRYPT_OFF(x)((x) >> 5) ((x) >> SWD_DCRYPT_SHIFT5) | |||
159 | #define SWD_DCRYPT_BIT(x)((x) & (32 - 1)) ((x) & SWD_DCRYPT_MASK(32 - 1)) | |||
160 | #define SWD_DCRYPT_SIZE(x)((((x) + (32 - 1)) >> 5) * sizeof(u_int32_t)) (SWD_DCRYPT_OFF((x) + SWD_DCRYPT_MASK)(((x) + (32 - 1)) >> 5) * sizeof(u_int32_t)) | |||
161 | u_int32_t *swd_decrypt; /* bitmap for decryption */ | |||
162 | struct swap_key *swd_keys; /* keys for different parts */ | |||
163 | #endif | |||
164 | }; | |||
165 | ||||
166 | /* | |||
167 | * swap device priority entry; the list is kept sorted on `spi_priority'. | |||
168 | */ | |||
169 | struct swappri { | |||
170 | int spi_priority; /* priority */ | |||
171 | TAILQ_HEAD(spi_swapdev, swapdev)struct spi_swapdev { struct swapdev *tqh_first; struct swapdev **tqh_last; } spi_swapdev; | |||
172 | /* tailq of swapdevs at this priority */ | |||
173 | LIST_ENTRY(swappri)struct { struct swappri *le_next; struct swappri **le_prev; } spi_swappri; /* global list of pri's */ | |||
174 | }; | |||
175 | ||||
176 | /* | |||
177 | * The following two structures are used to keep track of data transfers | |||
178 | * on swap devices associated with regular files. | |||
179 | * NOTE: this code is more or less a copy of vnd.c; we use the same | |||
180 | * structure names here to ease porting.. | |||
181 | */ | |||
182 | struct vndxfer { | |||
183 | struct buf *vx_bp; /* Pointer to parent buffer */ | |||
184 | struct swapdev *vx_sdp; | |||
185 | int vx_error; | |||
186 | int vx_pending; /* # of pending aux buffers */ | |||
187 | int vx_flags; | |||
188 | #define VX_BUSY1 1 | |||
189 | #define VX_DEAD2 2 | |||
190 | }; | |||
191 | ||||
192 | struct vndbuf { | |||
193 | struct buf vb_buf; | |||
194 | struct vndxfer *vb_vnx; | |||
195 | struct task vb_task; | |||
196 | }; | |||
197 | ||||
198 | /* | |||
199 | * We keep a of pool vndbuf's and vndxfer structures. | |||
200 | */ | |||
201 | struct pool vndxfer_pool; | |||
202 | struct pool vndbuf_pool; | |||
203 | ||||
204 | ||||
205 | /* | |||
206 | * local variables | |||
207 | */ | |||
208 | struct extent *swapmap; /* controls the mapping of /dev/drum */ | |||
209 | ||||
210 | /* list of all active swap devices [by priority] */ | |||
211 | LIST_HEAD(swap_priority, swappri)struct swap_priority { struct swappri *lh_first; }; | |||
212 | struct swap_priority swap_priority; | |||
213 | ||||
214 | /* locks */ | |||
215 | struct rwlock swap_syscall_lock = RWLOCK_INITIALIZER("swplk"){ 0, "swplk" }; | |||
216 | ||||
217 | /* | |||
218 | * prototypes | |||
219 | */ | |||
220 | void swapdrum_add(struct swapdev *, int); | |||
221 | struct swapdev *swapdrum_getsdp(int); | |||
222 | ||||
223 | struct swapdev *swaplist_find(struct vnode *, int); | |||
224 | void swaplist_insert(struct swapdev *, | |||
225 | struct swappri *, int); | |||
226 | void swaplist_trim(void); | |||
227 | ||||
228 | int swap_on(struct proc *, struct swapdev *); | |||
229 | int swap_off(struct proc *, struct swapdev *); | |||
230 | ||||
231 | void sw_reg_strategy(struct swapdev *, struct buf *, int); | |||
232 | void sw_reg_iodone(struct buf *); | |||
233 | void sw_reg_iodone_internal(void *); | |||
234 | void sw_reg_start(struct swapdev *); | |||
235 | ||||
236 | int uvm_swap_io(struct vm_page **, int, int, int); | |||
237 | ||||
238 | void swapmount(void); | |||
239 | boolean_t uvm_swap_allocpages(struct vm_page **, int); | |||
240 | ||||
241 | #ifdef UVM_SWAP_ENCRYPT1 | |||
242 | /* for swap encrypt */ | |||
243 | void uvm_swap_markdecrypt(struct swapdev *, int, int, int); | |||
244 | boolean_t uvm_swap_needdecrypt(struct swapdev *, int); | |||
245 | void uvm_swap_initcrypt(struct swapdev *, int); | |||
246 | #endif | |||
247 | ||||
248 | /* | |||
249 | * uvm_swap_init: init the swap system data structures and locks | |||
250 | * | |||
251 | * => called at boot time from init_main.c after the filesystems | |||
252 | * are brought up (which happens after uvm_init()) | |||
253 | */ | |||
254 | void | |||
255 | uvm_swap_init(void) | |||
256 | { | |||
257 | /* | |||
258 | * first, init the swap list, its counter, and its lock. | |||
259 | * then get a handle on the vnode for /dev/drum by using | |||
260 | * the its dev_t number ("swapdev", from MD conf.c). | |||
261 | */ | |||
262 | LIST_INIT(&swap_priority)do { ((&swap_priority)->lh_first) = ((void *)0); } while (0); | |||
263 | uvmexp.nswapdev = 0; | |||
264 | ||||
265 | if (!swapdev_vp && bdevvp(swapdev, &swapdev_vp)) | |||
266 | panic("uvm_swap_init: can't get vnode for swap device"); | |||
267 | ||||
268 | /* | |||
269 | * create swap block extent to map /dev/drum. The extent spans | |||
270 | * 1 to INT_MAX allows 2 gigablocks of swap space. Note that | |||
271 | * block 0 is reserved (used to indicate an allocation failure, | |||
272 | * or no allocation). | |||
273 | */ | |||
274 | swapmap = extent_create("swapmap", 1, INT_MAX0x7fffffff, | |||
275 | M_VMSWAP92, 0, 0, EX_NOWAIT0x0000); | |||
276 | if (swapmap == 0) | |||
277 | panic("uvm_swap_init: extent_create failed"); | |||
278 | ||||
279 | /* allocate pools for structures used for swapping to files. */ | |||
280 | pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, IPL_BIO0x6, 0, | |||
281 | "swp vnx", NULL((void *)0)); | |||
282 | pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, IPL_BIO0x6, 0, | |||
283 | "swp vnd", NULL((void *)0)); | |||
284 | ||||
285 | /* Setup the initial swap partition */ | |||
286 | swapmount(); | |||
287 | } | |||
288 | ||||
289 | #ifdef UVM_SWAP_ENCRYPT1 | |||
290 | void | |||
291 | uvm_swap_initcrypt_all(void) | |||
292 | { | |||
293 | struct swapdev *sdp; | |||
294 | struct swappri *spp; | |||
295 | int npages; | |||
296 | ||||
297 | ||||
298 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
299 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
300 | if (sdp->swd_decrypt == NULL((void *)0)) { | |||
301 | npages = dbtob((uint64_t)sdp->swd_nblks)(((uint64_t)sdp->swd_se.se_nblks) << 9) >> | |||
302 | PAGE_SHIFT12; | |||
303 | uvm_swap_initcrypt(sdp, npages); | |||
304 | } | |||
305 | } | |||
306 | } | |||
307 | } | |||
308 | ||||
309 | void | |||
310 | uvm_swap_initcrypt(struct swapdev *sdp, int npages) | |||
311 | { | |||
312 | /* | |||
313 | * keep information if a page needs to be decrypted when we get it | |||
314 | * from the swap device. | |||
315 | * We cannot chance a malloc later, if we are doing ASYNC puts, | |||
316 | * we may not call malloc with M_WAITOK. This consumes only | |||
317 | * 8KB memory for a 256MB swap partition. | |||
318 | */ | |||
319 | sdp->swd_decrypt = malloc(SWD_DCRYPT_SIZE(npages)((((npages) + (32 - 1)) >> 5) * sizeof(u_int32_t)), M_VMSWAP92, | |||
320 | M_WAITOK0x0001|M_ZERO0x0008); | |||
321 | sdp->swd_keys = mallocarray(SWD_KEY_SIZE(npages)(((npages) + (1 << 7) - 1) >> 7), | |||
322 | sizeof(struct swap_key), M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008); | |||
323 | } | |||
324 | ||||
325 | #endif /* UVM_SWAP_ENCRYPT */ | |||
326 | ||||
327 | boolean_t | |||
328 | uvm_swap_allocpages(struct vm_page **pps, int npages) | |||
329 | { | |||
330 | struct pglist pgl; | |||
331 | int i; | |||
332 | boolean_t fail; | |||
333 | ||||
334 | /* Estimate if we will succeed */ | |||
335 | uvm_lock_fpageq()mtx_enter(&uvm.fpageqlock); | |||
336 | ||||
337 | fail = uvmexp.free - npages < uvmexp.reserve_kernel; | |||
338 | ||||
339 | uvm_unlock_fpageq()mtx_leave(&uvm.fpageqlock); | |||
340 | ||||
341 | if (fail) | |||
342 | return FALSE0; | |||
343 | ||||
344 | TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last = &(&pgl)->tqh_first; } while (0); | |||
345 | if (uvm_pglistalloc(npages * PAGE_SIZE(1 << 12), dma_constraint.ucr_low, | |||
346 | dma_constraint.ucr_high, 0, 0, &pgl, npages, UVM_PLA_NOWAIT0x0002)) | |||
347 | return FALSE0; | |||
348 | ||||
349 | for (i = 0; i < npages; i++) { | |||
350 | pps[i] = TAILQ_FIRST(&pgl)((&pgl)->tqh_first); | |||
351 | /* *sigh* */ | |||
352 | atomic_setbits_intx86_atomic_setbits_u32(&pps[i]->pg_flags, PG_BUSY0x00000001); | |||
353 | TAILQ_REMOVE(&pgl, pps[i], pageq)do { if (((pps[i])->pageq.tqe_next) != ((void *)0)) (pps[i ])->pageq.tqe_next->pageq.tqe_prev = (pps[i])->pageq .tqe_prev; else (&pgl)->tqh_last = (pps[i])->pageq. tqe_prev; *(pps[i])->pageq.tqe_prev = (pps[i])->pageq.tqe_next ; ((pps[i])->pageq.tqe_prev) = ((void *)-1); ((pps[i])-> pageq.tqe_next) = ((void *)-1); } while (0); | |||
354 | } | |||
355 | ||||
356 | return TRUE1; | |||
357 | } | |||
358 | ||||
359 | void | |||
360 | uvm_swap_freepages(struct vm_page **pps, int npages) | |||
361 | { | |||
362 | int i; | |||
363 | ||||
364 | uvm_lock_pageq()mtx_enter(&uvm.pageqlock); | |||
365 | for (i = 0; i < npages; i++) | |||
366 | uvm_pagefree(pps[i]); | |||
367 | uvm_unlock_pageq()mtx_leave(&uvm.pageqlock); | |||
368 | } | |||
369 | ||||
370 | #ifdef UVM_SWAP_ENCRYPT1 | |||
371 | /* | |||
372 | * Mark pages on the swap device for later decryption | |||
373 | */ | |||
374 | ||||
375 | void | |||
376 | uvm_swap_markdecrypt(struct swapdev *sdp, int startslot, int npages, | |||
377 | int decrypt) | |||
378 | { | |||
379 | int pagestart, i; | |||
380 | int off, bit; | |||
381 | ||||
382 | if (!sdp) | |||
383 | return; | |||
384 | ||||
385 | pagestart = startslot - sdp->swd_drumoffset; | |||
386 | for (i = 0; i < npages; i++, pagestart++) { | |||
387 | off = SWD_DCRYPT_OFF(pagestart)((pagestart) >> 5); | |||
388 | bit = SWD_DCRYPT_BIT(pagestart)((pagestart) & (32 - 1)); | |||
389 | if (decrypt) | |||
390 | /* pages read need decryption */ | |||
391 | sdp->swd_decrypt[off] |= 1 << bit; | |||
392 | else | |||
393 | /* pages read do not need decryption */ | |||
394 | sdp->swd_decrypt[off] &= ~(1 << bit); | |||
395 | } | |||
396 | } | |||
397 | ||||
398 | /* | |||
399 | * Check if the page that we got from disk needs to be decrypted | |||
400 | */ | |||
401 | ||||
402 | boolean_t | |||
403 | uvm_swap_needdecrypt(struct swapdev *sdp, int off) | |||
404 | { | |||
405 | if (!sdp) | |||
406 | return FALSE0; | |||
407 | ||||
408 | off -= sdp->swd_drumoffset; | |||
409 | return sdp->swd_decrypt[SWD_DCRYPT_OFF(off)((off) >> 5)] & (1 << SWD_DCRYPT_BIT(off)((off) & (32 - 1))) ? | |||
410 | TRUE1 : FALSE0; | |||
411 | } | |||
412 | ||||
413 | void | |||
414 | uvm_swap_finicrypt_all(void) | |||
415 | { | |||
416 | struct swapdev *sdp; | |||
417 | struct swappri *spp; | |||
418 | struct swap_key *key; | |||
419 | unsigned int nkeys; | |||
420 | ||||
421 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
422 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
423 | if (sdp->swd_decrypt == NULL((void *)0)) | |||
424 | continue; | |||
425 | ||||
426 | nkeys = dbtob((uint64_t)sdp->swd_nblks)(((uint64_t)sdp->swd_se.se_nblks) << 9) >> PAGE_SHIFT12; | |||
427 | key = sdp->swd_keys + (SWD_KEY_SIZE(nkeys)(((nkeys) + (1 << 7) - 1) >> 7) - 1); | |||
428 | do { | |||
429 | if (key->refcount != 0) | |||
430 | swap_key_delete(key); | |||
431 | } while (key-- != sdp->swd_keys); | |||
432 | } | |||
433 | } | |||
434 | } | |||
435 | #endif /* UVM_SWAP_ENCRYPT */ | |||
436 | ||||
437 | /* | |||
438 | * swaplist functions: functions that operate on the list of swap | |||
439 | * devices on the system. | |||
440 | */ | |||
441 | ||||
442 | /* | |||
443 | * swaplist_insert: insert swap device "sdp" into the global list | |||
444 | * | |||
445 | * => caller must hold both swap_syscall_lock and uvm.swap_data_lock | |||
446 | * => caller must provide a newly malloc'd swappri structure (we will | |||
447 | * FREE it if we don't need it... this it to prevent malloc blocking | |||
448 | * here while adding swap) | |||
449 | */ | |||
450 | void | |||
451 | swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) | |||
452 | { | |||
453 | struct swappri *spp, *pspp; | |||
454 | ||||
455 | /* | |||
456 | * find entry at or after which to insert the new device. | |||
457 | */ | |||
458 | pspp = NULL((void *)0); | |||
459 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
460 | if (priority <= spp->spi_priority) | |||
461 | break; | |||
462 | pspp = spp; | |||
463 | } | |||
464 | ||||
465 | /* | |||
466 | * new priority? | |||
467 | */ | |||
468 | if (spp == NULL((void *)0) || spp->spi_priority != priority) { | |||
469 | spp = newspp; /* use newspp! */ | |||
470 | ||||
471 | spp->spi_priority = priority; | |||
472 | TAILQ_INIT(&spp->spi_swapdev)do { (&spp->spi_swapdev)->tqh_first = ((void *)0); ( &spp->spi_swapdev)->tqh_last = &(&spp->spi_swapdev )->tqh_first; } while (0); | |||
473 | ||||
474 | if (pspp) | |||
475 | LIST_INSERT_AFTER(pspp, spp, spi_swappri)do { if (((spp)->spi_swappri.le_next = (pspp)->spi_swappri .le_next) != ((void *)0)) (pspp)->spi_swappri.le_next-> spi_swappri.le_prev = &(spp)->spi_swappri.le_next; (pspp )->spi_swappri.le_next = (spp); (spp)->spi_swappri.le_prev = &(pspp)->spi_swappri.le_next; } while (0); | |||
476 | else | |||
477 | LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri)do { if (((spp)->spi_swappri.le_next = (&swap_priority )->lh_first) != ((void *)0)) (&swap_priority)->lh_first ->spi_swappri.le_prev = &(spp)->spi_swappri.le_next ; (&swap_priority)->lh_first = (spp); (spp)->spi_swappri .le_prev = &(&swap_priority)->lh_first; } while (0 ); | |||
478 | } else { | |||
479 | /* we don't need a new priority structure, free it */ | |||
480 | free(newspp, M_VMSWAP92, sizeof(*newspp)); | |||
481 | } | |||
482 | ||||
483 | /* | |||
484 | * priority found (or created). now insert on the priority's | |||
485 | * tailq list and bump the total number of swapdevs. | |||
486 | */ | |||
487 | sdp->swd_priorityswd_se.se_priority = priority; | |||
488 | TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next)do { (sdp)->swd_next.tqe_next = ((void *)0); (sdp)->swd_next .tqe_prev = (&spp->spi_swapdev)->tqh_last; *(&spp ->spi_swapdev)->tqh_last = (sdp); (&spp->spi_swapdev )->tqh_last = &(sdp)->swd_next.tqe_next; } while (0 ); | |||
489 | uvmexp.nswapdev++; | |||
490 | } | |||
491 | ||||
492 | /* | |||
493 | * swaplist_find: find and optionally remove a swap device from the | |||
494 | * global list. | |||
495 | * | |||
496 | * => caller must hold both swap_syscall_lock and uvm.swap_data_lock | |||
497 | * => we return the swapdev we found (and removed) | |||
498 | */ | |||
499 | struct swapdev * | |||
500 | swaplist_find(struct vnode *vp, boolean_t remove) | |||
501 | { | |||
502 | struct swapdev *sdp; | |||
503 | struct swappri *spp; | |||
504 | ||||
505 | /* | |||
506 | * search the lists for the requested vp | |||
507 | */ | |||
508 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
509 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
510 | if (sdp->swd_vp != vp) | |||
511 | continue; | |||
512 | if (remove) { | |||
513 | TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next)do { if (((sdp)->swd_next.tqe_next) != ((void *)0)) (sdp)-> swd_next.tqe_next->swd_next.tqe_prev = (sdp)->swd_next. tqe_prev; else (&spp->spi_swapdev)->tqh_last = (sdp )->swd_next.tqe_prev; *(sdp)->swd_next.tqe_prev = (sdp) ->swd_next.tqe_next; ((sdp)->swd_next.tqe_prev) = ((void *)-1); ((sdp)->swd_next.tqe_next) = ((void *)-1); } while (0); | |||
514 | uvmexp.nswapdev--; | |||
515 | } | |||
516 | return (sdp); | |||
517 | } | |||
518 | } | |||
519 | return (NULL((void *)0)); | |||
520 | } | |||
521 | ||||
522 | ||||
523 | /* | |||
524 | * swaplist_trim: scan priority list for empty priority entries and kill | |||
525 | * them. | |||
526 | * | |||
527 | * => caller must hold both swap_syscall_lock and uvm.swap_data_lock | |||
528 | */ | |||
529 | void | |||
530 | swaplist_trim(void) | |||
531 | { | |||
532 | struct swappri *spp, *nextspp; | |||
533 | ||||
534 | LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp)for ((spp) = ((&swap_priority)->lh_first); (spp) && ((nextspp) = ((spp)->spi_swappri.le_next), 1); (spp) = (nextspp )) { | |||
535 | if (!TAILQ_EMPTY(&spp->spi_swapdev)(((&spp->spi_swapdev)->tqh_first) == ((void *)0))) | |||
536 | continue; | |||
537 | LIST_REMOVE(spp, spi_swappri)do { if ((spp)->spi_swappri.le_next != ((void *)0)) (spp)-> spi_swappri.le_next->spi_swappri.le_prev = (spp)->spi_swappri .le_prev; *(spp)->spi_swappri.le_prev = (spp)->spi_swappri .le_next; ((spp)->spi_swappri.le_prev) = ((void *)-1); ((spp )->spi_swappri.le_next) = ((void *)-1); } while (0); | |||
538 | free(spp, M_VMSWAP92, sizeof(*spp)); | |||
539 | } | |||
540 | } | |||
541 | ||||
542 | /* | |||
543 | * swapdrum_add: add a "swapdev"'s blocks into /dev/drum's area. | |||
544 | * | |||
545 | * => caller must hold swap_syscall_lock | |||
546 | * => uvm.swap_data_lock should be unlocked (we may sleep) | |||
547 | */ | |||
548 | void | |||
549 | swapdrum_add(struct swapdev *sdp, int npages) | |||
550 | { | |||
551 | u_long result; | |||
552 | ||||
553 | if (extent_alloc(swapmap, npages, EX_NOALIGN, 0, EX_NOBOUNDARY,extent_alloc_subregion((swapmap), (swapmap)->ex_start, (swapmap )->ex_end, (npages), (1), (0), (0), (0x0001), (&result )) | |||
554 | EX_WAITOK, &result)extent_alloc_subregion((swapmap), (swapmap)->ex_start, (swapmap )->ex_end, (npages), (1), (0), (0), (0x0001), (&result ))) | |||
555 | panic("swapdrum_add"); | |||
556 | ||||
557 | sdp->swd_drumoffset = result; | |||
558 | sdp->swd_drumsize = npages; | |||
559 | } | |||
560 | ||||
561 | /* | |||
562 | * swapdrum_getsdp: given a page offset in /dev/drum, convert it back | |||
563 | * to the "swapdev" that maps that section of the drum. | |||
564 | * | |||
565 | * => each swapdev takes one big contig chunk of the drum | |||
566 | * => caller must hold uvm.swap_data_lock | |||
567 | */ | |||
568 | struct swapdev * | |||
569 | swapdrum_getsdp(int pgno) | |||
570 | { | |||
571 | struct swapdev *sdp; | |||
572 | struct swappri *spp; | |||
573 | ||||
574 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
575 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
576 | if (pgno >= sdp->swd_drumoffset && | |||
577 | pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { | |||
578 | return sdp; | |||
579 | } | |||
580 | } | |||
581 | } | |||
582 | return NULL((void *)0); | |||
583 | } | |||
584 | ||||
585 | ||||
586 | /* | |||
587 | * sys_swapctl: main entry point for swapctl(2) system call | |||
588 | * [with two helper functions: swap_on and swap_off] | |||
589 | */ | |||
590 | int | |||
591 | sys_swapctl(struct proc *p, void *v, register_t *retval) | |||
592 | { | |||
593 | struct sys_swapctl_args /* { | |||
594 | syscallarg(int) cmd; | |||
595 | syscallarg(void *) arg; | |||
596 | syscallarg(int) misc; | |||
597 | } */ *uap = (struct sys_swapctl_args *)v; | |||
598 | struct vnode *vp; | |||
599 | struct nameidata nd; | |||
600 | struct swappri *spp; | |||
601 | struct swapdev *sdp; | |||
602 | struct swapent *sep; | |||
603 | char userpath[MAXPATHLEN1024]; | |||
604 | size_t len; | |||
605 | int count, error, misc; | |||
606 | int priority; | |||
607 | ||||
608 | misc = SCARG(uap, misc)((uap)->misc.le.datum); | |||
609 | ||||
610 | /* | |||
611 | * ensure serialized syscall access by grabbing the swap_syscall_lock | |||
612 | */ | |||
613 | rw_enter_write(&swap_syscall_lock); | |||
614 | ||||
615 | /* | |||
616 | * we handle the non-priv NSWAP and STATS request first. | |||
617 | * | |||
618 | * SWAP_NSWAP: return number of config'd swap devices | |||
619 | * [can also be obtained with uvmexp sysctl] | |||
620 | */ | |||
621 | if (SCARG(uap, cmd)((uap)->cmd.le.datum) == SWAP_NSWAP3) { | |||
622 | *retval = uvmexp.nswapdev; | |||
623 | error = 0; | |||
624 | goto out; | |||
625 | } | |||
626 | ||||
627 | /* | |||
628 | * SWAP_STATS: get stats on current # of configured swap devs | |||
629 | * | |||
630 | * note that the swap_priority list can't change as long | |||
631 | * as we are holding the swap_syscall_lock. we don't want | |||
632 | * to grab the uvm.swap_data_lock because we may fault&sleep during | |||
633 | * copyout() and we don't want to be holding that lock then! | |||
634 | */ | |||
635 | if (SCARG(uap, cmd)((uap)->cmd.le.datum) == SWAP_STATS4) { | |||
636 | sep = (struct swapent *)SCARG(uap, arg)((uap)->arg.le.datum); | |||
637 | count = 0; | |||
638 | ||||
639 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
640 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
641 | if (count >= misc) | |||
642 | continue; | |||
643 | ||||
644 | sdp->swd_inuseswd_se.se_inuse = | |||
645 | btodb((u_int64_t)sdp->swd_npginuse <<(((u_int64_t)sdp->swd_npginuse << 12) >> 9) | |||
646 | PAGE_SHIFT)(((u_int64_t)sdp->swd_npginuse << 12) >> 9); | |||
647 | error = copyout(&sdp->swd_se, sep, | |||
648 | sizeof(struct swapent)); | |||
649 | if (error) | |||
650 | goto out; | |||
651 | ||||
652 | /* now copy out the path if necessary */ | |||
653 | error = copyoutstr(sdp->swd_path, | |||
654 | sep->se_path, sizeof(sep->se_path), NULL((void *)0)); | |||
655 | if (error) | |||
656 | goto out; | |||
657 | ||||
658 | count++; | |||
659 | sep++; | |||
660 | } | |||
661 | } | |||
662 | ||||
663 | *retval = count; | |||
664 | error = 0; | |||
665 | goto out; | |||
666 | } | |||
667 | ||||
668 | /* all other requests require superuser privs. verify. */ | |||
669 | if ((error = suser(p)) || (error = pledge_swapctl(p))) | |||
670 | goto out; | |||
671 | ||||
672 | /* | |||
673 | * at this point we expect a path name in arg. we will | |||
674 | * use namei() to gain a vnode reference (vref), and lock | |||
675 | * the vnode (VOP_LOCK). | |||
676 | */ | |||
677 | error = copyinstr(SCARG(uap, arg)((uap)->arg.le.datum), userpath, sizeof(userpath), &len); | |||
678 | if (error) | |||
679 | goto out; | |||
680 | disk_map(userpath, userpath, sizeof(userpath), DM_OPENBLCK0x2); | |||
681 | NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, UIO_SYSSPACE, userpath, p)ndinitat(&nd, 0, 0x0040|0x0004, UIO_SYSSPACE, -100, userpath , p); | |||
682 | if ((error = namei(&nd))) | |||
683 | goto out; | |||
684 | vp = nd.ni_vp; | |||
685 | /* note: "vp" is referenced and locked */ | |||
686 | ||||
687 | error = 0; /* assume no error */ | |||
688 | switch(SCARG(uap, cmd)((uap)->cmd.le.datum)) { | |||
689 | case SWAP_DUMPDEV7: | |||
690 | if (vp->v_type != VBLK) { | |||
691 | error = ENOTBLK15; | |||
692 | break; | |||
693 | } | |||
694 | dumpdev = vp->v_rdevv_un.vu_specinfo->si_rdev; | |||
695 | break; | |||
696 | case SWAP_CTL5: | |||
697 | /* | |||
698 | * get new priority, remove old entry (if any) and then | |||
699 | * reinsert it in the correct place. finally, prune out | |||
700 | * any empty priority structures. | |||
701 | */ | |||
702 | priority = SCARG(uap, misc)((uap)->misc.le.datum); | |||
703 | spp = malloc(sizeof *spp, M_VMSWAP92, M_WAITOK0x0001); | |||
704 | if ((sdp = swaplist_find(vp, 1)) == NULL((void *)0)) { | |||
705 | error = ENOENT2; | |||
706 | } else { | |||
707 | swaplist_insert(sdp, spp, priority); | |||
708 | swaplist_trim(); | |||
709 | } | |||
710 | if (error) | |||
711 | free(spp, M_VMSWAP92, sizeof(*spp)); | |||
712 | break; | |||
713 | case SWAP_ON1: | |||
714 | /* | |||
715 | * If the device is a regular file, make sure the filesystem | |||
716 | * can be used for swapping. | |||
717 | */ | |||
718 | if (vp->v_type == VREG && | |||
719 | (vp->v_mount->mnt_flag & MNT_SWAPPABLE0x00200000) == 0) { | |||
720 | error = ENOTSUP91; | |||
721 | break; | |||
722 | } | |||
723 | ||||
724 | /* | |||
725 | * check for duplicates. if none found, then insert a | |||
726 | * dummy entry on the list to prevent someone else from | |||
727 | * trying to enable this device while we are working on | |||
728 | * it. | |||
729 | */ | |||
730 | priority = SCARG(uap, misc)((uap)->misc.le.datum); | |||
731 | if ((sdp = swaplist_find(vp, 0)) != NULL((void *)0)) { | |||
732 | error = EBUSY16; | |||
733 | break; | |||
734 | } | |||
735 | sdp = malloc(sizeof *sdp, M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008); | |||
736 | spp = malloc(sizeof *spp, M_VMSWAP92, M_WAITOK0x0001); | |||
737 | sdp->swd_flagsswd_se.se_flags = SWF_FAKE0x00000008; /* placeholder only */ | |||
738 | sdp->swd_vp = vp; | |||
739 | sdp->swd_devswd_se.se_dev = (vp->v_type == VBLK) ? vp->v_rdevv_un.vu_specinfo->si_rdev : NODEV(dev_t)(-1); | |||
740 | ||||
741 | /* | |||
742 | * XXX Is NFS elaboration necessary? | |||
743 | */ | |||
744 | if (vp->v_type == VREG) { | |||
745 | sdp->swd_cred = crdup(p->p_ucred); | |||
746 | } | |||
747 | ||||
748 | swaplist_insert(sdp, spp, priority); | |||
749 | ||||
750 | sdp->swd_pathlen = len; | |||
751 | sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP92, M_WAITOK0x0001); | |||
752 | strlcpy(sdp->swd_path, userpath, len); | |||
753 | ||||
754 | /* | |||
755 | * we've now got a FAKE placeholder in the swap list. | |||
756 | * now attempt to enable swap on it. if we fail, undo | |||
757 | * what we've done and kill the fake entry we just inserted. | |||
758 | * if swap_on is a success, it will clear the SWF_FAKE flag | |||
759 | */ | |||
760 | ||||
761 | if ((error = swap_on(p, sdp)) != 0) { | |||
762 | (void) swaplist_find(vp, 1); /* kill fake entry */ | |||
763 | swaplist_trim(); | |||
764 | if (vp->v_type == VREG) { | |||
765 | crfree(sdp->swd_cred); | |||
766 | } | |||
767 | free(sdp->swd_path, M_VMSWAP92, sdp->swd_pathlen); | |||
768 | free(sdp, M_VMSWAP92, sizeof(*sdp)); | |||
769 | break; | |||
770 | } | |||
771 | break; | |||
772 | case SWAP_OFF2: | |||
773 | if ((sdp = swaplist_find(vp, 0)) == NULL((void *)0)) { | |||
774 | error = ENXIO6; | |||
775 | break; | |||
776 | } | |||
777 | ||||
778 | /* | |||
779 | * If a device isn't in use or enabled, we | |||
780 | * can't stop swapping from it (again). | |||
781 | */ | |||
782 | if ((sdp->swd_flagsswd_se.se_flags & (SWF_INUSE0x00000001|SWF_ENABLE0x00000002)) == 0) { | |||
783 | error = EBUSY16; | |||
784 | break; | |||
785 | } | |||
786 | ||||
787 | /* | |||
788 | * do the real work. | |||
789 | */ | |||
790 | error = swap_off(p, sdp); | |||
791 | break; | |||
792 | default: | |||
793 | error = EINVAL22; | |||
794 | } | |||
795 | ||||
796 | /* done! release the ref gained by namei() and unlock. */ | |||
797 | vput(vp); | |||
798 | ||||
799 | out: | |||
800 | rw_exit_write(&swap_syscall_lock); | |||
801 | ||||
802 | return (error); | |||
803 | } | |||
804 | ||||
805 | /* | |||
806 | * swap_on: attempt to enable a swapdev for swapping. note that the | |||
807 | * swapdev is already on the global list, but disabled (marked | |||
808 | * SWF_FAKE). | |||
809 | * | |||
810 | * => we avoid the start of the disk (to protect disk labels) | |||
811 | * => caller should leave uvm.swap_data_lock unlocked, we may lock it | |||
812 | * if needed. | |||
813 | */ | |||
814 | int | |||
815 | swap_on(struct proc *p, struct swapdev *sdp) | |||
816 | { | |||
817 | static int count = 0; /* static */ | |||
818 | struct vnode *vp; | |||
819 | int error, npages, nblocks, size; | |||
820 | long addr; | |||
821 | struct vattr va; | |||
822 | #if defined(NFSCLIENT1) | |||
823 | extern const struct vops nfs_vops; | |||
824 | #endif /* defined(NFSCLIENT) */ | |||
825 | dev_t dev; | |||
826 | ||||
827 | /* | |||
828 | * we want to enable swapping on sdp. the swd_vp contains | |||
829 | * the vnode we want (locked and ref'd), and the swd_dev | |||
830 | * contains the dev_t of the file, if it a block device. | |||
831 | */ | |||
832 | ||||
833 | vp = sdp->swd_vp; | |||
834 | dev = sdp->swd_devswd_se.se_dev; | |||
835 | ||||
836 | #if NVND1 > 0 | |||
837 | /* no swapping to vnds. */ | |||
838 | if (bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_strategy == vndstrategy) | |||
839 | return (EOPNOTSUPP45); | |||
840 | #endif | |||
841 | ||||
842 | /* | |||
843 | * open the swap file (mostly useful for block device files to | |||
844 | * let device driver know what is up). | |||
845 | * | |||
846 | * we skip the open/close for root on swap because the root | |||
847 | * has already been opened when root was mounted (mountroot). | |||
848 | */ | |||
849 | if (vp != rootvp) { | |||
850 | if ((error = VOP_OPEN(vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p))) | |||
851 | return (error); | |||
852 | } | |||
853 | ||||
854 | /* XXX this only works for block devices */ | |||
855 | /* | |||
856 | * we now need to determine the size of the swap area. for | |||
857 | * block specials we can call the d_psize function. | |||
858 | * for normal files, we must stat [get attrs]. | |||
859 | * | |||
860 | * we put the result in nblks. | |||
861 | * for normal files, we also want the filesystem block size | |||
862 | * (which we get with statfs). | |||
863 | */ | |||
864 | switch (vp->v_type) { | |||
865 | case VBLK: | |||
866 | if (bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_psize == 0 || | |||
867 | (nblocks = (*bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_psize)(dev)) == -1) { | |||
868 | error = ENXIO6; | |||
869 | goto bad; | |||
870 | } | |||
871 | break; | |||
872 | ||||
873 | case VREG: | |||
874 | if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p))) | |||
875 | goto bad; | |||
876 | nblocks = (int)btodb(va.va_size)((va.va_size) >> 9); | |||
877 | if ((error = | |||
878 | VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)(*(vp->v_mount)->mnt_op->vfs_statfs)(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0) | |||
879 | goto bad; | |||
880 | ||||
881 | sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; | |||
882 | /* | |||
883 | * limit the max # of outstanding I/O requests we issue | |||
884 | * at any one time. take it easy on NFS servers. | |||
885 | */ | |||
886 | #if defined(NFSCLIENT1) | |||
887 | if (vp->v_op == &nfs_vops) | |||
888 | sdp->swd_maxactive = 2; /* XXX */ | |||
889 | else | |||
890 | #endif /* defined(NFSCLIENT) */ | |||
891 | sdp->swd_maxactive = 8; /* XXX */ | |||
892 | bufq_init(&sdp->swd_bufq, BUFQ_FIFO0); | |||
893 | break; | |||
894 | ||||
895 | default: | |||
896 | error = ENXIO6; | |||
897 | goto bad; | |||
898 | } | |||
899 | ||||
900 | /* | |||
901 | * save nblocks in a safe place and convert to pages. | |||
902 | */ | |||
903 | ||||
904 | sdp->swd_nblksswd_se.se_nblks = nblocks; | |||
905 | npages = dbtob((u_int64_t)nblocks)(((u_int64_t)nblocks) << 9) >> PAGE_SHIFT12; | |||
906 | ||||
907 | /* | |||
908 | * for block special files, we want to make sure that leave | |||
909 | * the disklabel and bootblocks alone, so we arrange to skip | |||
910 | * over them (arbitrarily choosing to skip PAGE_SIZE bytes). | |||
911 | * note that because of this the "size" can be less than the | |||
912 | * actual number of blocks on the device. | |||
913 | */ | |||
914 | if (vp->v_type == VBLK) { | |||
915 | /* we use pages 1 to (size - 1) [inclusive] */ | |||
916 | size = npages - 1; | |||
917 | addr = 1; | |||
918 | } else { | |||
919 | /* we use pages 0 to (size - 1) [inclusive] */ | |||
920 | size = npages; | |||
921 | addr = 0; | |||
922 | } | |||
923 | ||||
924 | /* | |||
925 | * make sure we have enough blocks for a reasonable sized swap | |||
926 | * area. we want at least one page. | |||
927 | */ | |||
928 | ||||
929 | if (size < 1) { | |||
930 | error = EINVAL22; | |||
931 | goto bad; | |||
932 | } | |||
933 | ||||
934 | /* | |||
935 | * now we need to allocate an extent to manage this swap device | |||
936 | */ | |||
937 | snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x", | |||
938 | count++); | |||
939 | ||||
940 | /* note that extent_create's 3rd arg is inclusive, thus "- 1" */ | |||
941 | sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP92, | |||
942 | 0, 0, EX_WAITOK0x0001); | |||
943 | /* allocate the `saved' region from the extent so it won't be used */ | |||
944 | if (addr) { | |||
945 | if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK0x0001)) | |||
946 | panic("disklabel reserve"); | |||
947 | /* XXX: is extent synchronized with swd_npginuse? */ | |||
948 | } | |||
949 | #ifdef HIBERNATE1 | |||
950 | /* | |||
951 | * Lock down the last region of primary disk swap, in case | |||
952 | * hibernate needs to place a signature there. | |||
953 | */ | |||
954 | if (dev == swdevt[0].sw_dev && vp->v_type == VBLK && size > 3 ) { | |||
955 | if (extent_alloc_region(sdp->swd_ex, | |||
956 | npages - 1 - 1, 1, EX_WAITOK0x0001)) | |||
957 | panic("hibernate reserve"); | |||
958 | /* XXX: is extent synchronized with swd_npginuse? */ | |||
959 | } | |||
960 | #endif | |||
961 | ||||
962 | /* add a ref to vp to reflect usage as a swap device. */ | |||
963 | vref(vp); | |||
964 | ||||
965 | #ifdef UVM_SWAP_ENCRYPT1 | |||
966 | if (uvm_doswapencrypt) | |||
967 | uvm_swap_initcrypt(sdp, npages); | |||
968 | #endif | |||
969 | /* now add the new swapdev to the drum and enable. */ | |||
970 | swapdrum_add(sdp, npages); | |||
971 | sdp->swd_npages = size; | |||
972 | sdp->swd_flagsswd_se.se_flags &= ~SWF_FAKE0x00000008; /* going live */ | |||
973 | sdp->swd_flagsswd_se.se_flags |= (SWF_INUSE0x00000001|SWF_ENABLE0x00000002); | |||
974 | uvmexp.swpages += size; | |||
975 | return (0); | |||
976 | ||||
977 | bad: | |||
978 | /* failure: close device if necessary and return error. */ | |||
979 | if (vp != rootvp) | |||
980 | (void)VOP_CLOSE(vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p); | |||
981 | return (error); | |||
982 | } | |||
983 | ||||
984 | /* | |||
985 | * swap_off: stop swapping on swapdev | |||
986 | * | |||
987 | * => swap data should be locked, we will unlock. | |||
988 | */ | |||
989 | int | |||
990 | swap_off(struct proc *p, struct swapdev *sdp) | |||
991 | { | |||
992 | int error = 0; | |||
993 | ||||
994 | /* disable the swap area being removed */ | |||
995 | sdp->swd_flagsswd_se.se_flags &= ~SWF_ENABLE0x00000002; | |||
996 | ||||
997 | /* | |||
998 | * the idea is to find all the pages that are paged out to this | |||
999 | * device, and page them all in. in uvm, swap-backed pageable | |||
1000 | * memory can take two forms: aobjs and anons. call the | |||
1001 | * swapoff hook for each subsystem to bring in pages. | |||
1002 | */ | |||
1003 | ||||
1004 | if (uao_swap_off(sdp->swd_drumoffset, | |||
1005 | sdp->swd_drumoffset + sdp->swd_drumsize) || | |||
1006 | amap_swap_off(sdp->swd_drumoffset, | |||
1007 | sdp->swd_drumoffset + sdp->swd_drumsize)) { | |||
1008 | ||||
1009 | error = ENOMEM12; | |||
1010 | } else if (sdp->swd_npginuse > sdp->swd_npgbad) { | |||
1011 | error = EBUSY16; | |||
1012 | } | |||
1013 | ||||
1014 | if (error) { | |||
1015 | sdp->swd_flagsswd_se.se_flags |= SWF_ENABLE0x00000002; | |||
1016 | return (error); | |||
1017 | } | |||
1018 | ||||
1019 | /* | |||
1020 | * done with the vnode and saved creds. | |||
1021 | * drop our ref on the vnode before calling VOP_CLOSE() | |||
1022 | * so that spec_close() can tell if this is the last close. | |||
1023 | */ | |||
1024 | if (sdp->swd_vp->v_type == VREG) { | |||
1025 | crfree(sdp->swd_cred); | |||
1026 | } | |||
1027 | vrele(sdp->swd_vp); | |||
1028 | if (sdp->swd_vp != rootvp) { | |||
1029 | (void) VOP_CLOSE(sdp->swd_vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p); | |||
1030 | } | |||
1031 | ||||
1032 | uvmexp.swpages -= sdp->swd_npages; | |||
1033 | ||||
1034 | if (swaplist_find(sdp->swd_vp, 1) == NULL((void *)0)) | |||
1035 | panic("swap_off: swapdev not in list"); | |||
1036 | swaplist_trim(); | |||
1037 | ||||
1038 | /* | |||
1039 | * free all resources! | |||
1040 | */ | |||
1041 | extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, | |||
1042 | EX_WAITOK0x0001); | |||
1043 | extent_destroy(sdp->swd_ex); | |||
1044 | /* free sdp->swd_path ? */ | |||
1045 | free(sdp, M_VMSWAP92, sizeof(*sdp)); | |||
1046 | return (0); | |||
1047 | } | |||
1048 | ||||
1049 | /* | |||
1050 | * /dev/drum interface and i/o functions | |||
1051 | */ | |||
1052 | ||||
1053 | /* | |||
1054 | * swstrategy: perform I/O on the drum | |||
1055 | * | |||
1056 | * => we must map the i/o request from the drum to the correct swapdev. | |||
1057 | */ | |||
1058 | void | |||
1059 | swstrategy(struct buf *bp) | |||
1060 | { | |||
1061 | struct swapdev *sdp; | |||
1062 | int s, pageno, bn; | |||
1063 | ||||
1064 | /* | |||
1065 | * convert block number to swapdev. note that swapdev can't | |||
1066 | * be yanked out from under us because we are holding resources | |||
1067 | * in it (i.e. the blocks we are doing I/O on). | |||
1068 | */ | |||
1069 | pageno = dbtob((u_int64_t)bp->b_blkno)(((u_int64_t)bp->b_blkno) << 9) >> PAGE_SHIFT12; | |||
1070 | sdp = swapdrum_getsdp(pageno); | |||
1071 | if (sdp == NULL((void *)0)) { | |||
1072 | bp->b_error = EINVAL22; | |||
1073 | bp->b_flags |= B_ERROR0x00000400; | |||
1074 | s = splbio()splraise(0x6); | |||
1075 | biodone(bp); | |||
1076 | splx(s)spllower(s); | |||
1077 | return; | |||
1078 | } | |||
1079 | ||||
1080 | /* convert drum page number to block number on this swapdev. */ | |||
1081 | pageno -= sdp->swd_drumoffset; /* page # on swapdev */ | |||
1082 | bn = btodb((u_int64_t)pageno << PAGE_SHIFT)(((u_int64_t)pageno << 12) >> 9); /* convert to diskblock */ | |||
1083 | ||||
1084 | /* | |||
1085 | * for block devices we finish up here. | |||
1086 | * for regular files we have to do more work which we delegate | |||
1087 | * to sw_reg_strategy(). | |||
1088 | */ | |||
1089 | switch (sdp->swd_vp->v_type) { | |||
1090 | default: | |||
1091 | panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); | |||
1092 | case VBLK: | |||
1093 | /* | |||
1094 | * must convert "bp" from an I/O on /dev/drum to an I/O | |||
1095 | * on the swapdev (sdp). | |||
1096 | */ | |||
1097 | s = splbio()splraise(0x6); | |||
1098 | buf_replacevnode(bp, sdp->swd_vp); | |||
1099 | ||||
1100 | bp->b_blkno = bn; | |||
1101 | splx(s)spllower(s); | |||
1102 | VOP_STRATEGY(bp->b_vp, bp); | |||
1103 | return; | |||
1104 | case VREG: | |||
1105 | /* delegate to sw_reg_strategy function. */ | |||
1106 | sw_reg_strategy(sdp, bp, bn); | |||
1107 | return; | |||
1108 | } | |||
1109 | /* NOTREACHED */ | |||
1110 | } | |||
1111 | ||||
1112 | /* | |||
1113 | * sw_reg_strategy: handle swap i/o to regular files | |||
1114 | */ | |||
1115 | void | |||
1116 | sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) | |||
1117 | { | |||
1118 | struct vnode *vp; | |||
1119 | struct vndxfer *vnx; | |||
1120 | daddr_t nbn; | |||
1121 | caddr_t addr; | |||
1122 | off_t byteoff; | |||
1123 | int s, off, nra, error, sz, resid; | |||
1124 | ||||
1125 | /* | |||
1126 | * allocate a vndxfer head for this transfer and point it to | |||
1127 | * our buffer. | |||
1128 | */ | |||
1129 | vnx = pool_get(&vndxfer_pool, PR_WAITOK0x0001); | |||
1130 | vnx->vx_flags = VX_BUSY1; | |||
1131 | vnx->vx_error = 0; | |||
1132 | vnx->vx_pending = 0; | |||
1133 | vnx->vx_bp = bp; | |||
1134 | vnx->vx_sdp = sdp; | |||
1135 | ||||
1136 | /* | |||
1137 | * setup for main loop where we read filesystem blocks into | |||
1138 | * our buffer. | |||
1139 | */ | |||
1140 | error = 0; | |||
1141 | bp->b_resid = bp->b_bcount; /* nothing transferred yet! */ | |||
1142 | addr = bp->b_data; /* current position in buffer */ | |||
1143 | byteoff = dbtob((u_int64_t)bn)(((u_int64_t)bn) << 9); | |||
1144 | ||||
1145 | for (resid = bp->b_resid; resid; resid -= sz) { | |||
1146 | struct vndbuf *nbp; | |||
1147 | /* | |||
1148 | * translate byteoffset into block number. return values: | |||
1149 | * vp = vnode of underlying device | |||
1150 | * nbn = new block number (on underlying vnode dev) | |||
1151 | * nra = num blocks we can read-ahead (excludes requested | |||
1152 | * block) | |||
1153 | */ | |||
1154 | nra = 0; | |||
1155 | error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, | |||
1156 | &vp, &nbn, &nra); | |||
1157 | ||||
1158 | if (error == 0 && nbn == -1) { | |||
1159 | /* | |||
1160 | * this used to just set error, but that doesn't | |||
1161 | * do the right thing. Instead, it causes random | |||
1162 | * memory errors. The panic() should remain until | |||
1163 | * this condition doesn't destabilize the system. | |||
1164 | */ | |||
1165 | #if 1 | |||
1166 | panic("sw_reg_strategy: swap to sparse file"); | |||
1167 | #else | |||
1168 | error = EIO5; /* failure */ | |||
1169 | #endif | |||
1170 | } | |||
1171 | ||||
1172 | /* | |||
1173 | * punt if there was an error or a hole in the file. | |||
1174 | * we must wait for any i/o ops we have already started | |||
1175 | * to finish before returning. | |||
1176 | * | |||
1177 | * XXX we could deal with holes here but it would be | |||
1178 | * a hassle (in the write case). | |||
1179 | */ | |||
1180 | if (error) { | |||
1181 | s = splbio()splraise(0x6); | |||
1182 | vnx->vx_error = error; /* pass error up */ | |||
1183 | goto out; | |||
1184 | } | |||
1185 | ||||
1186 | /* | |||
1187 | * compute the size ("sz") of this transfer (in bytes). | |||
1188 | */ | |||
1189 | off = byteoff % sdp->swd_bsize; | |||
1190 | sz = (1 + nra) * sdp->swd_bsize - off; | |||
1191 | if (sz > resid) | |||
1192 | sz = resid; | |||
1193 | ||||
1194 | /* | |||
1195 | * now get a buf structure. note that the vb_buf is | |||
1196 | * at the front of the nbp structure so that you can | |||
1197 | * cast pointers between the two structure easily. | |||
1198 | */ | |||
1199 | nbp = pool_get(&vndbuf_pool, PR_WAITOK0x0001); | |||
1200 | nbp->vb_buf.b_flags = bp->b_flags | B_CALL0x00000040; | |||
1201 | nbp->vb_buf.b_bcount = sz; | |||
1202 | nbp->vb_buf.b_bufsize = sz; | |||
1203 | nbp->vb_buf.b_error = 0; | |||
1204 | nbp->vb_buf.b_data = addr; | |||
1205 | nbp->vb_buf.b_bq = NULL((void *)0); | |||
1206 | nbp->vb_buf.b_blkno = nbn + btodb(off)((off) >> 9); | |||
1207 | nbp->vb_buf.b_proc = bp->b_proc; | |||
1208 | nbp->vb_buf.b_iodone = sw_reg_iodone; | |||
1209 | nbp->vb_buf.b_vp = NULLVP((struct vnode *)((void *)0)); | |||
1210 | nbp->vb_buf.b_vnbufs.le_next = NOLIST((struct buf *)0x87654321); | |||
1211 | LIST_INIT(&nbp->vb_buf.b_dep)do { ((&nbp->vb_buf.b_dep)->lh_first) = ((void *)0) ; } while (0); | |||
1212 | ||||
1213 | /* | |||
1214 | * set b_dirtyoff/end and b_validoff/end. this is | |||
1215 | * required by the NFS client code (otherwise it will | |||
1216 | * just discard our I/O request). | |||
1217 | */ | |||
1218 | if (bp->b_dirtyend == 0) { | |||
1219 | nbp->vb_buf.b_dirtyoff = 0; | |||
1220 | nbp->vb_buf.b_dirtyend = sz; | |||
1221 | } else { | |||
1222 | nbp->vb_buf.b_dirtyoff = | |||
1223 | max(0, bp->b_dirtyoff - (bp->b_bcount-resid)); | |||
1224 | nbp->vb_buf.b_dirtyend = | |||
1225 | min(sz, | |||
1226 | max(0, bp->b_dirtyend - (bp->b_bcount-resid))); | |||
1227 | } | |||
1228 | if (bp->b_validend == 0) { | |||
1229 | nbp->vb_buf.b_validoff = 0; | |||
1230 | nbp->vb_buf.b_validend = sz; | |||
1231 | } else { | |||
1232 | nbp->vb_buf.b_validoff = | |||
1233 | max(0, bp->b_validoff - (bp->b_bcount-resid)); | |||
1234 | nbp->vb_buf.b_validend = | |||
1235 | min(sz, | |||
1236 | max(0, bp->b_validend - (bp->b_bcount-resid))); | |||
1237 | } | |||
1238 | ||||
1239 | /* patch it back to the vnx */ | |||
1240 | nbp->vb_vnx = vnx; | |||
1241 | task_set(&nbp->vb_task, sw_reg_iodone_internal, nbp); | |||
1242 | ||||
1243 | s = splbio()splraise(0x6); | |||
1244 | if (vnx->vx_error != 0) { | |||
1245 | pool_put(&vndbuf_pool, nbp); | |||
1246 | goto out; | |||
1247 | } | |||
1248 | vnx->vx_pending++; | |||
1249 | ||||
1250 | /* assoc new buffer with underlying vnode */ | |||
1251 | bgetvp(vp, &nbp->vb_buf); | |||
1252 | ||||
1253 | /* start I/O if we are not over our limit */ | |||
1254 | bufq_queue(&sdp->swd_bufq, &nbp->vb_buf); | |||
1255 | sw_reg_start(sdp); | |||
1256 | splx(s)spllower(s); | |||
1257 | ||||
1258 | /* | |||
1259 | * advance to the next I/O | |||
1260 | */ | |||
1261 | byteoff += sz; | |||
1262 | addr += sz; | |||
1263 | } | |||
1264 | ||||
1265 | s = splbio()splraise(0x6); | |||
1266 | ||||
1267 | out: /* Arrive here at splbio */ | |||
1268 | vnx->vx_flags &= ~VX_BUSY1; | |||
1269 | if (vnx->vx_pending == 0) { | |||
1270 | if (vnx->vx_error != 0) { | |||
1271 | bp->b_error = vnx->vx_error; | |||
1272 | bp->b_flags |= B_ERROR0x00000400; | |||
1273 | } | |||
1274 | pool_put(&vndxfer_pool, vnx); | |||
1275 | biodone(bp); | |||
1276 | } | |||
1277 | splx(s)spllower(s); | |||
1278 | } | |||
1279 | ||||
1280 | /* sw_reg_start: start an I/O request on the requested swapdev. */ | |||
1281 | void | |||
1282 | sw_reg_start(struct swapdev *sdp) | |||
1283 | { | |||
1284 | struct buf *bp; | |||
1285 | ||||
1286 | /* XXX: recursion control */ | |||
1287 | if ((sdp->swd_flagsswd_se.se_flags & SWF_BUSY0x00000004) != 0) | |||
1288 | return; | |||
1289 | ||||
1290 | sdp->swd_flagsswd_se.se_flags |= SWF_BUSY0x00000004; | |||
1291 | ||||
1292 | while (sdp->swd_active < sdp->swd_maxactive) { | |||
1293 | bp = bufq_dequeue(&sdp->swd_bufq); | |||
1294 | if (bp == NULL((void *)0)) | |||
1295 | break; | |||
1296 | ||||
1297 | sdp->swd_active++; | |||
1298 | ||||
1299 | if ((bp->b_flags & B_READ0x00008000) == 0) | |||
1300 | bp->b_vp->v_numoutput++; | |||
1301 | ||||
1302 | VOP_STRATEGY(bp->b_vp, bp); | |||
1303 | } | |||
1304 | sdp->swd_flagsswd_se.se_flags &= ~SWF_BUSY0x00000004; | |||
1305 | } | |||
1306 | ||||
1307 | /* | |||
1308 | * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup | |||
1309 | * | |||
1310 | * => note that we can recover the vndbuf struct by casting the buf ptr | |||
1311 | * | |||
1312 | * XXX: | |||
1313 | * We only put this onto a taskq here, because of the maxactive game since | |||
1314 | * it basically requires us to call back into VOP_STRATEGY() (where we must | |||
1315 | * be able to sleep) via sw_reg_start(). | |||
1316 | */ | |||
1317 | void | |||
1318 | sw_reg_iodone(struct buf *bp) | |||
1319 | { | |||
1320 | struct vndbuf *vbp = (struct vndbuf *)bp; | |||
1321 | task_add(systq, &vbp->vb_task); | |||
1322 | } | |||
1323 | ||||
1324 | void | |||
1325 | sw_reg_iodone_internal(void *xvbp) | |||
1326 | { | |||
1327 | struct vndbuf *vbp = xvbp; | |||
1328 | struct vndxfer *vnx = vbp->vb_vnx; | |||
1329 | struct buf *pbp = vnx->vx_bp; /* parent buffer */ | |||
1330 | struct swapdev *sdp = vnx->vx_sdp; | |||
1331 | int resid, s; | |||
1332 | ||||
1333 | s = splbio()splraise(0x6); | |||
1334 | ||||
1335 | resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; | |||
1336 | pbp->b_resid -= resid; | |||
1337 | vnx->vx_pending--; | |||
1338 | ||||
1339 | /* pass error upward */ | |||
1340 | if (vbp->vb_buf.b_error) | |||
1341 | vnx->vx_error = vbp->vb_buf.b_error; | |||
1342 | ||||
1343 | /* disassociate this buffer from the vnode (if any). */ | |||
1344 | if (vbp->vb_buf.b_vp != NULL((void *)0)) { | |||
1345 | brelvp(&vbp->vb_buf); | |||
1346 | } | |||
1347 | ||||
1348 | /* kill vbp structure */ | |||
1349 | pool_put(&vndbuf_pool, vbp); | |||
1350 | ||||
1351 | /* | |||
1352 | * wrap up this transaction if it has run to completion or, in | |||
1353 | * case of an error, when all auxiliary buffers have returned. | |||
1354 | */ | |||
1355 | if (vnx->vx_error != 0) { | |||
1356 | /* pass error upward */ | |||
1357 | pbp->b_flags |= B_ERROR0x00000400; | |||
1358 | pbp->b_error = vnx->vx_error; | |||
1359 | if ((vnx->vx_flags & VX_BUSY1) == 0 && vnx->vx_pending == 0) { | |||
1360 | pool_put(&vndxfer_pool, vnx); | |||
1361 | biodone(pbp); | |||
1362 | } | |||
1363 | } else if (pbp->b_resid == 0) { | |||
1364 | KASSERT(vnx->vx_pending == 0)((vnx->vx_pending == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_swap.c", 1364, "vnx->vx_pending == 0" )); | |||
1365 | if ((vnx->vx_flags & VX_BUSY1) == 0) { | |||
1366 | pool_put(&vndxfer_pool, vnx); | |||
1367 | biodone(pbp); | |||
1368 | } | |||
1369 | } | |||
1370 | ||||
1371 | /* | |||
1372 | * done! start next swapdev I/O if one is pending | |||
1373 | */ | |||
1374 | sdp->swd_active--; | |||
1375 | sw_reg_start(sdp); | |||
1376 | splx(s)spllower(s); | |||
1377 | } | |||
1378 | ||||
1379 | ||||
1380 | /* | |||
1381 | * uvm_swap_alloc: allocate space on swap | |||
1382 | * | |||
1383 | * => allocation is done "round robin" down the priority list, as we | |||
1384 | * allocate in a priority we "rotate" the tail queue. | |||
1385 | * => space can be freed with uvm_swap_free | |||
1386 | * => we return the page slot number in /dev/drum (0 == invalid slot) | |||
1387 | * => we lock uvm.swap_data_lock | |||
1388 | * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM | |||
1389 | */ | |||
1390 | int | |||
1391 | uvm_swap_alloc(int *nslots, boolean_t lessok) | |||
1392 | { | |||
1393 | struct swapdev *sdp; | |||
1394 | struct swappri *spp; | |||
1395 | u_long result; | |||
1396 | ||||
1397 | /* | |||
1398 | * no swap devices configured yet? definite failure. | |||
1399 | */ | |||
1400 | if (uvmexp.nswapdev < 1) | |||
1401 | return 0; | |||
1402 | ||||
1403 | /* | |||
1404 | * lock data lock, convert slots into blocks, and enter loop | |||
1405 | */ | |||
1406 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1406, "_kernel_lock_held()")); | |||
1407 | ReTry: /* XXXMRG */ | |||
1408 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
1409 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
1410 | /* if it's not enabled, then we can't swap from it */ | |||
1411 | if ((sdp->swd_flagsswd_se.se_flags & SWF_ENABLE0x00000002) == 0) | |||
1412 | continue; | |||
1413 | if (sdp->swd_npginuse + *nslots > sdp->swd_npages) | |||
1414 | continue; | |||
1415 | if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN, 0,extent_alloc_subregion((sdp->swd_ex), (sdp->swd_ex)-> ex_start, (sdp->swd_ex)->ex_end, (*nslots), (1), (0), ( 0), (0x0010|0x0000), (&result)) | |||
1416 | EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT,extent_alloc_subregion((sdp->swd_ex), (sdp->swd_ex)-> ex_start, (sdp->swd_ex)->ex_end, (*nslots), (1), (0), ( 0), (0x0010|0x0000), (&result)) | |||
1417 | &result)extent_alloc_subregion((sdp->swd_ex), (sdp->swd_ex)-> ex_start, (sdp->swd_ex)->ex_end, (*nslots), (1), (0), ( 0), (0x0010|0x0000), (&result)) != 0) { | |||
1418 | continue; | |||
1419 | } | |||
1420 | ||||
1421 | /* | |||
1422 | * successful allocation! now rotate the tailq. | |||
1423 | */ | |||
1424 | TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next)do { if (((sdp)->swd_next.tqe_next) != ((void *)0)) (sdp)-> swd_next.tqe_next->swd_next.tqe_prev = (sdp)->swd_next. tqe_prev; else (&spp->spi_swapdev)->tqh_last = (sdp )->swd_next.tqe_prev; *(sdp)->swd_next.tqe_prev = (sdp) ->swd_next.tqe_next; ((sdp)->swd_next.tqe_prev) = ((void *)-1); ((sdp)->swd_next.tqe_next) = ((void *)-1); } while (0); | |||
1425 | TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next)do { (sdp)->swd_next.tqe_next = ((void *)0); (sdp)->swd_next .tqe_prev = (&spp->spi_swapdev)->tqh_last; *(&spp ->spi_swapdev)->tqh_last = (sdp); (&spp->spi_swapdev )->tqh_last = &(sdp)->swd_next.tqe_next; } while (0 ); | |||
1426 | sdp->swd_npginuse += *nslots; | |||
1427 | uvmexp.swpginuse += *nslots; | |||
1428 | /* done! return drum slot number */ | |||
1429 | return result + sdp->swd_drumoffset; | |||
1430 | } | |||
1431 | } | |||
1432 | ||||
1433 | /* XXXMRG: BEGIN HACK */ | |||
1434 | if (*nslots > 1 && lessok) { | |||
1435 | *nslots = 1; | |||
1436 | goto ReTry; /* XXXMRG: ugh! extent should support this for us */ | |||
1437 | } | |||
1438 | /* XXXMRG: END HACK */ | |||
1439 | ||||
1440 | return 0; /* failed */ | |||
1441 | } | |||
1442 | ||||
1443 | /* | |||
1444 | * uvm_swapisfull: return true if all of available swap is allocated | |||
1445 | * and in use. | |||
1446 | */ | |||
1447 | int | |||
1448 | uvm_swapisfull(void) | |||
1449 | { | |||
1450 | int result; | |||
1451 | ||||
1452 | KERNEL_LOCK()_kernel_lock(); | |||
1453 | KASSERT(uvmexp.swpgonly <= uvmexp.swpages)((uvmexp.swpgonly <= uvmexp.swpages) ? (void)0 : __assert( "diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 1453, "uvmexp.swpgonly <= uvmexp.swpages" )); | |||
1454 | result = (uvmexp.swpgonly == uvmexp.swpages); | |||
1455 | KERNEL_UNLOCK()_kernel_unlock(); | |||
1456 | ||||
1457 | return result; | |||
1458 | } | |||
1459 | ||||
1460 | /* | |||
1461 | * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors | |||
1462 | * | |||
1463 | * => we lock uvm.swap_data_lock | |||
1464 | */ | |||
1465 | void | |||
1466 | uvm_swap_markbad(int startslot, int nslots) | |||
1467 | { | |||
1468 | struct swapdev *sdp; | |||
1469 | ||||
1470 | KERNEL_LOCK()_kernel_lock(); | |||
1471 | sdp = swapdrum_getsdp(startslot); | |||
1472 | if (sdp != NULL((void *)0)) { | |||
1473 | /* | |||
1474 | * we just keep track of how many pages have been marked bad | |||
1475 | * in this device, to make everything add up in swap_off(). | |||
1476 | * we assume here that the range of slots will all be within | |||
1477 | * one swap device. | |||
1478 | */ | |||
1479 | sdp->swd_npgbad += nslots; | |||
1480 | } | |||
1481 | KERNEL_UNLOCK()_kernel_unlock(); | |||
1482 | } | |||
1483 | ||||
1484 | /* | |||
1485 | * uvm_swap_free: free swap slots | |||
1486 | * | |||
1487 | * => this can be all or part of an allocation made by uvm_swap_alloc | |||
1488 | * => we lock uvm.swap_data_lock | |||
1489 | */ | |||
1490 | void | |||
1491 | uvm_swap_free(int startslot, int nslots) | |||
1492 | { | |||
1493 | struct swapdev *sdp; | |||
1494 | ||||
1495 | /* | |||
1496 | * ignore attempts to free the "bad" slot. | |||
1497 | */ | |||
1498 | ||||
1499 | if (startslot == SWSLOT_BAD(-1)) { | |||
1500 | return; | |||
1501 | } | |||
1502 | ||||
1503 | /* | |||
1504 | * convert drum slot offset back to sdp, free the blocks | |||
1505 | * in the extent, and return. must hold pri lock to do | |||
1506 | * lookup and access the extent. | |||
1507 | */ | |||
1508 | KERNEL_LOCK()_kernel_lock(); | |||
1509 | sdp = swapdrum_getsdp(startslot); | |||
1510 | KASSERT(uvmexp.nswapdev >= 1)((uvmexp.nswapdev >= 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_swap.c", 1510, "uvmexp.nswapdev >= 1" )); | |||
1511 | KASSERT(sdp != NULL)((sdp != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1511, "sdp != NULL")); | |||
1512 | KASSERT(sdp->swd_npginuse >= nslots)((sdp->swd_npginuse >= nslots) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_swap.c", 1512, "sdp->swd_npginuse >= nslots" )); | |||
1513 | if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots, | |||
1514 | EX_MALLOCOK0x0010|EX_NOWAIT0x0000) != 0) { | |||
1515 | printf("warning: resource shortage: %d pages of swap lost\n", | |||
1516 | nslots); | |||
1517 | } | |||
1518 | ||||
1519 | sdp->swd_npginuse -= nslots; | |||
1520 | uvmexp.swpginuse -= nslots; | |||
1521 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1522 | { | |||
1523 | int i; | |||
1524 | if (swap_encrypt_initialized) { | |||
1525 | /* Dereference keys */ | |||
1526 | for (i = 0; i < nslots; i++) | |||
1527 | if (uvm_swap_needdecrypt(sdp, startslot + i)) { | |||
1528 | struct swap_key *key; | |||
1529 | ||||
1530 | key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]); | |||
1531 | if (key->refcount != 0) | |||
1532 | SWAP_KEY_PUT(sdp, key)do { (key)->refcount--; if ((key)->refcount == 0) { swap_key_delete (key); } } while(0);; | |||
1533 | } | |||
1534 | ||||
1535 | /* Mark range as not decrypt */ | |||
1536 | uvm_swap_markdecrypt(sdp, startslot, nslots, 0); | |||
1537 | } | |||
1538 | } | |||
1539 | #endif /* UVM_SWAP_ENCRYPT */ | |||
1540 | KERNEL_UNLOCK()_kernel_unlock(); | |||
1541 | } | |||
1542 | ||||
1543 | /* | |||
1544 | * uvm_swap_put: put any number of pages into a contig place on swap | |||
1545 | * | |||
1546 | * => can be sync or async | |||
1547 | */ | |||
1548 | int | |||
1549 | uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) | |||
1550 | { | |||
1551 | int result; | |||
1552 | ||||
1553 | result = uvm_swap_io(ppsp, swslot, npages, B_WRITE0x00000000 | | |||
1554 | ((flags & PGO_SYNCIO0x002) ? 0 : B_ASYNC0x00000004)); | |||
1555 | ||||
1556 | return (result); | |||
1557 | } | |||
1558 | ||||
1559 | /* | |||
1560 | * uvm_swap_get: get a single page from swap | |||
1561 | * | |||
1562 | * => usually a sync op (from fault) | |||
1563 | */ | |||
1564 | int | |||
1565 | uvm_swap_get(struct vm_page *page, int swslot, int flags) | |||
1566 | { | |||
1567 | int result; | |||
1568 | ||||
1569 | uvmexp.nswget++; | |||
1570 | KASSERT(flags & PGO_SYNCIO)((flags & 0x002) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1570, "flags & PGO_SYNCIO")); | |||
1571 | if (swslot == SWSLOT_BAD(-1)) { | |||
1572 | return VM_PAGER_ERROR4; | |||
1573 | } | |||
1574 | ||||
1575 | KERNEL_LOCK()_kernel_lock(); | |||
1576 | /* this page is (about to be) no longer only in swap. */ | |||
1577 | atomic_dec_int(&uvmexp.swpgonly)_atomic_dec_int(&uvmexp.swpgonly); | |||
1578 | ||||
1579 | result = uvm_swap_io(&page, swslot, 1, B_READ0x00008000 | | |||
1580 | ((flags & PGO_SYNCIO0x002) ? 0 : B_ASYNC0x00000004)); | |||
1581 | ||||
1582 | if (result != VM_PAGER_OK0 && result != VM_PAGER_PEND3) { | |||
1583 | /* oops, the read failed so it really is still only in swap. */ | |||
1584 | atomic_inc_int(&uvmexp.swpgonly)_atomic_inc_int(&uvmexp.swpgonly); | |||
1585 | } | |||
1586 | KERNEL_UNLOCK()_kernel_unlock(); | |||
1587 | return (result); | |||
1588 | } | |||
1589 | ||||
1590 | /* | |||
1591 | * uvm_swap_io: do an i/o operation to swap | |||
1592 | */ | |||
1593 | ||||
1594 | int | |||
1595 | uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) | |||
1596 | { | |||
1597 | daddr_t startblk; | |||
1598 | struct buf *bp; | |||
1599 | vaddr_t kva; | |||
1600 | int result, s, mapinflags, pflag, bounce = 0, i; | |||
1601 | boolean_t write, async; | |||
1602 | vaddr_t bouncekva; | |||
1603 | struct vm_page *tpps[MAXBSIZE(64 * 1024) >> PAGE_SHIFT12]; | |||
1604 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1605 | struct swapdev *sdp; | |||
| ||||
1606 | int encrypt = 0; | |||
1607 | #endif | |||
1608 | ||||
1609 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1609, "_kernel_lock_held()")); | |||
1610 | ||||
1611 | write = (flags & B_READ0x00008000) == 0; | |||
1612 | async = (flags & B_ASYNC0x00000004) != 0; | |||
1613 | ||||
1614 | /* convert starting drum slot to block number */ | |||
1615 | startblk = btodb((u_int64_t)startslot << PAGE_SHIFT)(((u_int64_t)startslot << 12) >> 9); | |||
1616 | ||||
1617 | /* | |||
1618 | * first, map the pages into the kernel (XXX: currently required | |||
1619 | * by buffer system). | |||
1620 | */ | |||
1621 | mapinflags = !write
| |||
1622 | if (!async
| |||
1623 | mapinflags |= UVMPAGER_MAPIN_WAITOK0x01; | |||
1624 | kva = uvm_pagermapin(pps, npages, mapinflags); | |||
1625 | if (kva == 0) | |||
1626 | return (VM_PAGER_AGAIN5); | |||
1627 | ||||
1628 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1629 | if (write
| |||
1630 | /* | |||
1631 | * Check if we need to do swap encryption on old pages. | |||
1632 | * Later we need a different scheme, that swap encrypts | |||
1633 | * all pages of a process that had at least one page swap | |||
1634 | * encrypted. Then we might not need to copy all pages | |||
1635 | * in the cluster, and avoid the memory overheard in | |||
1636 | * swapping. | |||
1637 | */ | |||
1638 | if (uvm_doswapencrypt) | |||
1639 | encrypt = 1; | |||
1640 | } | |||
1641 | ||||
1642 | if (swap_encrypt_initialized || encrypt
| |||
1643 | /* | |||
1644 | * we need to know the swap device that we are swapping to/from | |||
1645 | * to see if the pages need to be marked for decryption or | |||
1646 | * actually need to be decrypted. | |||
1647 | * XXX - does this information stay the same over the whole | |||
1648 | * execution of this function? | |||
1649 | */ | |||
1650 | sdp = swapdrum_getsdp(startslot); | |||
1651 | } | |||
1652 | ||||
1653 | /* | |||
1654 | * Check that we are dma capable for read (write always bounces | |||
1655 | * through the swapencrypt anyway... | |||
1656 | */ | |||
1657 | if (write
| |||
1658 | bounce = 1; /* bounce through swapencrypt always */ | |||
1659 | } else { | |||
1660 | #else | |||
1661 | { | |||
1662 | #endif | |||
1663 | ||||
1664 | for (i = 0; i < npages; i++) { | |||
1665 | if (VM_PAGE_TO_PHYS(pps[i])((pps[i])->phys_addr) < dma_constraint.ucr_low || | |||
1666 | VM_PAGE_TO_PHYS(pps[i])((pps[i])->phys_addr) > dma_constraint.ucr_high) { | |||
1667 | bounce = 1; | |||
1668 | break; | |||
1669 | } | |||
1670 | } | |||
1671 | } | |||
1672 | ||||
1673 | if (bounce
| |||
1674 | int swmapflags; | |||
1675 | ||||
1676 | /* We always need write access. */ | |||
1677 | swmapflags = UVMPAGER_MAPIN_READ0x02; | |||
1678 | if (!async) | |||
1679 | swmapflags |= UVMPAGER_MAPIN_WAITOK0x01; | |||
1680 | ||||
1681 | if (!uvm_swap_allocpages(tpps, npages)) { | |||
1682 | uvm_pagermapout(kva, npages); | |||
1683 | return (VM_PAGER_AGAIN5); | |||
1684 | } | |||
1685 | ||||
1686 | bouncekva = uvm_pagermapin(tpps, npages, swmapflags); | |||
1687 | if (bouncekva == 0) { | |||
1688 | uvm_pagermapout(kva, npages); | |||
1689 | uvm_swap_freepages(tpps, npages); | |||
1690 | return (VM_PAGER_AGAIN5); | |||
1691 | } | |||
1692 | } | |||
1693 | ||||
1694 | /* encrypt to swap */ | |||
1695 | if (write
| |||
1696 | int i, opages; | |||
1697 | caddr_t src, dst; | |||
1698 | u_int64_t block; | |||
1699 | ||||
1700 | src = (caddr_t) kva; | |||
1701 | dst = (caddr_t) bouncekva; | |||
1702 | block = startblk; | |||
1703 | for (i = 0; i < npages; i++) { | |||
1704 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1705 | struct swap_key *key; | |||
1706 | ||||
1707 | if (encrypt) { | |||
1708 | key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]); | |||
1709 | SWAP_KEY_GET(sdp, key)do { if ((key)->refcount == 0) { swap_key_create(key); } ( key)->refcount++; } while(0);; /* add reference */ | |||
1710 | ||||
1711 | swap_encrypt(key, src, dst, block, PAGE_SIZE(1 << 12)); | |||
1712 | block += btodb(PAGE_SIZE)(((1 << 12)) >> 9); | |||
1713 | } else { | |||
1714 | #else | |||
1715 | { | |||
1716 | #endif /* UVM_SWAP_ENCRYPT */ | |||
1717 | memcpy(dst, src, PAGE_SIZE)__builtin_memcpy((dst), (src), ((1 << 12))); | |||
1718 | } | |||
1719 | /* this just tells async callbacks to free */ | |||
1720 | atomic_setbits_intx86_atomic_setbits_u32(&tpps[i]->pg_flags, PQ_ENCRYPT0x00400000); | |||
1721 | src += PAGE_SIZE(1 << 12); | |||
1722 | dst += PAGE_SIZE(1 << 12); | |||
1723 | } | |||
1724 | ||||
1725 | uvm_pagermapout(kva, npages); | |||
1726 | ||||
1727 | /* dispose of pages we dont use anymore */ | |||
1728 | opages = npages; | |||
1729 | uvm_pager_dropcluster(NULL((void *)0), NULL((void *)0), pps, &opages, | |||
1730 | PGO_PDFREECLUST0x080); | |||
1731 | ||||
1732 | kva = bouncekva; | |||
1733 | } | |||
1734 | ||||
1735 | /* | |||
1736 | * now allocate a buf for the i/o. | |||
1737 | * [make sure we don't put the pagedaemon to sleep...] | |||
1738 | */ | |||
1739 | pflag = (async
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc == uvm.pagedaemon_proc) ? PR_NOWAIT0x0002 : | |||
1740 | PR_WAITOK0x0001; | |||
1741 | bp = pool_get(&bufpool, pflag | PR_ZERO0x0008); | |||
1742 | ||||
1743 | /* | |||
1744 | * if we failed to get a swapbuf, return "try again" | |||
1745 | */ | |||
1746 | if (bp == NULL((void *)0)) { | |||
1747 | if (write && bounce) { | |||
1748 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1749 | int i; | |||
1750 | ||||
1751 | /* swap encrypt needs cleanup */ | |||
1752 | if (encrypt) | |||
1753 | for (i = 0; i < npages; i++) | |||
1754 | SWAP_KEY_PUT(sdp, SWD_KEY(sdp,do { (&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]))->refcount--; if ((&((sdp)->swd_keys [((startslot + i) - (sdp)->swd_drumoffset) >> 7]))-> refcount == 0) { swap_key_delete(&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset) >> 7])); } } while(0) ; | |||
1755 | startslot + i))do { (&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]))->refcount--; if ((&((sdp)->swd_keys [((startslot + i) - (sdp)->swd_drumoffset) >> 7]))-> refcount == 0) { swap_key_delete(&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset) >> 7])); } } while(0) ;; | |||
1756 | #endif | |||
1757 | ||||
1758 | uvm_pagermapout(kva, npages); | |||
1759 | uvm_swap_freepages(tpps, npages); | |||
1760 | } | |||
1761 | return (VM_PAGER_AGAIN5); | |||
1762 | } | |||
1763 | ||||
1764 | /* | |||
1765 | * prevent ASYNC reads. | |||
1766 | * uvm_swap_io is only called from uvm_swap_get, uvm_swap_get | |||
1767 | * assumes that all gets are SYNCIO. Just make sure here. | |||
1768 | * XXXARTUBC - might not be true anymore. | |||
1769 | */ | |||
1770 | if (!write
| |||
1771 | flags &= ~B_ASYNC0x00000004; | |||
1772 | async = 0; | |||
1773 | } | |||
1774 | ||||
1775 | /* | |||
1776 | * fill in the bp. we currently route our i/o through | |||
1777 | * /dev/drum's vnode [swapdev_vp]. | |||
1778 | */ | |||
1779 | bp->b_flags = B_BUSY0x00000010 | B_NOCACHE0x00001000 | B_RAW0x00004000 | (flags & (B_READ0x00008000|B_ASYNC0x00000004)); | |||
1780 | bp->b_proc = &proc0; /* XXX */ | |||
1781 | bp->b_vnbufs.le_next = NOLIST((struct buf *)0x87654321); | |||
1782 | if (bounce
| |||
1783 | bp->b_data = (caddr_t)bouncekva; | |||
1784 | else | |||
1785 | bp->b_data = (caddr_t)kva; | |||
1786 | bp->b_bq = NULL((void *)0); | |||
1787 | bp->b_blkno = startblk; | |||
1788 | LIST_INIT(&bp->b_dep)do { ((&bp->b_dep)->lh_first) = ((void *)0); } while (0); | |||
1789 | s = splbio()splraise(0x6); | |||
1790 | bp->b_vp = NULL((void *)0); | |||
1791 | buf_replacevnode(bp, swapdev_vp); | |||
1792 | splx(s)spllower(s); | |||
1793 | bp->b_bufsize = bp->b_bcount = (long)npages << PAGE_SHIFT12; | |||
1794 | ||||
1795 | /* | |||
1796 | * for pageouts we must set "dirtyoff" [NFS client code needs it]. | |||
1797 | * and we bump v_numoutput (counter of number of active outputs). | |||
1798 | */ | |||
1799 | if (write
| |||
1800 | bp->b_dirtyoff = 0; | |||
1801 | bp->b_dirtyend = npages << PAGE_SHIFT12; | |||
1802 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1803 | /* mark the pages in the drum for decryption */ | |||
1804 | if (swap_encrypt_initialized) | |||
1805 | uvm_swap_markdecrypt(sdp, startslot, npages, encrypt); | |||
1806 | #endif | |||
1807 | s = splbio()splraise(0x6); | |||
1808 | swapdev_vp->v_numoutput++; | |||
1809 | splx(s)spllower(s); | |||
1810 | } | |||
1811 | ||||
1812 | /* for async ops we must set up the iodone handler. */ | |||
1813 | if (async
| |||
1814 | bp->b_flags |= B_CALL0x00000040 | (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc == uvm.pagedaemon_proc ? | |||
1815 | B_PDAEMON0x00200000 : 0); | |||
1816 | bp->b_iodone = uvm_aio_biodone; | |||
1817 | } | |||
1818 | ||||
1819 | /* now we start the I/O, and if async, return. */ | |||
1820 | VOP_STRATEGY(bp->b_vp, bp); | |||
1821 | if (async
| |||
1822 | return (VM_PAGER_PEND3); | |||
1823 | ||||
1824 | /* must be sync i/o. wait for it to finish */ | |||
1825 | (void) biowait(bp); | |||
1826 | result = (bp->b_flags & B_ERROR0x00000400) ? VM_PAGER_ERROR4 : VM_PAGER_OK0; | |||
1827 | ||||
1828 | /* decrypt swap */ | |||
1829 | if (!write
| |||
1830 | int i; | |||
1831 | caddr_t data = (caddr_t)kva; | |||
1832 | caddr_t dst = (caddr_t)kva; | |||
1833 | u_int64_t block = startblk; | |||
1834 | ||||
1835 | if (bounce
| |||
1836 | data = (caddr_t)bouncekva; | |||
1837 | ||||
1838 | for (i = 0; i < npages; i++) { | |||
1839 | #ifdef UVM_SWAP_ENCRYPT1 | |||
1840 | struct swap_key *key; | |||
1841 | ||||
1842 | /* Check if we need to decrypt */ | |||
1843 | if (swap_encrypt_initialized && | |||
1844 | uvm_swap_needdecrypt(sdp, startslot + i)) { | |||
| ||||
1845 | key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]); | |||
1846 | if (key->refcount == 0) { | |||
1847 | result = VM_PAGER_ERROR4; | |||
1848 | break; | |||
1849 | } | |||
1850 | swap_decrypt(key, data, dst, block, PAGE_SIZE(1 << 12)); | |||
1851 | } else if (bounce) { | |||
1852 | #else | |||
1853 | if (bounce) { | |||
1854 | #endif | |||
1855 | memcpy(dst, data, PAGE_SIZE)__builtin_memcpy((dst), (data), ((1 << 12))); | |||
1856 | } | |||
1857 | data += PAGE_SIZE(1 << 12); | |||
1858 | dst += PAGE_SIZE(1 << 12); | |||
1859 | block += btodb(PAGE_SIZE)(((1 << 12)) >> 9); | |||
1860 | } | |||
1861 | if (bounce) | |||
1862 | uvm_pagermapout(bouncekva, npages); | |||
1863 | } | |||
1864 | /* kill the pager mapping */ | |||
1865 | uvm_pagermapout(kva, npages); | |||
1866 | ||||
1867 | /* Not anymore needed, free after encryption/bouncing */ | |||
1868 | if (!write && bounce) | |||
1869 | uvm_swap_freepages(tpps, npages); | |||
1870 | ||||
1871 | /* now dispose of the buf */ | |||
1872 | s = splbio()splraise(0x6); | |||
1873 | if (bp->b_vp) | |||
1874 | brelvp(bp); | |||
1875 | ||||
1876 | if (write && bp->b_vp) | |||
1877 | vwakeup(bp->b_vp); | |||
1878 | pool_put(&bufpool, bp); | |||
1879 | splx(s)spllower(s); | |||
1880 | ||||
1881 | /* finally return. */ | |||
1882 | return (result); | |||
1883 | } | |||
1884 | ||||
1885 | void | |||
1886 | swapmount(void) | |||
1887 | { | |||
1888 | struct swapdev *sdp; | |||
1889 | struct swappri *spp; | |||
1890 | struct vnode *vp; | |||
1891 | dev_t swap_dev = swdevt[0].sw_dev; | |||
1892 | char *nam; | |||
1893 | char path[MNAMELEN90 + 1]; | |||
1894 | ||||
1895 | /* | |||
1896 | * No locking here since we happen to know that we will just be called | |||
1897 | * once before any other process has forked. | |||
1898 | */ | |||
1899 | if (swap_dev == NODEV(dev_t)(-1)) | |||
1900 | return; | |||
1901 | ||||
1902 | #if defined(NFSCLIENT1) | |||
1903 | if (swap_dev == NETDEV(dev_t)(-2)) { | |||
1904 | extern struct nfs_diskless nfs_diskless; | |||
1905 | ||||
1906 | snprintf(path, sizeof(path), "%s", | |||
1907 | nfs_diskless.nd_swap.ndm_host); | |||
1908 | vp = nfs_diskless.sw_vp; | |||
1909 | goto gotit; | |||
1910 | } else | |||
1911 | #endif | |||
1912 | if (bdevvp(swap_dev, &vp)) | |||
1913 | return; | |||
1914 | ||||
1915 | /* Construct a potential path to swap */ | |||
1916 | if ((nam = findblkname(major(swap_dev)(((unsigned)(swap_dev) >> 8) & 0xff)))) | |||
1917 | snprintf(path, sizeof(path), "/dev/%s%d%c", nam, | |||
1918 | DISKUNIT(swap_dev)(((unsigned)((swap_dev) & 0xff) | (((swap_dev) & 0xffff0000 ) >> 8)) / 16), 'a' + DISKPART(swap_dev)(((unsigned)((swap_dev) & 0xff) | (((swap_dev) & 0xffff0000 ) >> 8)) % 16)); | |||
1919 | else | |||
1920 | snprintf(path, sizeof(path), "blkdev0x%x", | |||
1921 | swap_dev); | |||
1922 | ||||
1923 | #if defined(NFSCLIENT1) | |||
1924 | gotit: | |||
1925 | #endif | |||
1926 | sdp = malloc(sizeof(*sdp), M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008); | |||
1927 | spp = malloc(sizeof(*spp), M_VMSWAP92, M_WAITOK0x0001); | |||
1928 | ||||
1929 | sdp->swd_flagsswd_se.se_flags = SWF_FAKE0x00000008; | |||
1930 | sdp->swd_devswd_se.se_dev = swap_dev; | |||
1931 | ||||
1932 | sdp->swd_pathlen = strlen(path) + 1; | |||
1933 | sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP92, M_WAITOK0x0001 | M_ZERO0x0008); | |||
1934 | strlcpy(sdp->swd_path, path, sdp->swd_pathlen); | |||
1935 | ||||
1936 | sdp->swd_vp = vp; | |||
1937 | ||||
1938 | swaplist_insert(sdp, spp, 0); | |||
1939 | ||||
1940 | if (swap_on(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc, sdp)) { | |||
1941 | swaplist_find(vp, 1); | |||
1942 | swaplist_trim(); | |||
1943 | vput(sdp->swd_vp); | |||
1944 | free(sdp->swd_path, M_VMSWAP92, sdp->swd_pathlen); | |||
1945 | free(sdp, M_VMSWAP92, sizeof(*sdp)); | |||
1946 | return; | |||
1947 | } | |||
1948 | } | |||
1949 | ||||
1950 | #ifdef HIBERNATE1 | |||
1951 | int | |||
1952 | uvm_hibswap(dev_t dev, u_long *sp, u_long *ep) | |||
1953 | { | |||
1954 | struct swapdev *sdp, *swd = NULL((void *)0); | |||
1955 | struct swappri *spp; | |||
1956 | struct extent_region *exr, *exrn; | |||
1957 | u_long start = 0, end = 0, size = 0; | |||
1958 | ||||
1959 | /* no swap devices configured yet? */ | |||
1960 | if (uvmexp.nswapdev < 1 || dev != swdevt[0].sw_dev) | |||
1961 | return (1); | |||
1962 | ||||
1963 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { | |||
1964 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { | |||
1965 | if (sdp->swd_devswd_se.se_dev == dev) | |||
1966 | swd = sdp; | |||
1967 | } | |||
1968 | } | |||
1969 | ||||
1970 | if (swd == NULL((void *)0) || (swd->swd_flagsswd_se.se_flags & SWF_ENABLE0x00000002) == 0) | |||
1971 | return (1); | |||
1972 | ||||
1973 | LIST_FOREACH(exr, &swd->swd_ex->ex_regions, er_link)for((exr) = ((&swd->swd_ex->ex_regions)->lh_first ); (exr)!= ((void *)0); (exr) = ((exr)->er_link.le_next)) { | |||
1974 | u_long gapstart, gapend, gapsize; | |||
1975 | ||||
1976 | gapstart = exr->er_end + 1; | |||
1977 | exrn = LIST_NEXT(exr, er_link)((exr)->er_link.le_next); | |||
1978 | if (!exrn) | |||
1979 | break; | |||
1980 | gapend = exrn->er_start - 1; | |||
1981 | gapsize = gapend - gapstart; | |||
1982 | if (gapsize > size) { | |||
1983 | start = gapstart; | |||
1984 | end = gapend; | |||
1985 | size = gapsize; | |||
1986 | } | |||
1987 | } | |||
1988 | ||||
1989 | if (size) { | |||
1990 | *sp = start; | |||
1991 | *ep = end; | |||
1992 | return (0); | |||
1993 | } | |||
1994 | return (1); | |||
1995 | } | |||
1996 | #endif /* HIBERNATE */ |