File: | uvm/uvm_swap.c |
Warning: | line 1140, column 2 Value stored to 'error' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* $OpenBSD: uvm_swap.c,v 1.152 2021/12/12 09:14:59 visa Exp $ */ |
2 | /* $NetBSD: uvm_swap.c,v 1.40 2000/11/17 11:39:39 mrg Exp $ */ |
3 | |
4 | /* |
5 | * Copyright (c) 1995, 1996, 1997 Matthew R. Green |
6 | * All rights reserved. |
7 | * |
8 | * Redistribution and use in source and binary forms, with or without |
9 | * modification, are permitted provided that the following conditions |
10 | * are met: |
11 | * 1. Redistributions of source code must retain the above copyright |
12 | * notice, this list of conditions and the following disclaimer. |
13 | * 2. Redistributions in binary form must reproduce the above copyright |
14 | * notice, this list of conditions and the following disclaimer in the |
15 | * documentation and/or other materials provided with the distribution. |
16 | * |
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, |
22 | * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
23 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
24 | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
25 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
26 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
27 | * SUCH DAMAGE. |
28 | * |
29 | * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp |
30 | * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp |
31 | */ |
32 | |
33 | #include <sys/param.h> |
34 | #include <sys/systm.h> |
35 | #include <sys/buf.h> |
36 | #include <sys/conf.h> |
37 | #include <sys/proc.h> |
38 | #include <sys/namei.h> |
39 | #include <sys/disklabel.h> |
40 | #include <sys/errno.h> |
41 | #include <sys/kernel.h> |
42 | #include <sys/malloc.h> |
43 | #include <sys/vnode.h> |
44 | #include <sys/fcntl.h> |
45 | #include <sys/extent.h> |
46 | #include <sys/mount.h> |
47 | #include <sys/pool.h> |
48 | #include <sys/syscallargs.h> |
49 | #include <sys/swap.h> |
50 | #include <sys/disk.h> |
51 | #include <sys/task.h> |
52 | #include <sys/pledge.h> |
53 | #if defined(NFSCLIENT1) |
54 | #include <sys/socket.h> |
55 | #include <sys/domain.h> |
56 | #include <netinet/in.h> |
57 | #include <nfs/nfsproto.h> |
58 | #include <nfs/nfsdiskless.h> |
59 | #endif |
60 | |
61 | #include <uvm/uvm.h> |
62 | #ifdef UVM_SWAP_ENCRYPT1 |
63 | #include <uvm/uvm_swap_encrypt.h> |
64 | #endif |
65 | |
66 | #include <sys/specdev.h> |
67 | |
68 | #include "vnd.h" |
69 | |
70 | /* |
71 | * uvm_swap.c: manage configuration and i/o to swap space. |
72 | */ |
73 | |
74 | /* |
75 | * swap space is managed in the following way: |
76 | * |
77 | * each swap partition or file is described by a "swapdev" structure. |
78 | * each "swapdev" structure contains a "swapent" structure which contains |
79 | * information that is passed up to the user (via system calls). |
80 | * |
81 | * each swap partition is assigned a "priority" (int) which controls |
82 | * swap partition usage. |
83 | * |
84 | * the system maintains a global data structure describing all swap |
85 | * partitions/files. there is a sorted LIST of "swappri" structures |
86 | * which describe "swapdev"'s at that priority. this LIST is headed |
87 | * by the "swap_priority" global var. each "swappri" contains a |
88 | * TAILQ of "swapdev" structures at that priority. |
89 | * |
90 | * locking: |
91 | * - swap_syscall_lock (sleep lock): this lock serializes the swapctl |
92 | * system call and prevents the swap priority list from changing |
93 | * while we are in the middle of a system call (e.g. SWAP_STATS). |
94 | * |
95 | * each swap device has the following info: |
96 | * - swap device in use (could be disabled, preventing future use) |
97 | * - swap enabled (allows new allocations on swap) |
98 | * - map info in /dev/drum |
99 | * - vnode pointer |
100 | * for swap files only: |
101 | * - block size |
102 | * - max byte count in buffer |
103 | * - buffer |
104 | * - credentials to use when doing i/o to file |
105 | * |
106 | * userland controls and configures swap with the swapctl(2) system call. |
107 | * the sys_swapctl performs the following operations: |
108 | * [1] SWAP_NSWAP: returns the number of swap devices currently configured |
109 | * [2] SWAP_STATS: given a pointer to an array of swapent structures |
110 | * (passed in via "arg") of a size passed in via "misc" ... we load |
111 | * the current swap config into the array. |
112 | * [3] SWAP_ON: given a pathname in arg (could be device or file) and a |
113 | * priority in "misc", start swapping on it. |
114 | * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device |
115 | * [5] SWAP_CTL: changes the priority of a swap device (new priority in |
116 | * "misc") |
117 | */ |
118 | |
119 | /* |
120 | * swapdev: describes a single swap partition/file |
121 | * |
122 | * note the following should be true: |
123 | * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] |
124 | * swd_nblks <= swd_mapsize [because mapsize includes disklabel] |
125 | */ |
126 | struct swapdev { |
127 | struct swapent swd_se; |
128 | #define swd_devswd_se.se_dev swd_se.se_dev /* device id */ |
129 | #define swd_flagsswd_se.se_flags swd_se.se_flags /* flags:inuse/enable/fake */ |
130 | #define swd_priorityswd_se.se_priority swd_se.se_priority /* our priority */ |
131 | #define swd_inuseswd_se.se_inuse swd_se.se_inuse /* blocks used */ |
132 | #define swd_nblksswd_se.se_nblks swd_se.se_nblks /* total blocks */ |
133 | char *swd_path; /* saved pathname of device */ |
134 | int swd_pathlen; /* length of pathname */ |
135 | int swd_npages; /* #pages we can use */ |
136 | int swd_npginuse; /* #pages in use */ |
137 | int swd_npgbad; /* #pages bad */ |
138 | int swd_drumoffset; /* page0 offset in drum */ |
139 | int swd_drumsize; /* #pages in drum */ |
140 | struct extent *swd_ex; /* extent for this swapdev */ |
141 | char swd_exname[12]; /* name of extent above */ |
142 | struct vnode *swd_vp; /* backing vnode */ |
143 | TAILQ_ENTRY(swapdev)struct { struct swapdev *tqe_next; struct swapdev **tqe_prev; } swd_next; /* priority tailq */ |
144 | |
145 | int swd_bsize; /* blocksize (bytes) */ |
146 | int swd_maxactive; /* max active i/o reqs */ |
147 | int swd_active; /* # of active i/o reqs */ |
148 | struct bufq swd_bufq; |
149 | struct ucred *swd_cred; /* cred for file access */ |
150 | #ifdef UVM_SWAP_ENCRYPT1 |
151 | #define SWD_KEY_SHIFT7 7 /* One key per 0.5 MByte */ |
152 | #define SWD_KEY(x,y)&((x)->swd_keys[((y) - (x)->swd_drumoffset) >> 7]) &((x)->swd_keys[((y) - (x)->swd_drumoffset) >> SWD_KEY_SHIFT7]) |
153 | #define SWD_KEY_SIZE(x)(((x) + (1 << 7) - 1) >> 7) (((x) + (1 << SWD_KEY_SHIFT7) - 1) >> SWD_KEY_SHIFT7) |
154 | |
155 | #define SWD_DCRYPT_SHIFT5 5 |
156 | #define SWD_DCRYPT_BITS32 32 |
157 | #define SWD_DCRYPT_MASK(32 - 1) (SWD_DCRYPT_BITS32 - 1) |
158 | #define SWD_DCRYPT_OFF(x)((x) >> 5) ((x) >> SWD_DCRYPT_SHIFT5) |
159 | #define SWD_DCRYPT_BIT(x)((x) & (32 - 1)) ((x) & SWD_DCRYPT_MASK(32 - 1)) |
160 | #define SWD_DCRYPT_SIZE(x)((((x) + (32 - 1)) >> 5) * sizeof(u_int32_t)) (SWD_DCRYPT_OFF((x) + SWD_DCRYPT_MASK)(((x) + (32 - 1)) >> 5) * sizeof(u_int32_t)) |
161 | u_int32_t *swd_decrypt; /* bitmap for decryption */ |
162 | struct swap_key *swd_keys; /* keys for different parts */ |
163 | #endif |
164 | }; |
165 | |
166 | /* |
167 | * swap device priority entry; the list is kept sorted on `spi_priority'. |
168 | */ |
169 | struct swappri { |
170 | int spi_priority; /* priority */ |
171 | TAILQ_HEAD(spi_swapdev, swapdev)struct spi_swapdev { struct swapdev *tqh_first; struct swapdev **tqh_last; } spi_swapdev; |
172 | /* tailq of swapdevs at this priority */ |
173 | LIST_ENTRY(swappri)struct { struct swappri *le_next; struct swappri **le_prev; } spi_swappri; /* global list of pri's */ |
174 | }; |
175 | |
176 | /* |
177 | * The following two structures are used to keep track of data transfers |
178 | * on swap devices associated with regular files. |
179 | * NOTE: this code is more or less a copy of vnd.c; we use the same |
180 | * structure names here to ease porting.. |
181 | */ |
182 | struct vndxfer { |
183 | struct buf *vx_bp; /* Pointer to parent buffer */ |
184 | struct swapdev *vx_sdp; |
185 | int vx_error; |
186 | int vx_pending; /* # of pending aux buffers */ |
187 | int vx_flags; |
188 | #define VX_BUSY1 1 |
189 | #define VX_DEAD2 2 |
190 | }; |
191 | |
192 | struct vndbuf { |
193 | struct buf vb_buf; |
194 | struct vndxfer *vb_vnx; |
195 | struct task vb_task; |
196 | }; |
197 | |
198 | /* |
199 | * We keep a of pool vndbuf's and vndxfer structures. |
200 | */ |
201 | struct pool vndxfer_pool; |
202 | struct pool vndbuf_pool; |
203 | |
204 | |
205 | /* |
206 | * local variables |
207 | */ |
208 | struct extent *swapmap; /* controls the mapping of /dev/drum */ |
209 | |
210 | /* list of all active swap devices [by priority] */ |
211 | LIST_HEAD(swap_priority, swappri)struct swap_priority { struct swappri *lh_first; }; |
212 | struct swap_priority swap_priority; |
213 | |
214 | /* locks */ |
215 | struct rwlock swap_syscall_lock = RWLOCK_INITIALIZER("swplk"){ 0, "swplk" }; |
216 | |
217 | /* |
218 | * prototypes |
219 | */ |
220 | void swapdrum_add(struct swapdev *, int); |
221 | struct swapdev *swapdrum_getsdp(int); |
222 | |
223 | struct swapdev *swaplist_find(struct vnode *, int); |
224 | void swaplist_insert(struct swapdev *, |
225 | struct swappri *, int); |
226 | void swaplist_trim(void); |
227 | |
228 | int swap_on(struct proc *, struct swapdev *); |
229 | int swap_off(struct proc *, struct swapdev *); |
230 | |
231 | void sw_reg_strategy(struct swapdev *, struct buf *, int); |
232 | void sw_reg_iodone(struct buf *); |
233 | void sw_reg_iodone_internal(void *); |
234 | void sw_reg_start(struct swapdev *); |
235 | |
236 | int uvm_swap_io(struct vm_page **, int, int, int); |
237 | |
238 | void swapmount(void); |
239 | boolean_t uvm_swap_allocpages(struct vm_page **, int); |
240 | |
241 | #ifdef UVM_SWAP_ENCRYPT1 |
242 | /* for swap encrypt */ |
243 | void uvm_swap_markdecrypt(struct swapdev *, int, int, int); |
244 | boolean_t uvm_swap_needdecrypt(struct swapdev *, int); |
245 | void uvm_swap_initcrypt(struct swapdev *, int); |
246 | #endif |
247 | |
248 | /* |
249 | * uvm_swap_init: init the swap system data structures and locks |
250 | * |
251 | * => called at boot time from init_main.c after the filesystems |
252 | * are brought up (which happens after uvm_init()) |
253 | */ |
254 | void |
255 | uvm_swap_init(void) |
256 | { |
257 | /* |
258 | * first, init the swap list, its counter, and its lock. |
259 | * then get a handle on the vnode for /dev/drum by using |
260 | * the its dev_t number ("swapdev", from MD conf.c). |
261 | */ |
262 | LIST_INIT(&swap_priority)do { ((&swap_priority)->lh_first) = ((void *)0); } while (0); |
263 | uvmexp.nswapdev = 0; |
264 | |
265 | if (!swapdev_vp && bdevvp(swapdev, &swapdev_vp)) |
266 | panic("uvm_swap_init: can't get vnode for swap device"); |
267 | |
268 | /* |
269 | * create swap block extent to map /dev/drum. The extent spans |
270 | * 1 to INT_MAX allows 2 gigablocks of swap space. Note that |
271 | * block 0 is reserved (used to indicate an allocation failure, |
272 | * or no allocation). |
273 | */ |
274 | swapmap = extent_create("swapmap", 1, INT_MAX0x7fffffff, |
275 | M_VMSWAP92, 0, 0, EX_NOWAIT0x0000); |
276 | if (swapmap == 0) |
277 | panic("uvm_swap_init: extent_create failed"); |
278 | |
279 | /* allocate pools for structures used for swapping to files. */ |
280 | pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, IPL_BIO0x6, 0, |
281 | "swp vnx", NULL((void *)0)); |
282 | pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, IPL_BIO0x6, 0, |
283 | "swp vnd", NULL((void *)0)); |
284 | |
285 | /* Setup the initial swap partition */ |
286 | swapmount(); |
287 | } |
288 | |
289 | #ifdef UVM_SWAP_ENCRYPT1 |
290 | void |
291 | uvm_swap_initcrypt_all(void) |
292 | { |
293 | struct swapdev *sdp; |
294 | struct swappri *spp; |
295 | int npages; |
296 | |
297 | |
298 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
299 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
300 | if (sdp->swd_decrypt == NULL((void *)0)) { |
301 | npages = dbtob((uint64_t)sdp->swd_nblks)(((uint64_t)sdp->swd_se.se_nblks) << 9) >> |
302 | PAGE_SHIFT12; |
303 | uvm_swap_initcrypt(sdp, npages); |
304 | } |
305 | } |
306 | } |
307 | } |
308 | |
309 | void |
310 | uvm_swap_initcrypt(struct swapdev *sdp, int npages) |
311 | { |
312 | /* |
313 | * keep information if a page needs to be decrypted when we get it |
314 | * from the swap device. |
315 | * We cannot chance a malloc later, if we are doing ASYNC puts, |
316 | * we may not call malloc with M_WAITOK. This consumes only |
317 | * 8KB memory for a 256MB swap partition. |
318 | */ |
319 | sdp->swd_decrypt = malloc(SWD_DCRYPT_SIZE(npages)((((npages) + (32 - 1)) >> 5) * sizeof(u_int32_t)), M_VMSWAP92, |
320 | M_WAITOK0x0001|M_ZERO0x0008); |
321 | sdp->swd_keys = mallocarray(SWD_KEY_SIZE(npages)(((npages) + (1 << 7) - 1) >> 7), |
322 | sizeof(struct swap_key), M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008); |
323 | } |
324 | |
325 | #endif /* UVM_SWAP_ENCRYPT */ |
326 | |
327 | boolean_t |
328 | uvm_swap_allocpages(struct vm_page **pps, int npages) |
329 | { |
330 | struct pglist pgl; |
331 | int i; |
332 | boolean_t fail; |
333 | |
334 | /* Estimate if we will succeed */ |
335 | uvm_lock_fpageq()mtx_enter(&uvm.fpageqlock); |
336 | |
337 | fail = uvmexp.free - npages < uvmexp.reserve_kernel; |
338 | |
339 | uvm_unlock_fpageq()mtx_leave(&uvm.fpageqlock); |
340 | |
341 | if (fail) |
342 | return FALSE0; |
343 | |
344 | TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last = &(&pgl)->tqh_first; } while (0); |
345 | if (uvm_pglistalloc(npages * PAGE_SIZE(1 << 12), dma_constraint.ucr_low, |
346 | dma_constraint.ucr_high, 0, 0, &pgl, npages, UVM_PLA_NOWAIT0x0002)) |
347 | return FALSE0; |
348 | |
349 | for (i = 0; i < npages; i++) { |
350 | pps[i] = TAILQ_FIRST(&pgl)((&pgl)->tqh_first); |
351 | /* *sigh* */ |
352 | atomic_setbits_intx86_atomic_setbits_u32(&pps[i]->pg_flags, PG_BUSY0x00000001); |
353 | TAILQ_REMOVE(&pgl, pps[i], pageq)do { if (((pps[i])->pageq.tqe_next) != ((void *)0)) (pps[i ])->pageq.tqe_next->pageq.tqe_prev = (pps[i])->pageq .tqe_prev; else (&pgl)->tqh_last = (pps[i])->pageq. tqe_prev; *(pps[i])->pageq.tqe_prev = (pps[i])->pageq.tqe_next ; ((pps[i])->pageq.tqe_prev) = ((void *)-1); ((pps[i])-> pageq.tqe_next) = ((void *)-1); } while (0); |
354 | } |
355 | |
356 | return TRUE1; |
357 | } |
358 | |
359 | void |
360 | uvm_swap_freepages(struct vm_page **pps, int npages) |
361 | { |
362 | int i; |
363 | |
364 | uvm_lock_pageq()mtx_enter(&uvm.pageqlock); |
365 | for (i = 0; i < npages; i++) |
366 | uvm_pagefree(pps[i]); |
367 | uvm_unlock_pageq()mtx_leave(&uvm.pageqlock); |
368 | } |
369 | |
370 | #ifdef UVM_SWAP_ENCRYPT1 |
371 | /* |
372 | * Mark pages on the swap device for later decryption |
373 | */ |
374 | |
375 | void |
376 | uvm_swap_markdecrypt(struct swapdev *sdp, int startslot, int npages, |
377 | int decrypt) |
378 | { |
379 | int pagestart, i; |
380 | int off, bit; |
381 | |
382 | if (!sdp) |
383 | return; |
384 | |
385 | pagestart = startslot - sdp->swd_drumoffset; |
386 | for (i = 0; i < npages; i++, pagestart++) { |
387 | off = SWD_DCRYPT_OFF(pagestart)((pagestart) >> 5); |
388 | bit = SWD_DCRYPT_BIT(pagestart)((pagestart) & (32 - 1)); |
389 | if (decrypt) |
390 | /* pages read need decryption */ |
391 | sdp->swd_decrypt[off] |= 1 << bit; |
392 | else |
393 | /* pages read do not need decryption */ |
394 | sdp->swd_decrypt[off] &= ~(1 << bit); |
395 | } |
396 | } |
397 | |
398 | /* |
399 | * Check if the page that we got from disk needs to be decrypted |
400 | */ |
401 | |
402 | boolean_t |
403 | uvm_swap_needdecrypt(struct swapdev *sdp, int off) |
404 | { |
405 | if (!sdp) |
406 | return FALSE0; |
407 | |
408 | off -= sdp->swd_drumoffset; |
409 | return sdp->swd_decrypt[SWD_DCRYPT_OFF(off)((off) >> 5)] & (1 << SWD_DCRYPT_BIT(off)((off) & (32 - 1))) ? |
410 | TRUE1 : FALSE0; |
411 | } |
412 | |
413 | void |
414 | uvm_swap_finicrypt_all(void) |
415 | { |
416 | struct swapdev *sdp; |
417 | struct swappri *spp; |
418 | struct swap_key *key; |
419 | unsigned int nkeys; |
420 | |
421 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
422 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
423 | if (sdp->swd_decrypt == NULL((void *)0)) |
424 | continue; |
425 | |
426 | nkeys = dbtob((uint64_t)sdp->swd_nblks)(((uint64_t)sdp->swd_se.se_nblks) << 9) >> PAGE_SHIFT12; |
427 | key = sdp->swd_keys + (SWD_KEY_SIZE(nkeys)(((nkeys) + (1 << 7) - 1) >> 7) - 1); |
428 | do { |
429 | if (key->refcount != 0) |
430 | swap_key_delete(key); |
431 | } while (key-- != sdp->swd_keys); |
432 | } |
433 | } |
434 | } |
435 | #endif /* UVM_SWAP_ENCRYPT */ |
436 | |
437 | /* |
438 | * swaplist functions: functions that operate on the list of swap |
439 | * devices on the system. |
440 | */ |
441 | |
442 | /* |
443 | * swaplist_insert: insert swap device "sdp" into the global list |
444 | * |
445 | * => caller must hold both swap_syscall_lock and uvm.swap_data_lock |
446 | * => caller must provide a newly malloc'd swappri structure (we will |
447 | * FREE it if we don't need it... this it to prevent malloc blocking |
448 | * here while adding swap) |
449 | */ |
450 | void |
451 | swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) |
452 | { |
453 | struct swappri *spp, *pspp; |
454 | |
455 | /* |
456 | * find entry at or after which to insert the new device. |
457 | */ |
458 | pspp = NULL((void *)0); |
459 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
460 | if (priority <= spp->spi_priority) |
461 | break; |
462 | pspp = spp; |
463 | } |
464 | |
465 | /* |
466 | * new priority? |
467 | */ |
468 | if (spp == NULL((void *)0) || spp->spi_priority != priority) { |
469 | spp = newspp; /* use newspp! */ |
470 | |
471 | spp->spi_priority = priority; |
472 | TAILQ_INIT(&spp->spi_swapdev)do { (&spp->spi_swapdev)->tqh_first = ((void *)0); ( &spp->spi_swapdev)->tqh_last = &(&spp->spi_swapdev )->tqh_first; } while (0); |
473 | |
474 | if (pspp) |
475 | LIST_INSERT_AFTER(pspp, spp, spi_swappri)do { if (((spp)->spi_swappri.le_next = (pspp)->spi_swappri .le_next) != ((void *)0)) (pspp)->spi_swappri.le_next-> spi_swappri.le_prev = &(spp)->spi_swappri.le_next; (pspp )->spi_swappri.le_next = (spp); (spp)->spi_swappri.le_prev = &(pspp)->spi_swappri.le_next; } while (0); |
476 | else |
477 | LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri)do { if (((spp)->spi_swappri.le_next = (&swap_priority )->lh_first) != ((void *)0)) (&swap_priority)->lh_first ->spi_swappri.le_prev = &(spp)->spi_swappri.le_next ; (&swap_priority)->lh_first = (spp); (spp)->spi_swappri .le_prev = &(&swap_priority)->lh_first; } while (0 ); |
478 | } else { |
479 | /* we don't need a new priority structure, free it */ |
480 | free(newspp, M_VMSWAP92, sizeof(*newspp)); |
481 | } |
482 | |
483 | /* |
484 | * priority found (or created). now insert on the priority's |
485 | * tailq list and bump the total number of swapdevs. |
486 | */ |
487 | sdp->swd_priorityswd_se.se_priority = priority; |
488 | TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next)do { (sdp)->swd_next.tqe_next = ((void *)0); (sdp)->swd_next .tqe_prev = (&spp->spi_swapdev)->tqh_last; *(&spp ->spi_swapdev)->tqh_last = (sdp); (&spp->spi_swapdev )->tqh_last = &(sdp)->swd_next.tqe_next; } while (0 ); |
489 | uvmexp.nswapdev++; |
490 | } |
491 | |
492 | /* |
493 | * swaplist_find: find and optionally remove a swap device from the |
494 | * global list. |
495 | * |
496 | * => caller must hold both swap_syscall_lock and uvm.swap_data_lock |
497 | * => we return the swapdev we found (and removed) |
498 | */ |
499 | struct swapdev * |
500 | swaplist_find(struct vnode *vp, boolean_t remove) |
501 | { |
502 | struct swapdev *sdp; |
503 | struct swappri *spp; |
504 | |
505 | /* |
506 | * search the lists for the requested vp |
507 | */ |
508 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
509 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
510 | if (sdp->swd_vp != vp) |
511 | continue; |
512 | if (remove) { |
513 | TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next)do { if (((sdp)->swd_next.tqe_next) != ((void *)0)) (sdp)-> swd_next.tqe_next->swd_next.tqe_prev = (sdp)->swd_next. tqe_prev; else (&spp->spi_swapdev)->tqh_last = (sdp )->swd_next.tqe_prev; *(sdp)->swd_next.tqe_prev = (sdp) ->swd_next.tqe_next; ((sdp)->swd_next.tqe_prev) = ((void *)-1); ((sdp)->swd_next.tqe_next) = ((void *)-1); } while (0); |
514 | uvmexp.nswapdev--; |
515 | } |
516 | return (sdp); |
517 | } |
518 | } |
519 | return (NULL((void *)0)); |
520 | } |
521 | |
522 | |
523 | /* |
524 | * swaplist_trim: scan priority list for empty priority entries and kill |
525 | * them. |
526 | * |
527 | * => caller must hold both swap_syscall_lock and uvm.swap_data_lock |
528 | */ |
529 | void |
530 | swaplist_trim(void) |
531 | { |
532 | struct swappri *spp, *nextspp; |
533 | |
534 | LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp)for ((spp) = ((&swap_priority)->lh_first); (spp) && ((nextspp) = ((spp)->spi_swappri.le_next), 1); (spp) = (nextspp )) { |
535 | if (!TAILQ_EMPTY(&spp->spi_swapdev)(((&spp->spi_swapdev)->tqh_first) == ((void *)0))) |
536 | continue; |
537 | LIST_REMOVE(spp, spi_swappri)do { if ((spp)->spi_swappri.le_next != ((void *)0)) (spp)-> spi_swappri.le_next->spi_swappri.le_prev = (spp)->spi_swappri .le_prev; *(spp)->spi_swappri.le_prev = (spp)->spi_swappri .le_next; ((spp)->spi_swappri.le_prev) = ((void *)-1); ((spp )->spi_swappri.le_next) = ((void *)-1); } while (0); |
538 | free(spp, M_VMSWAP92, sizeof(*spp)); |
539 | } |
540 | } |
541 | |
542 | /* |
543 | * swapdrum_add: add a "swapdev"'s blocks into /dev/drum's area. |
544 | * |
545 | * => caller must hold swap_syscall_lock |
546 | * => uvm.swap_data_lock should be unlocked (we may sleep) |
547 | */ |
548 | void |
549 | swapdrum_add(struct swapdev *sdp, int npages) |
550 | { |
551 | u_long result; |
552 | |
553 | if (extent_alloc(swapmap, npages, EX_NOALIGN, 0, EX_NOBOUNDARY,extent_alloc_subregion((swapmap), (swapmap)->ex_start, (swapmap )->ex_end, (npages), (1), (0), (0), (0x0001), (&result )) |
554 | EX_WAITOK, &result)extent_alloc_subregion((swapmap), (swapmap)->ex_start, (swapmap )->ex_end, (npages), (1), (0), (0), (0x0001), (&result ))) |
555 | panic("swapdrum_add"); |
556 | |
557 | sdp->swd_drumoffset = result; |
558 | sdp->swd_drumsize = npages; |
559 | } |
560 | |
561 | /* |
562 | * swapdrum_getsdp: given a page offset in /dev/drum, convert it back |
563 | * to the "swapdev" that maps that section of the drum. |
564 | * |
565 | * => each swapdev takes one big contig chunk of the drum |
566 | * => caller must hold uvm.swap_data_lock |
567 | */ |
568 | struct swapdev * |
569 | swapdrum_getsdp(int pgno) |
570 | { |
571 | struct swapdev *sdp; |
572 | struct swappri *spp; |
573 | |
574 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
575 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
576 | if (pgno >= sdp->swd_drumoffset && |
577 | pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { |
578 | return sdp; |
579 | } |
580 | } |
581 | } |
582 | return NULL((void *)0); |
583 | } |
584 | |
585 | |
586 | /* |
587 | * sys_swapctl: main entry point for swapctl(2) system call |
588 | * [with two helper functions: swap_on and swap_off] |
589 | */ |
590 | int |
591 | sys_swapctl(struct proc *p, void *v, register_t *retval) |
592 | { |
593 | struct sys_swapctl_args /* { |
594 | syscallarg(int) cmd; |
595 | syscallarg(void *) arg; |
596 | syscallarg(int) misc; |
597 | } */ *uap = (struct sys_swapctl_args *)v; |
598 | struct vnode *vp; |
599 | struct nameidata nd; |
600 | struct swappri *spp; |
601 | struct swapdev *sdp; |
602 | struct swapent *sep; |
603 | char userpath[MAXPATHLEN1024]; |
604 | size_t len; |
605 | int count, error, misc; |
606 | int priority; |
607 | |
608 | misc = SCARG(uap, misc)((uap)->misc.le.datum); |
609 | |
610 | /* |
611 | * ensure serialized syscall access by grabbing the swap_syscall_lock |
612 | */ |
613 | rw_enter_write(&swap_syscall_lock); |
614 | |
615 | /* |
616 | * we handle the non-priv NSWAP and STATS request first. |
617 | * |
618 | * SWAP_NSWAP: return number of config'd swap devices |
619 | * [can also be obtained with uvmexp sysctl] |
620 | */ |
621 | if (SCARG(uap, cmd)((uap)->cmd.le.datum) == SWAP_NSWAP3) { |
622 | *retval = uvmexp.nswapdev; |
623 | error = 0; |
624 | goto out; |
625 | } |
626 | |
627 | /* |
628 | * SWAP_STATS: get stats on current # of configured swap devs |
629 | * |
630 | * note that the swap_priority list can't change as long |
631 | * as we are holding the swap_syscall_lock. we don't want |
632 | * to grab the uvm.swap_data_lock because we may fault&sleep during |
633 | * copyout() and we don't want to be holding that lock then! |
634 | */ |
635 | if (SCARG(uap, cmd)((uap)->cmd.le.datum) == SWAP_STATS4) { |
636 | sep = (struct swapent *)SCARG(uap, arg)((uap)->arg.le.datum); |
637 | count = 0; |
638 | |
639 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
640 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
641 | if (count >= misc) |
642 | continue; |
643 | |
644 | sdp->swd_inuseswd_se.se_inuse = |
645 | btodb((u_int64_t)sdp->swd_npginuse <<(((u_int64_t)sdp->swd_npginuse << 12) >> 9) |
646 | PAGE_SHIFT)(((u_int64_t)sdp->swd_npginuse << 12) >> 9); |
647 | error = copyout(&sdp->swd_se, sep, |
648 | sizeof(struct swapent)); |
649 | if (error) |
650 | goto out; |
651 | |
652 | /* now copy out the path if necessary */ |
653 | error = copyoutstr(sdp->swd_path, |
654 | sep->se_path, sizeof(sep->se_path), NULL((void *)0)); |
655 | if (error) |
656 | goto out; |
657 | |
658 | count++; |
659 | sep++; |
660 | } |
661 | } |
662 | |
663 | *retval = count; |
664 | error = 0; |
665 | goto out; |
666 | } |
667 | |
668 | /* all other requests require superuser privs. verify. */ |
669 | if ((error = suser(p)) || (error = pledge_swapctl(p))) |
670 | goto out; |
671 | |
672 | /* |
673 | * at this point we expect a path name in arg. we will |
674 | * use namei() to gain a vnode reference (vref), and lock |
675 | * the vnode (VOP_LOCK). |
676 | */ |
677 | error = copyinstr(SCARG(uap, arg)((uap)->arg.le.datum), userpath, sizeof(userpath), &len); |
678 | if (error) |
679 | goto out; |
680 | disk_map(userpath, userpath, sizeof(userpath), DM_OPENBLCK0x2); |
681 | NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, UIO_SYSSPACE, userpath, p)ndinitat(&nd, 0, 0x0040|0x0004, UIO_SYSSPACE, -100, userpath , p); |
682 | if ((error = namei(&nd))) |
683 | goto out; |
684 | vp = nd.ni_vp; |
685 | /* note: "vp" is referenced and locked */ |
686 | |
687 | error = 0; /* assume no error */ |
688 | switch(SCARG(uap, cmd)((uap)->cmd.le.datum)) { |
689 | case SWAP_DUMPDEV7: |
690 | if (vp->v_type != VBLK) { |
691 | error = ENOTBLK15; |
692 | break; |
693 | } |
694 | dumpdev = vp->v_rdevv_un.vu_specinfo->si_rdev; |
695 | break; |
696 | case SWAP_CTL5: |
697 | /* |
698 | * get new priority, remove old entry (if any) and then |
699 | * reinsert it in the correct place. finally, prune out |
700 | * any empty priority structures. |
701 | */ |
702 | priority = SCARG(uap, misc)((uap)->misc.le.datum); |
703 | spp = malloc(sizeof *spp, M_VMSWAP92, M_WAITOK0x0001); |
704 | if ((sdp = swaplist_find(vp, 1)) == NULL((void *)0)) { |
705 | error = ENOENT2; |
706 | } else { |
707 | swaplist_insert(sdp, spp, priority); |
708 | swaplist_trim(); |
709 | } |
710 | if (error) |
711 | free(spp, M_VMSWAP92, sizeof(*spp)); |
712 | break; |
713 | case SWAP_ON1: |
714 | /* |
715 | * If the device is a regular file, make sure the filesystem |
716 | * can be used for swapping. |
717 | */ |
718 | if (vp->v_type == VREG && |
719 | (vp->v_mount->mnt_flag & MNT_SWAPPABLE0x00200000) == 0) { |
720 | error = ENOTSUP91; |
721 | break; |
722 | } |
723 | |
724 | /* |
725 | * check for duplicates. if none found, then insert a |
726 | * dummy entry on the list to prevent someone else from |
727 | * trying to enable this device while we are working on |
728 | * it. |
729 | */ |
730 | priority = SCARG(uap, misc)((uap)->misc.le.datum); |
731 | if ((sdp = swaplist_find(vp, 0)) != NULL((void *)0)) { |
732 | error = EBUSY16; |
733 | break; |
734 | } |
735 | sdp = malloc(sizeof *sdp, M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008); |
736 | spp = malloc(sizeof *spp, M_VMSWAP92, M_WAITOK0x0001); |
737 | sdp->swd_flagsswd_se.se_flags = SWF_FAKE0x00000008; /* placeholder only */ |
738 | sdp->swd_vp = vp; |
739 | sdp->swd_devswd_se.se_dev = (vp->v_type == VBLK) ? vp->v_rdevv_un.vu_specinfo->si_rdev : NODEV(dev_t)(-1); |
740 | |
741 | /* |
742 | * XXX Is NFS elaboration necessary? |
743 | */ |
744 | if (vp->v_type == VREG) { |
745 | sdp->swd_cred = crdup(p->p_ucred); |
746 | } |
747 | |
748 | swaplist_insert(sdp, spp, priority); |
749 | |
750 | sdp->swd_pathlen = len; |
751 | sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP92, M_WAITOK0x0001); |
752 | strlcpy(sdp->swd_path, userpath, len); |
753 | |
754 | /* |
755 | * we've now got a FAKE placeholder in the swap list. |
756 | * now attempt to enable swap on it. if we fail, undo |
757 | * what we've done and kill the fake entry we just inserted. |
758 | * if swap_on is a success, it will clear the SWF_FAKE flag |
759 | */ |
760 | |
761 | if ((error = swap_on(p, sdp)) != 0) { |
762 | (void) swaplist_find(vp, 1); /* kill fake entry */ |
763 | swaplist_trim(); |
764 | if (vp->v_type == VREG) { |
765 | crfree(sdp->swd_cred); |
766 | } |
767 | free(sdp->swd_path, M_VMSWAP92, sdp->swd_pathlen); |
768 | free(sdp, M_VMSWAP92, sizeof(*sdp)); |
769 | break; |
770 | } |
771 | break; |
772 | case SWAP_OFF2: |
773 | if ((sdp = swaplist_find(vp, 0)) == NULL((void *)0)) { |
774 | error = ENXIO6; |
775 | break; |
776 | } |
777 | |
778 | /* |
779 | * If a device isn't in use or enabled, we |
780 | * can't stop swapping from it (again). |
781 | */ |
782 | if ((sdp->swd_flagsswd_se.se_flags & (SWF_INUSE0x00000001|SWF_ENABLE0x00000002)) == 0) { |
783 | error = EBUSY16; |
784 | break; |
785 | } |
786 | |
787 | /* |
788 | * do the real work. |
789 | */ |
790 | error = swap_off(p, sdp); |
791 | break; |
792 | default: |
793 | error = EINVAL22; |
794 | } |
795 | |
796 | /* done! release the ref gained by namei() and unlock. */ |
797 | vput(vp); |
798 | |
799 | out: |
800 | rw_exit_write(&swap_syscall_lock); |
801 | |
802 | return (error); |
803 | } |
804 | |
805 | /* |
806 | * swap_on: attempt to enable a swapdev for swapping. note that the |
807 | * swapdev is already on the global list, but disabled (marked |
808 | * SWF_FAKE). |
809 | * |
810 | * => we avoid the start of the disk (to protect disk labels) |
811 | * => caller should leave uvm.swap_data_lock unlocked, we may lock it |
812 | * if needed. |
813 | */ |
814 | int |
815 | swap_on(struct proc *p, struct swapdev *sdp) |
816 | { |
817 | static int count = 0; /* static */ |
818 | struct vnode *vp; |
819 | int error, npages, nblocks, size; |
820 | long addr; |
821 | struct vattr va; |
822 | #if defined(NFSCLIENT1) |
823 | extern const struct vops nfs_vops; |
824 | #endif /* defined(NFSCLIENT) */ |
825 | dev_t dev; |
826 | |
827 | /* |
828 | * we want to enable swapping on sdp. the swd_vp contains |
829 | * the vnode we want (locked and ref'd), and the swd_dev |
830 | * contains the dev_t of the file, if it a block device. |
831 | */ |
832 | |
833 | vp = sdp->swd_vp; |
834 | dev = sdp->swd_devswd_se.se_dev; |
835 | |
836 | #if NVND1 > 0 |
837 | /* no swapping to vnds. */ |
838 | if (bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_strategy == vndstrategy) |
839 | return (EOPNOTSUPP45); |
840 | #endif |
841 | |
842 | /* |
843 | * open the swap file (mostly useful for block device files to |
844 | * let device driver know what is up). |
845 | * |
846 | * we skip the open/close for root on swap because the root |
847 | * has already been opened when root was mounted (mountroot). |
848 | */ |
849 | if (vp != rootvp) { |
850 | if ((error = VOP_OPEN(vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p))) |
851 | return (error); |
852 | } |
853 | |
854 | /* XXX this only works for block devices */ |
855 | /* |
856 | * we now need to determine the size of the swap area. for |
857 | * block specials we can call the d_psize function. |
858 | * for normal files, we must stat [get attrs]. |
859 | * |
860 | * we put the result in nblks. |
861 | * for normal files, we also want the filesystem block size |
862 | * (which we get with statfs). |
863 | */ |
864 | switch (vp->v_type) { |
865 | case VBLK: |
866 | if (bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_psize == 0 || |
867 | (nblocks = (*bdevsw[major(dev)(((unsigned)(dev) >> 8) & 0xff)].d_psize)(dev)) == -1) { |
868 | error = ENXIO6; |
869 | goto bad; |
870 | } |
871 | break; |
872 | |
873 | case VREG: |
874 | if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p))) |
875 | goto bad; |
876 | nblocks = (int)btodb(va.va_size)((va.va_size) >> 9); |
877 | if ((error = |
878 | VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)(*(vp->v_mount)->mnt_op->vfs_statfs)(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0) |
879 | goto bad; |
880 | |
881 | sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; |
882 | /* |
883 | * limit the max # of outstanding I/O requests we issue |
884 | * at any one time. take it easy on NFS servers. |
885 | */ |
886 | #if defined(NFSCLIENT1) |
887 | if (vp->v_op == &nfs_vops) |
888 | sdp->swd_maxactive = 2; /* XXX */ |
889 | else |
890 | #endif /* defined(NFSCLIENT) */ |
891 | sdp->swd_maxactive = 8; /* XXX */ |
892 | bufq_init(&sdp->swd_bufq, BUFQ_FIFO0); |
893 | break; |
894 | |
895 | default: |
896 | error = ENXIO6; |
897 | goto bad; |
898 | } |
899 | |
900 | /* |
901 | * save nblocks in a safe place and convert to pages. |
902 | */ |
903 | |
904 | sdp->swd_nblksswd_se.se_nblks = nblocks; |
905 | npages = dbtob((u_int64_t)nblocks)(((u_int64_t)nblocks) << 9) >> PAGE_SHIFT12; |
906 | |
907 | /* |
908 | * for block special files, we want to make sure that leave |
909 | * the disklabel and bootblocks alone, so we arrange to skip |
910 | * over them (arbitrarily choosing to skip PAGE_SIZE bytes). |
911 | * note that because of this the "size" can be less than the |
912 | * actual number of blocks on the device. |
913 | */ |
914 | if (vp->v_type == VBLK) { |
915 | /* we use pages 1 to (size - 1) [inclusive] */ |
916 | size = npages - 1; |
917 | addr = 1; |
918 | } else { |
919 | /* we use pages 0 to (size - 1) [inclusive] */ |
920 | size = npages; |
921 | addr = 0; |
922 | } |
923 | |
924 | /* |
925 | * make sure we have enough blocks for a reasonable sized swap |
926 | * area. we want at least one page. |
927 | */ |
928 | |
929 | if (size < 1) { |
930 | error = EINVAL22; |
931 | goto bad; |
932 | } |
933 | |
934 | /* |
935 | * now we need to allocate an extent to manage this swap device |
936 | */ |
937 | snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x", |
938 | count++); |
939 | |
940 | /* note that extent_create's 3rd arg is inclusive, thus "- 1" */ |
941 | sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP92, |
942 | 0, 0, EX_WAITOK0x0001); |
943 | /* allocate the `saved' region from the extent so it won't be used */ |
944 | if (addr) { |
945 | if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK0x0001)) |
946 | panic("disklabel reserve"); |
947 | /* XXX: is extent synchronized with swd_npginuse? */ |
948 | } |
949 | #ifdef HIBERNATE1 |
950 | /* |
951 | * Lock down the last region of primary disk swap, in case |
952 | * hibernate needs to place a signature there. |
953 | */ |
954 | if (dev == swdevt[0].sw_dev && vp->v_type == VBLK && size > 3 ) { |
955 | if (extent_alloc_region(sdp->swd_ex, |
956 | npages - 1 - 1, 1, EX_WAITOK0x0001)) |
957 | panic("hibernate reserve"); |
958 | /* XXX: is extent synchronized with swd_npginuse? */ |
959 | } |
960 | #endif |
961 | |
962 | /* add a ref to vp to reflect usage as a swap device. */ |
963 | vref(vp); |
964 | |
965 | #ifdef UVM_SWAP_ENCRYPT1 |
966 | if (uvm_doswapencrypt) |
967 | uvm_swap_initcrypt(sdp, npages); |
968 | #endif |
969 | /* now add the new swapdev to the drum and enable. */ |
970 | swapdrum_add(sdp, npages); |
971 | sdp->swd_npages = size; |
972 | sdp->swd_flagsswd_se.se_flags &= ~SWF_FAKE0x00000008; /* going live */ |
973 | sdp->swd_flagsswd_se.se_flags |= (SWF_INUSE0x00000001|SWF_ENABLE0x00000002); |
974 | uvmexp.swpages += size; |
975 | return (0); |
976 | |
977 | bad: |
978 | /* failure: close device if necessary and return error. */ |
979 | if (vp != rootvp) |
980 | (void)VOP_CLOSE(vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p); |
981 | return (error); |
982 | } |
983 | |
984 | /* |
985 | * swap_off: stop swapping on swapdev |
986 | * |
987 | * => swap data should be locked, we will unlock. |
988 | */ |
989 | int |
990 | swap_off(struct proc *p, struct swapdev *sdp) |
991 | { |
992 | int error = 0; |
993 | |
994 | /* disable the swap area being removed */ |
995 | sdp->swd_flagsswd_se.se_flags &= ~SWF_ENABLE0x00000002; |
996 | |
997 | /* |
998 | * the idea is to find all the pages that are paged out to this |
999 | * device, and page them all in. in uvm, swap-backed pageable |
1000 | * memory can take two forms: aobjs and anons. call the |
1001 | * swapoff hook for each subsystem to bring in pages. |
1002 | */ |
1003 | |
1004 | if (uao_swap_off(sdp->swd_drumoffset, |
1005 | sdp->swd_drumoffset + sdp->swd_drumsize) || |
1006 | amap_swap_off(sdp->swd_drumoffset, |
1007 | sdp->swd_drumoffset + sdp->swd_drumsize)) { |
1008 | |
1009 | error = ENOMEM12; |
1010 | } else if (sdp->swd_npginuse > sdp->swd_npgbad) { |
1011 | error = EBUSY16; |
1012 | } |
1013 | |
1014 | if (error) { |
1015 | sdp->swd_flagsswd_se.se_flags |= SWF_ENABLE0x00000002; |
1016 | return (error); |
1017 | } |
1018 | |
1019 | /* |
1020 | * done with the vnode and saved creds. |
1021 | * drop our ref on the vnode before calling VOP_CLOSE() |
1022 | * so that spec_close() can tell if this is the last close. |
1023 | */ |
1024 | if (sdp->swd_vp->v_type == VREG) { |
1025 | crfree(sdp->swd_cred); |
1026 | } |
1027 | vrele(sdp->swd_vp); |
1028 | if (sdp->swd_vp != rootvp) { |
1029 | (void) VOP_CLOSE(sdp->swd_vp, FREAD0x0001|FWRITE0x0002, p->p_ucred, p); |
1030 | } |
1031 | |
1032 | uvmexp.swpages -= sdp->swd_npages; |
1033 | |
1034 | if (swaplist_find(sdp->swd_vp, 1) == NULL((void *)0)) |
1035 | panic("swap_off: swapdev not in list"); |
1036 | swaplist_trim(); |
1037 | |
1038 | /* |
1039 | * free all resources! |
1040 | */ |
1041 | extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, |
1042 | EX_WAITOK0x0001); |
1043 | extent_destroy(sdp->swd_ex); |
1044 | /* free sdp->swd_path ? */ |
1045 | free(sdp, M_VMSWAP92, sizeof(*sdp)); |
1046 | return (0); |
1047 | } |
1048 | |
1049 | /* |
1050 | * /dev/drum interface and i/o functions |
1051 | */ |
1052 | |
1053 | /* |
1054 | * swstrategy: perform I/O on the drum |
1055 | * |
1056 | * => we must map the i/o request from the drum to the correct swapdev. |
1057 | */ |
1058 | void |
1059 | swstrategy(struct buf *bp) |
1060 | { |
1061 | struct swapdev *sdp; |
1062 | int s, pageno, bn; |
1063 | |
1064 | /* |
1065 | * convert block number to swapdev. note that swapdev can't |
1066 | * be yanked out from under us because we are holding resources |
1067 | * in it (i.e. the blocks we are doing I/O on). |
1068 | */ |
1069 | pageno = dbtob((u_int64_t)bp->b_blkno)(((u_int64_t)bp->b_blkno) << 9) >> PAGE_SHIFT12; |
1070 | sdp = swapdrum_getsdp(pageno); |
1071 | if (sdp == NULL((void *)0)) { |
1072 | bp->b_error = EINVAL22; |
1073 | bp->b_flags |= B_ERROR0x00000400; |
1074 | s = splbio()splraise(0x6); |
1075 | biodone(bp); |
1076 | splx(s)spllower(s); |
1077 | return; |
1078 | } |
1079 | |
1080 | /* convert drum page number to block number on this swapdev. */ |
1081 | pageno -= sdp->swd_drumoffset; /* page # on swapdev */ |
1082 | bn = btodb((u_int64_t)pageno << PAGE_SHIFT)(((u_int64_t)pageno << 12) >> 9); /* convert to diskblock */ |
1083 | |
1084 | /* |
1085 | * for block devices we finish up here. |
1086 | * for regular files we have to do more work which we delegate |
1087 | * to sw_reg_strategy(). |
1088 | */ |
1089 | switch (sdp->swd_vp->v_type) { |
1090 | default: |
1091 | panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); |
1092 | case VBLK: |
1093 | /* |
1094 | * must convert "bp" from an I/O on /dev/drum to an I/O |
1095 | * on the swapdev (sdp). |
1096 | */ |
1097 | s = splbio()splraise(0x6); |
1098 | buf_replacevnode(bp, sdp->swd_vp); |
1099 | |
1100 | bp->b_blkno = bn; |
1101 | splx(s)spllower(s); |
1102 | VOP_STRATEGY(bp->b_vp, bp); |
1103 | return; |
1104 | case VREG: |
1105 | /* delegate to sw_reg_strategy function. */ |
1106 | sw_reg_strategy(sdp, bp, bn); |
1107 | return; |
1108 | } |
1109 | /* NOTREACHED */ |
1110 | } |
1111 | |
1112 | /* |
1113 | * sw_reg_strategy: handle swap i/o to regular files |
1114 | */ |
1115 | void |
1116 | sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) |
1117 | { |
1118 | struct vnode *vp; |
1119 | struct vndxfer *vnx; |
1120 | daddr_t nbn; |
1121 | caddr_t addr; |
1122 | off_t byteoff; |
1123 | int s, off, nra, error, sz, resid; |
1124 | |
1125 | /* |
1126 | * allocate a vndxfer head for this transfer and point it to |
1127 | * our buffer. |
1128 | */ |
1129 | vnx = pool_get(&vndxfer_pool, PR_WAITOK0x0001); |
1130 | vnx->vx_flags = VX_BUSY1; |
1131 | vnx->vx_error = 0; |
1132 | vnx->vx_pending = 0; |
1133 | vnx->vx_bp = bp; |
1134 | vnx->vx_sdp = sdp; |
1135 | |
1136 | /* |
1137 | * setup for main loop where we read filesystem blocks into |
1138 | * our buffer. |
1139 | */ |
1140 | error = 0; |
Value stored to 'error' is never read | |
1141 | bp->b_resid = bp->b_bcount; /* nothing transferred yet! */ |
1142 | addr = bp->b_data; /* current position in buffer */ |
1143 | byteoff = dbtob((u_int64_t)bn)(((u_int64_t)bn) << 9); |
1144 | |
1145 | for (resid = bp->b_resid; resid; resid -= sz) { |
1146 | struct vndbuf *nbp; |
1147 | /* |
1148 | * translate byteoffset into block number. return values: |
1149 | * vp = vnode of underlying device |
1150 | * nbn = new block number (on underlying vnode dev) |
1151 | * nra = num blocks we can read-ahead (excludes requested |
1152 | * block) |
1153 | */ |
1154 | nra = 0; |
1155 | error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, |
1156 | &vp, &nbn, &nra); |
1157 | |
1158 | if (error == 0 && nbn == -1) { |
1159 | /* |
1160 | * this used to just set error, but that doesn't |
1161 | * do the right thing. Instead, it causes random |
1162 | * memory errors. The panic() should remain until |
1163 | * this condition doesn't destabilize the system. |
1164 | */ |
1165 | #if 1 |
1166 | panic("sw_reg_strategy: swap to sparse file"); |
1167 | #else |
1168 | error = EIO5; /* failure */ |
1169 | #endif |
1170 | } |
1171 | |
1172 | /* |
1173 | * punt if there was an error or a hole in the file. |
1174 | * we must wait for any i/o ops we have already started |
1175 | * to finish before returning. |
1176 | * |
1177 | * XXX we could deal with holes here but it would be |
1178 | * a hassle (in the write case). |
1179 | */ |
1180 | if (error) { |
1181 | s = splbio()splraise(0x6); |
1182 | vnx->vx_error = error; /* pass error up */ |
1183 | goto out; |
1184 | } |
1185 | |
1186 | /* |
1187 | * compute the size ("sz") of this transfer (in bytes). |
1188 | */ |
1189 | off = byteoff % sdp->swd_bsize; |
1190 | sz = (1 + nra) * sdp->swd_bsize - off; |
1191 | if (sz > resid) |
1192 | sz = resid; |
1193 | |
1194 | /* |
1195 | * now get a buf structure. note that the vb_buf is |
1196 | * at the front of the nbp structure so that you can |
1197 | * cast pointers between the two structure easily. |
1198 | */ |
1199 | nbp = pool_get(&vndbuf_pool, PR_WAITOK0x0001); |
1200 | nbp->vb_buf.b_flags = bp->b_flags | B_CALL0x00000040; |
1201 | nbp->vb_buf.b_bcount = sz; |
1202 | nbp->vb_buf.b_bufsize = sz; |
1203 | nbp->vb_buf.b_error = 0; |
1204 | nbp->vb_buf.b_data = addr; |
1205 | nbp->vb_buf.b_bq = NULL((void *)0); |
1206 | nbp->vb_buf.b_blkno = nbn + btodb(off)((off) >> 9); |
1207 | nbp->vb_buf.b_proc = bp->b_proc; |
1208 | nbp->vb_buf.b_iodone = sw_reg_iodone; |
1209 | nbp->vb_buf.b_vp = NULLVP((struct vnode *)((void *)0)); |
1210 | nbp->vb_buf.b_vnbufs.le_next = NOLIST((struct buf *)0x87654321); |
1211 | LIST_INIT(&nbp->vb_buf.b_dep)do { ((&nbp->vb_buf.b_dep)->lh_first) = ((void *)0) ; } while (0); |
1212 | |
1213 | /* |
1214 | * set b_dirtyoff/end and b_validoff/end. this is |
1215 | * required by the NFS client code (otherwise it will |
1216 | * just discard our I/O request). |
1217 | */ |
1218 | if (bp->b_dirtyend == 0) { |
1219 | nbp->vb_buf.b_dirtyoff = 0; |
1220 | nbp->vb_buf.b_dirtyend = sz; |
1221 | } else { |
1222 | nbp->vb_buf.b_dirtyoff = |
1223 | max(0, bp->b_dirtyoff - (bp->b_bcount-resid)); |
1224 | nbp->vb_buf.b_dirtyend = |
1225 | min(sz, |
1226 | max(0, bp->b_dirtyend - (bp->b_bcount-resid))); |
1227 | } |
1228 | if (bp->b_validend == 0) { |
1229 | nbp->vb_buf.b_validoff = 0; |
1230 | nbp->vb_buf.b_validend = sz; |
1231 | } else { |
1232 | nbp->vb_buf.b_validoff = |
1233 | max(0, bp->b_validoff - (bp->b_bcount-resid)); |
1234 | nbp->vb_buf.b_validend = |
1235 | min(sz, |
1236 | max(0, bp->b_validend - (bp->b_bcount-resid))); |
1237 | } |
1238 | |
1239 | /* patch it back to the vnx */ |
1240 | nbp->vb_vnx = vnx; |
1241 | task_set(&nbp->vb_task, sw_reg_iodone_internal, nbp); |
1242 | |
1243 | s = splbio()splraise(0x6); |
1244 | if (vnx->vx_error != 0) { |
1245 | pool_put(&vndbuf_pool, nbp); |
1246 | goto out; |
1247 | } |
1248 | vnx->vx_pending++; |
1249 | |
1250 | /* assoc new buffer with underlying vnode */ |
1251 | bgetvp(vp, &nbp->vb_buf); |
1252 | |
1253 | /* start I/O if we are not over our limit */ |
1254 | bufq_queue(&sdp->swd_bufq, &nbp->vb_buf); |
1255 | sw_reg_start(sdp); |
1256 | splx(s)spllower(s); |
1257 | |
1258 | /* |
1259 | * advance to the next I/O |
1260 | */ |
1261 | byteoff += sz; |
1262 | addr += sz; |
1263 | } |
1264 | |
1265 | s = splbio()splraise(0x6); |
1266 | |
1267 | out: /* Arrive here at splbio */ |
1268 | vnx->vx_flags &= ~VX_BUSY1; |
1269 | if (vnx->vx_pending == 0) { |
1270 | if (vnx->vx_error != 0) { |
1271 | bp->b_error = vnx->vx_error; |
1272 | bp->b_flags |= B_ERROR0x00000400; |
1273 | } |
1274 | pool_put(&vndxfer_pool, vnx); |
1275 | biodone(bp); |
1276 | } |
1277 | splx(s)spllower(s); |
1278 | } |
1279 | |
1280 | /* sw_reg_start: start an I/O request on the requested swapdev. */ |
1281 | void |
1282 | sw_reg_start(struct swapdev *sdp) |
1283 | { |
1284 | struct buf *bp; |
1285 | |
1286 | /* XXX: recursion control */ |
1287 | if ((sdp->swd_flagsswd_se.se_flags & SWF_BUSY0x00000004) != 0) |
1288 | return; |
1289 | |
1290 | sdp->swd_flagsswd_se.se_flags |= SWF_BUSY0x00000004; |
1291 | |
1292 | while (sdp->swd_active < sdp->swd_maxactive) { |
1293 | bp = bufq_dequeue(&sdp->swd_bufq); |
1294 | if (bp == NULL((void *)0)) |
1295 | break; |
1296 | |
1297 | sdp->swd_active++; |
1298 | |
1299 | if ((bp->b_flags & B_READ0x00008000) == 0) |
1300 | bp->b_vp->v_numoutput++; |
1301 | |
1302 | VOP_STRATEGY(bp->b_vp, bp); |
1303 | } |
1304 | sdp->swd_flagsswd_se.se_flags &= ~SWF_BUSY0x00000004; |
1305 | } |
1306 | |
1307 | /* |
1308 | * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup |
1309 | * |
1310 | * => note that we can recover the vndbuf struct by casting the buf ptr |
1311 | * |
1312 | * XXX: |
1313 | * We only put this onto a taskq here, because of the maxactive game since |
1314 | * it basically requires us to call back into VOP_STRATEGY() (where we must |
1315 | * be able to sleep) via sw_reg_start(). |
1316 | */ |
1317 | void |
1318 | sw_reg_iodone(struct buf *bp) |
1319 | { |
1320 | struct vndbuf *vbp = (struct vndbuf *)bp; |
1321 | task_add(systq, &vbp->vb_task); |
1322 | } |
1323 | |
1324 | void |
1325 | sw_reg_iodone_internal(void *xvbp) |
1326 | { |
1327 | struct vndbuf *vbp = xvbp; |
1328 | struct vndxfer *vnx = vbp->vb_vnx; |
1329 | struct buf *pbp = vnx->vx_bp; /* parent buffer */ |
1330 | struct swapdev *sdp = vnx->vx_sdp; |
1331 | int resid, s; |
1332 | |
1333 | s = splbio()splraise(0x6); |
1334 | |
1335 | resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; |
1336 | pbp->b_resid -= resid; |
1337 | vnx->vx_pending--; |
1338 | |
1339 | /* pass error upward */ |
1340 | if (vbp->vb_buf.b_error) |
1341 | vnx->vx_error = vbp->vb_buf.b_error; |
1342 | |
1343 | /* disassociate this buffer from the vnode (if any). */ |
1344 | if (vbp->vb_buf.b_vp != NULL((void *)0)) { |
1345 | brelvp(&vbp->vb_buf); |
1346 | } |
1347 | |
1348 | /* kill vbp structure */ |
1349 | pool_put(&vndbuf_pool, vbp); |
1350 | |
1351 | /* |
1352 | * wrap up this transaction if it has run to completion or, in |
1353 | * case of an error, when all auxiliary buffers have returned. |
1354 | */ |
1355 | if (vnx->vx_error != 0) { |
1356 | /* pass error upward */ |
1357 | pbp->b_flags |= B_ERROR0x00000400; |
1358 | pbp->b_error = vnx->vx_error; |
1359 | if ((vnx->vx_flags & VX_BUSY1) == 0 && vnx->vx_pending == 0) { |
1360 | pool_put(&vndxfer_pool, vnx); |
1361 | biodone(pbp); |
1362 | } |
1363 | } else if (pbp->b_resid == 0) { |
1364 | KASSERT(vnx->vx_pending == 0)((vnx->vx_pending == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_swap.c", 1364, "vnx->vx_pending == 0" )); |
1365 | if ((vnx->vx_flags & VX_BUSY1) == 0) { |
1366 | pool_put(&vndxfer_pool, vnx); |
1367 | biodone(pbp); |
1368 | } |
1369 | } |
1370 | |
1371 | /* |
1372 | * done! start next swapdev I/O if one is pending |
1373 | */ |
1374 | sdp->swd_active--; |
1375 | sw_reg_start(sdp); |
1376 | splx(s)spllower(s); |
1377 | } |
1378 | |
1379 | |
1380 | /* |
1381 | * uvm_swap_alloc: allocate space on swap |
1382 | * |
1383 | * => allocation is done "round robin" down the priority list, as we |
1384 | * allocate in a priority we "rotate" the tail queue. |
1385 | * => space can be freed with uvm_swap_free |
1386 | * => we return the page slot number in /dev/drum (0 == invalid slot) |
1387 | * => we lock uvm.swap_data_lock |
1388 | * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM |
1389 | */ |
1390 | int |
1391 | uvm_swap_alloc(int *nslots, boolean_t lessok) |
1392 | { |
1393 | struct swapdev *sdp; |
1394 | struct swappri *spp; |
1395 | u_long result; |
1396 | |
1397 | /* |
1398 | * no swap devices configured yet? definite failure. |
1399 | */ |
1400 | if (uvmexp.nswapdev < 1) |
1401 | return 0; |
1402 | |
1403 | /* |
1404 | * lock data lock, convert slots into blocks, and enter loop |
1405 | */ |
1406 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1406, "_kernel_lock_held()")); |
1407 | ReTry: /* XXXMRG */ |
1408 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
1409 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
1410 | /* if it's not enabled, then we can't swap from it */ |
1411 | if ((sdp->swd_flagsswd_se.se_flags & SWF_ENABLE0x00000002) == 0) |
1412 | continue; |
1413 | if (sdp->swd_npginuse + *nslots > sdp->swd_npages) |
1414 | continue; |
1415 | if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN, 0,extent_alloc_subregion((sdp->swd_ex), (sdp->swd_ex)-> ex_start, (sdp->swd_ex)->ex_end, (*nslots), (1), (0), ( 0), (0x0010|0x0000), (&result)) |
1416 | EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT,extent_alloc_subregion((sdp->swd_ex), (sdp->swd_ex)-> ex_start, (sdp->swd_ex)->ex_end, (*nslots), (1), (0), ( 0), (0x0010|0x0000), (&result)) |
1417 | &result)extent_alloc_subregion((sdp->swd_ex), (sdp->swd_ex)-> ex_start, (sdp->swd_ex)->ex_end, (*nslots), (1), (0), ( 0), (0x0010|0x0000), (&result)) != 0) { |
1418 | continue; |
1419 | } |
1420 | |
1421 | /* |
1422 | * successful allocation! now rotate the tailq. |
1423 | */ |
1424 | TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next)do { if (((sdp)->swd_next.tqe_next) != ((void *)0)) (sdp)-> swd_next.tqe_next->swd_next.tqe_prev = (sdp)->swd_next. tqe_prev; else (&spp->spi_swapdev)->tqh_last = (sdp )->swd_next.tqe_prev; *(sdp)->swd_next.tqe_prev = (sdp) ->swd_next.tqe_next; ((sdp)->swd_next.tqe_prev) = ((void *)-1); ((sdp)->swd_next.tqe_next) = ((void *)-1); } while (0); |
1425 | TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next)do { (sdp)->swd_next.tqe_next = ((void *)0); (sdp)->swd_next .tqe_prev = (&spp->spi_swapdev)->tqh_last; *(&spp ->spi_swapdev)->tqh_last = (sdp); (&spp->spi_swapdev )->tqh_last = &(sdp)->swd_next.tqe_next; } while (0 ); |
1426 | sdp->swd_npginuse += *nslots; |
1427 | uvmexp.swpginuse += *nslots; |
1428 | /* done! return drum slot number */ |
1429 | return result + sdp->swd_drumoffset; |
1430 | } |
1431 | } |
1432 | |
1433 | /* XXXMRG: BEGIN HACK */ |
1434 | if (*nslots > 1 && lessok) { |
1435 | *nslots = 1; |
1436 | goto ReTry; /* XXXMRG: ugh! extent should support this for us */ |
1437 | } |
1438 | /* XXXMRG: END HACK */ |
1439 | |
1440 | return 0; /* failed */ |
1441 | } |
1442 | |
1443 | /* |
1444 | * uvm_swapisfull: return true if all of available swap is allocated |
1445 | * and in use. |
1446 | */ |
1447 | int |
1448 | uvm_swapisfull(void) |
1449 | { |
1450 | int result; |
1451 | |
1452 | KERNEL_LOCK()_kernel_lock(); |
1453 | KASSERT(uvmexp.swpgonly <= uvmexp.swpages)((uvmexp.swpgonly <= uvmexp.swpages) ? (void)0 : __assert( "diagnostic ", "/usr/src/sys/uvm/uvm_swap.c", 1453, "uvmexp.swpgonly <= uvmexp.swpages" )); |
1454 | result = (uvmexp.swpgonly == uvmexp.swpages); |
1455 | KERNEL_UNLOCK()_kernel_unlock(); |
1456 | |
1457 | return result; |
1458 | } |
1459 | |
1460 | /* |
1461 | * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors |
1462 | * |
1463 | * => we lock uvm.swap_data_lock |
1464 | */ |
1465 | void |
1466 | uvm_swap_markbad(int startslot, int nslots) |
1467 | { |
1468 | struct swapdev *sdp; |
1469 | |
1470 | KERNEL_LOCK()_kernel_lock(); |
1471 | sdp = swapdrum_getsdp(startslot); |
1472 | if (sdp != NULL((void *)0)) { |
1473 | /* |
1474 | * we just keep track of how many pages have been marked bad |
1475 | * in this device, to make everything add up in swap_off(). |
1476 | * we assume here that the range of slots will all be within |
1477 | * one swap device. |
1478 | */ |
1479 | sdp->swd_npgbad += nslots; |
1480 | } |
1481 | KERNEL_UNLOCK()_kernel_unlock(); |
1482 | } |
1483 | |
1484 | /* |
1485 | * uvm_swap_free: free swap slots |
1486 | * |
1487 | * => this can be all or part of an allocation made by uvm_swap_alloc |
1488 | * => we lock uvm.swap_data_lock |
1489 | */ |
1490 | void |
1491 | uvm_swap_free(int startslot, int nslots) |
1492 | { |
1493 | struct swapdev *sdp; |
1494 | |
1495 | /* |
1496 | * ignore attempts to free the "bad" slot. |
1497 | */ |
1498 | |
1499 | if (startslot == SWSLOT_BAD(-1)) { |
1500 | return; |
1501 | } |
1502 | |
1503 | /* |
1504 | * convert drum slot offset back to sdp, free the blocks |
1505 | * in the extent, and return. must hold pri lock to do |
1506 | * lookup and access the extent. |
1507 | */ |
1508 | KERNEL_LOCK()_kernel_lock(); |
1509 | sdp = swapdrum_getsdp(startslot); |
1510 | KASSERT(uvmexp.nswapdev >= 1)((uvmexp.nswapdev >= 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_swap.c", 1510, "uvmexp.nswapdev >= 1" )); |
1511 | KASSERT(sdp != NULL)((sdp != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1511, "sdp != NULL")); |
1512 | KASSERT(sdp->swd_npginuse >= nslots)((sdp->swd_npginuse >= nslots) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_swap.c", 1512, "sdp->swd_npginuse >= nslots" )); |
1513 | if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots, |
1514 | EX_MALLOCOK0x0010|EX_NOWAIT0x0000) != 0) { |
1515 | printf("warning: resource shortage: %d pages of swap lost\n", |
1516 | nslots); |
1517 | } |
1518 | |
1519 | sdp->swd_npginuse -= nslots; |
1520 | uvmexp.swpginuse -= nslots; |
1521 | #ifdef UVM_SWAP_ENCRYPT1 |
1522 | { |
1523 | int i; |
1524 | if (swap_encrypt_initialized) { |
1525 | /* Dereference keys */ |
1526 | for (i = 0; i < nslots; i++) |
1527 | if (uvm_swap_needdecrypt(sdp, startslot + i)) { |
1528 | struct swap_key *key; |
1529 | |
1530 | key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]); |
1531 | if (key->refcount != 0) |
1532 | SWAP_KEY_PUT(sdp, key)do { (key)->refcount--; if ((key)->refcount == 0) { swap_key_delete (key); } } while(0);; |
1533 | } |
1534 | |
1535 | /* Mark range as not decrypt */ |
1536 | uvm_swap_markdecrypt(sdp, startslot, nslots, 0); |
1537 | } |
1538 | } |
1539 | #endif /* UVM_SWAP_ENCRYPT */ |
1540 | KERNEL_UNLOCK()_kernel_unlock(); |
1541 | } |
1542 | |
1543 | /* |
1544 | * uvm_swap_put: put any number of pages into a contig place on swap |
1545 | * |
1546 | * => can be sync or async |
1547 | */ |
1548 | int |
1549 | uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) |
1550 | { |
1551 | int result; |
1552 | |
1553 | result = uvm_swap_io(ppsp, swslot, npages, B_WRITE0x00000000 | |
1554 | ((flags & PGO_SYNCIO0x002) ? 0 : B_ASYNC0x00000004)); |
1555 | |
1556 | return (result); |
1557 | } |
1558 | |
1559 | /* |
1560 | * uvm_swap_get: get a single page from swap |
1561 | * |
1562 | * => usually a sync op (from fault) |
1563 | */ |
1564 | int |
1565 | uvm_swap_get(struct vm_page *page, int swslot, int flags) |
1566 | { |
1567 | int result; |
1568 | |
1569 | uvmexp.nswget++; |
1570 | KASSERT(flags & PGO_SYNCIO)((flags & 0x002) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1570, "flags & PGO_SYNCIO")); |
1571 | if (swslot == SWSLOT_BAD(-1)) { |
1572 | return VM_PAGER_ERROR4; |
1573 | } |
1574 | |
1575 | KERNEL_LOCK()_kernel_lock(); |
1576 | /* this page is (about to be) no longer only in swap. */ |
1577 | atomic_dec_int(&uvmexp.swpgonly)_atomic_dec_int(&uvmexp.swpgonly); |
1578 | |
1579 | result = uvm_swap_io(&page, swslot, 1, B_READ0x00008000 | |
1580 | ((flags & PGO_SYNCIO0x002) ? 0 : B_ASYNC0x00000004)); |
1581 | |
1582 | if (result != VM_PAGER_OK0 && result != VM_PAGER_PEND3) { |
1583 | /* oops, the read failed so it really is still only in swap. */ |
1584 | atomic_inc_int(&uvmexp.swpgonly)_atomic_inc_int(&uvmexp.swpgonly); |
1585 | } |
1586 | KERNEL_UNLOCK()_kernel_unlock(); |
1587 | return (result); |
1588 | } |
1589 | |
1590 | /* |
1591 | * uvm_swap_io: do an i/o operation to swap |
1592 | */ |
1593 | |
1594 | int |
1595 | uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) |
1596 | { |
1597 | daddr_t startblk; |
1598 | struct buf *bp; |
1599 | vaddr_t kva; |
1600 | int result, s, mapinflags, pflag, bounce = 0, i; |
1601 | boolean_t write, async; |
1602 | vaddr_t bouncekva; |
1603 | struct vm_page *tpps[MAXBSIZE(64 * 1024) >> PAGE_SHIFT12]; |
1604 | #ifdef UVM_SWAP_ENCRYPT1 |
1605 | struct swapdev *sdp; |
1606 | int encrypt = 0; |
1607 | #endif |
1608 | |
1609 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_swap.c" , 1609, "_kernel_lock_held()")); |
1610 | |
1611 | write = (flags & B_READ0x00008000) == 0; |
1612 | async = (flags & B_ASYNC0x00000004) != 0; |
1613 | |
1614 | /* convert starting drum slot to block number */ |
1615 | startblk = btodb((u_int64_t)startslot << PAGE_SHIFT)(((u_int64_t)startslot << 12) >> 9); |
1616 | |
1617 | /* |
1618 | * first, map the pages into the kernel (XXX: currently required |
1619 | * by buffer system). |
1620 | */ |
1621 | mapinflags = !write ? UVMPAGER_MAPIN_READ0x02 : UVMPAGER_MAPIN_WRITE0x00; |
1622 | if (!async) |
1623 | mapinflags |= UVMPAGER_MAPIN_WAITOK0x01; |
1624 | kva = uvm_pagermapin(pps, npages, mapinflags); |
1625 | if (kva == 0) |
1626 | return (VM_PAGER_AGAIN5); |
1627 | |
1628 | #ifdef UVM_SWAP_ENCRYPT1 |
1629 | if (write) { |
1630 | /* |
1631 | * Check if we need to do swap encryption on old pages. |
1632 | * Later we need a different scheme, that swap encrypts |
1633 | * all pages of a process that had at least one page swap |
1634 | * encrypted. Then we might not need to copy all pages |
1635 | * in the cluster, and avoid the memory overheard in |
1636 | * swapping. |
1637 | */ |
1638 | if (uvm_doswapencrypt) |
1639 | encrypt = 1; |
1640 | } |
1641 | |
1642 | if (swap_encrypt_initialized || encrypt) { |
1643 | /* |
1644 | * we need to know the swap device that we are swapping to/from |
1645 | * to see if the pages need to be marked for decryption or |
1646 | * actually need to be decrypted. |
1647 | * XXX - does this information stay the same over the whole |
1648 | * execution of this function? |
1649 | */ |
1650 | sdp = swapdrum_getsdp(startslot); |
1651 | } |
1652 | |
1653 | /* |
1654 | * Check that we are dma capable for read (write always bounces |
1655 | * through the swapencrypt anyway... |
1656 | */ |
1657 | if (write && encrypt) { |
1658 | bounce = 1; /* bounce through swapencrypt always */ |
1659 | } else { |
1660 | #else |
1661 | { |
1662 | #endif |
1663 | |
1664 | for (i = 0; i < npages; i++) { |
1665 | if (VM_PAGE_TO_PHYS(pps[i])((pps[i])->phys_addr) < dma_constraint.ucr_low || |
1666 | VM_PAGE_TO_PHYS(pps[i])((pps[i])->phys_addr) > dma_constraint.ucr_high) { |
1667 | bounce = 1; |
1668 | break; |
1669 | } |
1670 | } |
1671 | } |
1672 | |
1673 | if (bounce) { |
1674 | int swmapflags; |
1675 | |
1676 | /* We always need write access. */ |
1677 | swmapflags = UVMPAGER_MAPIN_READ0x02; |
1678 | if (!async) |
1679 | swmapflags |= UVMPAGER_MAPIN_WAITOK0x01; |
1680 | |
1681 | if (!uvm_swap_allocpages(tpps, npages)) { |
1682 | uvm_pagermapout(kva, npages); |
1683 | return (VM_PAGER_AGAIN5); |
1684 | } |
1685 | |
1686 | bouncekva = uvm_pagermapin(tpps, npages, swmapflags); |
1687 | if (bouncekva == 0) { |
1688 | uvm_pagermapout(kva, npages); |
1689 | uvm_swap_freepages(tpps, npages); |
1690 | return (VM_PAGER_AGAIN5); |
1691 | } |
1692 | } |
1693 | |
1694 | /* encrypt to swap */ |
1695 | if (write && bounce) { |
1696 | int i, opages; |
1697 | caddr_t src, dst; |
1698 | u_int64_t block; |
1699 | |
1700 | src = (caddr_t) kva; |
1701 | dst = (caddr_t) bouncekva; |
1702 | block = startblk; |
1703 | for (i = 0; i < npages; i++) { |
1704 | #ifdef UVM_SWAP_ENCRYPT1 |
1705 | struct swap_key *key; |
1706 | |
1707 | if (encrypt) { |
1708 | key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]); |
1709 | SWAP_KEY_GET(sdp, key)do { if ((key)->refcount == 0) { swap_key_create(key); } ( key)->refcount++; } while(0);; /* add reference */ |
1710 | |
1711 | swap_encrypt(key, src, dst, block, PAGE_SIZE(1 << 12)); |
1712 | block += btodb(PAGE_SIZE)(((1 << 12)) >> 9); |
1713 | } else { |
1714 | #else |
1715 | { |
1716 | #endif /* UVM_SWAP_ENCRYPT */ |
1717 | memcpy(dst, src, PAGE_SIZE)__builtin_memcpy((dst), (src), ((1 << 12))); |
1718 | } |
1719 | /* this just tells async callbacks to free */ |
1720 | atomic_setbits_intx86_atomic_setbits_u32(&tpps[i]->pg_flags, PQ_ENCRYPT0x00400000); |
1721 | src += PAGE_SIZE(1 << 12); |
1722 | dst += PAGE_SIZE(1 << 12); |
1723 | } |
1724 | |
1725 | uvm_pagermapout(kva, npages); |
1726 | |
1727 | /* dispose of pages we dont use anymore */ |
1728 | opages = npages; |
1729 | uvm_pager_dropcluster(NULL((void *)0), NULL((void *)0), pps, &opages, |
1730 | PGO_PDFREECLUST0x080); |
1731 | |
1732 | kva = bouncekva; |
1733 | } |
1734 | |
1735 | /* |
1736 | * now allocate a buf for the i/o. |
1737 | * [make sure we don't put the pagedaemon to sleep...] |
1738 | */ |
1739 | pflag = (async || curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc == uvm.pagedaemon_proc) ? PR_NOWAIT0x0002 : |
1740 | PR_WAITOK0x0001; |
1741 | bp = pool_get(&bufpool, pflag | PR_ZERO0x0008); |
1742 | |
1743 | /* |
1744 | * if we failed to get a swapbuf, return "try again" |
1745 | */ |
1746 | if (bp == NULL((void *)0)) { |
1747 | if (write && bounce) { |
1748 | #ifdef UVM_SWAP_ENCRYPT1 |
1749 | int i; |
1750 | |
1751 | /* swap encrypt needs cleanup */ |
1752 | if (encrypt) |
1753 | for (i = 0; i < npages; i++) |
1754 | SWAP_KEY_PUT(sdp, SWD_KEY(sdp,do { (&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]))->refcount--; if ((&((sdp)->swd_keys [((startslot + i) - (sdp)->swd_drumoffset) >> 7]))-> refcount == 0) { swap_key_delete(&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset) >> 7])); } } while(0) ; |
1755 | startslot + i))do { (&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]))->refcount--; if ((&((sdp)->swd_keys [((startslot + i) - (sdp)->swd_drumoffset) >> 7]))-> refcount == 0) { swap_key_delete(&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset) >> 7])); } } while(0) ;; |
1756 | #endif |
1757 | |
1758 | uvm_pagermapout(kva, npages); |
1759 | uvm_swap_freepages(tpps, npages); |
1760 | } |
1761 | return (VM_PAGER_AGAIN5); |
1762 | } |
1763 | |
1764 | /* |
1765 | * prevent ASYNC reads. |
1766 | * uvm_swap_io is only called from uvm_swap_get, uvm_swap_get |
1767 | * assumes that all gets are SYNCIO. Just make sure here. |
1768 | * XXXARTUBC - might not be true anymore. |
1769 | */ |
1770 | if (!write) { |
1771 | flags &= ~B_ASYNC0x00000004; |
1772 | async = 0; |
1773 | } |
1774 | |
1775 | /* |
1776 | * fill in the bp. we currently route our i/o through |
1777 | * /dev/drum's vnode [swapdev_vp]. |
1778 | */ |
1779 | bp->b_flags = B_BUSY0x00000010 | B_NOCACHE0x00001000 | B_RAW0x00004000 | (flags & (B_READ0x00008000|B_ASYNC0x00000004)); |
1780 | bp->b_proc = &proc0; /* XXX */ |
1781 | bp->b_vnbufs.le_next = NOLIST((struct buf *)0x87654321); |
1782 | if (bounce) |
1783 | bp->b_data = (caddr_t)bouncekva; |
1784 | else |
1785 | bp->b_data = (caddr_t)kva; |
1786 | bp->b_bq = NULL((void *)0); |
1787 | bp->b_blkno = startblk; |
1788 | LIST_INIT(&bp->b_dep)do { ((&bp->b_dep)->lh_first) = ((void *)0); } while (0); |
1789 | s = splbio()splraise(0x6); |
1790 | bp->b_vp = NULL((void *)0); |
1791 | buf_replacevnode(bp, swapdev_vp); |
1792 | splx(s)spllower(s); |
1793 | bp->b_bufsize = bp->b_bcount = (long)npages << PAGE_SHIFT12; |
1794 | |
1795 | /* |
1796 | * for pageouts we must set "dirtyoff" [NFS client code needs it]. |
1797 | * and we bump v_numoutput (counter of number of active outputs). |
1798 | */ |
1799 | if (write) { |
1800 | bp->b_dirtyoff = 0; |
1801 | bp->b_dirtyend = npages << PAGE_SHIFT12; |
1802 | #ifdef UVM_SWAP_ENCRYPT1 |
1803 | /* mark the pages in the drum for decryption */ |
1804 | if (swap_encrypt_initialized) |
1805 | uvm_swap_markdecrypt(sdp, startslot, npages, encrypt); |
1806 | #endif |
1807 | s = splbio()splraise(0x6); |
1808 | swapdev_vp->v_numoutput++; |
1809 | splx(s)spllower(s); |
1810 | } |
1811 | |
1812 | /* for async ops we must set up the iodone handler. */ |
1813 | if (async) { |
1814 | bp->b_flags |= B_CALL0x00000040 | (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc == uvm.pagedaemon_proc ? |
1815 | B_PDAEMON0x00200000 : 0); |
1816 | bp->b_iodone = uvm_aio_biodone; |
1817 | } |
1818 | |
1819 | /* now we start the I/O, and if async, return. */ |
1820 | VOP_STRATEGY(bp->b_vp, bp); |
1821 | if (async) |
1822 | return (VM_PAGER_PEND3); |
1823 | |
1824 | /* must be sync i/o. wait for it to finish */ |
1825 | (void) biowait(bp); |
1826 | result = (bp->b_flags & B_ERROR0x00000400) ? VM_PAGER_ERROR4 : VM_PAGER_OK0; |
1827 | |
1828 | /* decrypt swap */ |
1829 | if (!write && !(bp->b_flags & B_ERROR0x00000400)) { |
1830 | int i; |
1831 | caddr_t data = (caddr_t)kva; |
1832 | caddr_t dst = (caddr_t)kva; |
1833 | u_int64_t block = startblk; |
1834 | |
1835 | if (bounce) |
1836 | data = (caddr_t)bouncekva; |
1837 | |
1838 | for (i = 0; i < npages; i++) { |
1839 | #ifdef UVM_SWAP_ENCRYPT1 |
1840 | struct swap_key *key; |
1841 | |
1842 | /* Check if we need to decrypt */ |
1843 | if (swap_encrypt_initialized && |
1844 | uvm_swap_needdecrypt(sdp, startslot + i)) { |
1845 | key = SWD_KEY(sdp, startslot + i)&((sdp)->swd_keys[((startslot + i) - (sdp)->swd_drumoffset ) >> 7]); |
1846 | if (key->refcount == 0) { |
1847 | result = VM_PAGER_ERROR4; |
1848 | break; |
1849 | } |
1850 | swap_decrypt(key, data, dst, block, PAGE_SIZE(1 << 12)); |
1851 | } else if (bounce) { |
1852 | #else |
1853 | if (bounce) { |
1854 | #endif |
1855 | memcpy(dst, data, PAGE_SIZE)__builtin_memcpy((dst), (data), ((1 << 12))); |
1856 | } |
1857 | data += PAGE_SIZE(1 << 12); |
1858 | dst += PAGE_SIZE(1 << 12); |
1859 | block += btodb(PAGE_SIZE)(((1 << 12)) >> 9); |
1860 | } |
1861 | if (bounce) |
1862 | uvm_pagermapout(bouncekva, npages); |
1863 | } |
1864 | /* kill the pager mapping */ |
1865 | uvm_pagermapout(kva, npages); |
1866 | |
1867 | /* Not anymore needed, free after encryption/bouncing */ |
1868 | if (!write && bounce) |
1869 | uvm_swap_freepages(tpps, npages); |
1870 | |
1871 | /* now dispose of the buf */ |
1872 | s = splbio()splraise(0x6); |
1873 | if (bp->b_vp) |
1874 | brelvp(bp); |
1875 | |
1876 | if (write && bp->b_vp) |
1877 | vwakeup(bp->b_vp); |
1878 | pool_put(&bufpool, bp); |
1879 | splx(s)spllower(s); |
1880 | |
1881 | /* finally return. */ |
1882 | return (result); |
1883 | } |
1884 | |
1885 | void |
1886 | swapmount(void) |
1887 | { |
1888 | struct swapdev *sdp; |
1889 | struct swappri *spp; |
1890 | struct vnode *vp; |
1891 | dev_t swap_dev = swdevt[0].sw_dev; |
1892 | char *nam; |
1893 | char path[MNAMELEN90 + 1]; |
1894 | |
1895 | /* |
1896 | * No locking here since we happen to know that we will just be called |
1897 | * once before any other process has forked. |
1898 | */ |
1899 | if (swap_dev == NODEV(dev_t)(-1)) |
1900 | return; |
1901 | |
1902 | #if defined(NFSCLIENT1) |
1903 | if (swap_dev == NETDEV(dev_t)(-2)) { |
1904 | extern struct nfs_diskless nfs_diskless; |
1905 | |
1906 | snprintf(path, sizeof(path), "%s", |
1907 | nfs_diskless.nd_swap.ndm_host); |
1908 | vp = nfs_diskless.sw_vp; |
1909 | goto gotit; |
1910 | } else |
1911 | #endif |
1912 | if (bdevvp(swap_dev, &vp)) |
1913 | return; |
1914 | |
1915 | /* Construct a potential path to swap */ |
1916 | if ((nam = findblkname(major(swap_dev)(((unsigned)(swap_dev) >> 8) & 0xff)))) |
1917 | snprintf(path, sizeof(path), "/dev/%s%d%c", nam, |
1918 | DISKUNIT(swap_dev)(((unsigned)((swap_dev) & 0xff) | (((swap_dev) & 0xffff0000 ) >> 8)) / 16), 'a' + DISKPART(swap_dev)(((unsigned)((swap_dev) & 0xff) | (((swap_dev) & 0xffff0000 ) >> 8)) % 16)); |
1919 | else |
1920 | snprintf(path, sizeof(path), "blkdev0x%x", |
1921 | swap_dev); |
1922 | |
1923 | #if defined(NFSCLIENT1) |
1924 | gotit: |
1925 | #endif |
1926 | sdp = malloc(sizeof(*sdp), M_VMSWAP92, M_WAITOK0x0001|M_ZERO0x0008); |
1927 | spp = malloc(sizeof(*spp), M_VMSWAP92, M_WAITOK0x0001); |
1928 | |
1929 | sdp->swd_flagsswd_se.se_flags = SWF_FAKE0x00000008; |
1930 | sdp->swd_devswd_se.se_dev = swap_dev; |
1931 | |
1932 | sdp->swd_pathlen = strlen(path) + 1; |
1933 | sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP92, M_WAITOK0x0001 | M_ZERO0x0008); |
1934 | strlcpy(sdp->swd_path, path, sdp->swd_pathlen); |
1935 | |
1936 | sdp->swd_vp = vp; |
1937 | |
1938 | swaplist_insert(sdp, spp, 0); |
1939 | |
1940 | if (swap_on(curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc, sdp)) { |
1941 | swaplist_find(vp, 1); |
1942 | swaplist_trim(); |
1943 | vput(sdp->swd_vp); |
1944 | free(sdp->swd_path, M_VMSWAP92, sdp->swd_pathlen); |
1945 | free(sdp, M_VMSWAP92, sizeof(*sdp)); |
1946 | return; |
1947 | } |
1948 | } |
1949 | |
1950 | #ifdef HIBERNATE1 |
1951 | int |
1952 | uvm_hibswap(dev_t dev, u_long *sp, u_long *ep) |
1953 | { |
1954 | struct swapdev *sdp, *swd = NULL((void *)0); |
1955 | struct swappri *spp; |
1956 | struct extent_region *exr, *exrn; |
1957 | u_long start = 0, end = 0, size = 0; |
1958 | |
1959 | /* no swap devices configured yet? */ |
1960 | if (uvmexp.nswapdev < 1 || dev != swdevt[0].sw_dev) |
1961 | return (1); |
1962 | |
1963 | LIST_FOREACH(spp, &swap_priority, spi_swappri)for((spp) = ((&swap_priority)->lh_first); (spp)!= ((void *)0); (spp) = ((spp)->spi_swappri.le_next)) { |
1964 | TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)for((sdp) = ((&spp->spi_swapdev)->tqh_first); (sdp) != ((void *)0); (sdp) = ((sdp)->swd_next.tqe_next)) { |
1965 | if (sdp->swd_devswd_se.se_dev == dev) |
1966 | swd = sdp; |
1967 | } |
1968 | } |
1969 | |
1970 | if (swd == NULL((void *)0) || (swd->swd_flagsswd_se.se_flags & SWF_ENABLE0x00000002) == 0) |
1971 | return (1); |
1972 | |
1973 | LIST_FOREACH(exr, &swd->swd_ex->ex_regions, er_link)for((exr) = ((&swd->swd_ex->ex_regions)->lh_first ); (exr)!= ((void *)0); (exr) = ((exr)->er_link.le_next)) { |
1974 | u_long gapstart, gapend, gapsize; |
1975 | |
1976 | gapstart = exr->er_end + 1; |
1977 | exrn = LIST_NEXT(exr, er_link)((exr)->er_link.le_next); |
1978 | if (!exrn) |
1979 | break; |
1980 | gapend = exrn->er_start - 1; |
1981 | gapsize = gapend - gapstart; |
1982 | if (gapsize > size) { |
1983 | start = gapstart; |
1984 | end = gapend; |
1985 | size = gapsize; |
1986 | } |
1987 | } |
1988 | |
1989 | if (size) { |
1990 | *sp = start; |
1991 | *ep = end; |
1992 | return (0); |
1993 | } |
1994 | return (1); |
1995 | } |
1996 | #endif /* HIBERNATE */ |