File: | arch/amd64/amd64/pmap.c |
Warning: | line 1295, column 31 The left operand of '|' is a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* $OpenBSD: pmap.c,v 1.165 2023/12/29 13:23:27 jca Exp $ */ | |||
2 | /* $NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $ */ | |||
3 | ||||
4 | /* | |||
5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | |||
6 | * All rights reserved. | |||
7 | * | |||
8 | * Redistribution and use in source and binary forms, with or without | |||
9 | * modification, are permitted provided that the following conditions | |||
10 | * are met: | |||
11 | * 1. Redistributions of source code must retain the above copyright | |||
12 | * notice, this list of conditions and the following disclaimer. | |||
13 | * 2. Redistributions in binary form must reproduce the above copyright | |||
14 | * notice, this list of conditions and the following disclaimer in the | |||
15 | * documentation and/or other materials provided with the distribution. | |||
16 | * | |||
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | |||
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |||
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | |||
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |||
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | |||
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
27 | */ | |||
28 | ||||
29 | /* | |||
30 | * Copyright 2001 (c) Wasabi Systems, Inc. | |||
31 | * All rights reserved. | |||
32 | * | |||
33 | * Written by Frank van der Linden for Wasabi Systems, Inc. | |||
34 | * | |||
35 | * Redistribution and use in source and binary forms, with or without | |||
36 | * modification, are permitted provided that the following conditions | |||
37 | * are met: | |||
38 | * 1. Redistributions of source code must retain the above copyright | |||
39 | * notice, this list of conditions and the following disclaimer. | |||
40 | * 2. Redistributions in binary form must reproduce the above copyright | |||
41 | * notice, this list of conditions and the following disclaimer in the | |||
42 | * documentation and/or other materials provided with the distribution. | |||
43 | * 3. All advertising materials mentioning features or use of this software | |||
44 | * must display the following acknowledgement: | |||
45 | * This product includes software developed for the NetBSD Project by | |||
46 | * Wasabi Systems, Inc. | |||
47 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse | |||
48 | * or promote products derived from this software without specific prior | |||
49 | * written permission. | |||
50 | * | |||
51 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND | |||
52 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | |||
53 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |||
54 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC | |||
55 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |||
56 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |||
57 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |||
58 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |||
59 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |||
60 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |||
61 | * POSSIBILITY OF SUCH DAMAGE. | |||
62 | */ | |||
63 | ||||
64 | /* | |||
65 | * This is the i386 pmap modified and generalized to support x86-64 | |||
66 | * as well. The idea is to hide the upper N levels of the page tables | |||
67 | * inside pmap_get_ptp, pmap_free_ptp and pmap_growkernel. The rest | |||
68 | * is mostly untouched, except that it uses some more generalized | |||
69 | * macros and interfaces. | |||
70 | * | |||
71 | * This pmap has been tested on the i386 as well, and it can be easily | |||
72 | * adapted to PAE. | |||
73 | * | |||
74 | * fvdl@wasabisystems.com 18-Jun-2001 | |||
75 | */ | |||
76 | ||||
77 | /* | |||
78 | * pmap.c: i386 pmap module rewrite | |||
79 | * Chuck Cranor <chuck@ccrc.wustl.edu> | |||
80 | * 11-Aug-97 | |||
81 | * | |||
82 | * history of this pmap module: in addition to my own input, i used | |||
83 | * the following references for this rewrite of the i386 pmap: | |||
84 | * | |||
85 | * [1] the NetBSD i386 pmap. this pmap appears to be based on the | |||
86 | * BSD hp300 pmap done by Mike Hibler at University of Utah. | |||
87 | * it was then ported to the i386 by William Jolitz of UUNET | |||
88 | * Technologies, Inc. Then Charles M. Hannum of the NetBSD | |||
89 | * project fixed some bugs and provided some speed ups. | |||
90 | * | |||
91 | * [2] the FreeBSD i386 pmap. this pmap seems to be the | |||
92 | * Hibler/Jolitz pmap, as modified for FreeBSD by John S. Dyson | |||
93 | * and David Greenman. | |||
94 | * | |||
95 | * [3] the Mach pmap. this pmap, from CMU, seems to have migrated | |||
96 | * between several processors. the VAX version was done by | |||
97 | * Avadis Tevanian, Jr., and Michael Wayne Young. the i386 | |||
98 | * version was done by Lance Berc, Mike Kupfer, Bob Baron, | |||
99 | * David Golub, and Richard Draves. the alpha version was | |||
100 | * done by Alessandro Forin (CMU/Mach) and Chris Demetriou | |||
101 | * (NetBSD/alpha). | |||
102 | */ | |||
103 | ||||
104 | #include <sys/param.h> | |||
105 | #include <sys/systm.h> | |||
106 | #include <sys/atomic.h> | |||
107 | #include <sys/proc.h> | |||
108 | #include <sys/pool.h> | |||
109 | #include <sys/user.h> | |||
110 | #include <sys/mutex.h> | |||
111 | ||||
112 | #include <uvm/uvm.h> | |||
113 | ||||
114 | #include <machine/cpu.h> | |||
115 | #ifdef MULTIPROCESSOR1 | |||
116 | #include <machine/i82489reg.h> | |||
117 | #include <machine/i82489var.h> | |||
118 | #endif | |||
119 | ||||
120 | #include "vmm.h" | |||
121 | ||||
122 | #if NVMM1 > 0 | |||
123 | #include <machine/vmmvar.h> | |||
124 | #endif /* NVMM > 0 */ | |||
125 | ||||
126 | #include "acpi.h" | |||
127 | ||||
128 | /* #define PMAP_DEBUG */ | |||
129 | ||||
130 | #ifdef PMAP_DEBUG | |||
131 | #define DPRINTF(x...) do { printf(x); } while(0) | |||
132 | #else | |||
133 | #define DPRINTF(x...) | |||
134 | #endif /* PMAP_DEBUG */ | |||
135 | ||||
136 | ||||
137 | /* | |||
138 | * general info: | |||
139 | * | |||
140 | * - for an explanation of how the i386 MMU hardware works see | |||
141 | * the comments in <machine/pte.h>. | |||
142 | * | |||
143 | * - for an explanation of the general memory structure used by | |||
144 | * this pmap (including the recursive mapping), see the comments | |||
145 | * in <machine/pmap.h>. | |||
146 | * | |||
147 | * this file contains the code for the "pmap module." the module's | |||
148 | * job is to manage the hardware's virtual to physical address mappings. | |||
149 | * note that there are two levels of mapping in the VM system: | |||
150 | * | |||
151 | * [1] the upper layer of the VM system uses vm_map's and vm_map_entry's | |||
152 | * to map ranges of virtual address space to objects/files. for | |||
153 | * example, the vm_map may say: "map VA 0x1000 to 0x22000 read-only | |||
154 | * to the file /bin/ls starting at offset zero." note that | |||
155 | * the upper layer mapping is not concerned with how individual | |||
156 | * vm_pages are mapped. | |||
157 | * | |||
158 | * [2] the lower layer of the VM system (the pmap) maintains the mappings | |||
159 | * from virtual addresses. it is concerned with which vm_page is | |||
160 | * mapped where. for example, when you run /bin/ls and start | |||
161 | * at page 0x1000 the fault routine may lookup the correct page | |||
162 | * of the /bin/ls file and then ask the pmap layer to establish | |||
163 | * a mapping for it. | |||
164 | * | |||
165 | * note that information in the lower layer of the VM system can be | |||
166 | * thrown away since it can easily be reconstructed from the info | |||
167 | * in the upper layer. | |||
168 | * | |||
169 | * data structures we use include: | |||
170 | * - struct pmap: describes the address space of one process | |||
171 | * - struct pv_entry: describes one <PMAP,VA> mapping of a PA | |||
172 | * - struct pg_to_free: a list of virtual addresses whose mappings | |||
173 | * have been changed. used for TLB flushing. | |||
174 | */ | |||
175 | ||||
176 | /* | |||
177 | * memory allocation | |||
178 | * | |||
179 | * - there are three data structures that we must dynamically allocate: | |||
180 | * | |||
181 | * [A] new process' page directory page (PDP) | |||
182 | * - plan 1: done at pmap_create() we use | |||
183 | * pool_get(&pmap_pmap_pool, PR_WAITOK) to do this allocation. | |||
184 | * | |||
185 | * if we are low in free physical memory then we sleep in | |||
186 | * pool_get() -- in this case this is ok since we are creating | |||
187 | * a new pmap and should not be holding any locks. | |||
188 | * | |||
189 | * XXX: the fork code currently has no way to return an "out of | |||
190 | * memory, try again" error code since uvm_fork [fka vm_fork] | |||
191 | * is a void function. | |||
192 | * | |||
193 | * [B] new page tables pages (PTP) | |||
194 | * call uvm_pagealloc() | |||
195 | * => success: zero page, add to pm_pdir | |||
196 | * => failure: we are out of free vm_pages, let pmap_enter() | |||
197 | * tell UVM about it. | |||
198 | * | |||
199 | * note: for kernel PTPs, we start with NKPTP of them. as we map | |||
200 | * kernel memory (at uvm_map time) we check to see if we've grown | |||
201 | * the kernel pmap. if so, we call the optional function | |||
202 | * pmap_growkernel() to grow the kernel PTPs in advance. | |||
203 | * | |||
204 | * [C] pv_entry structures | |||
205 | * - try to allocate one from the pool. | |||
206 | * If we fail, we simply let pmap_enter() tell UVM about it. | |||
207 | */ | |||
208 | ||||
209 | long nkptp[] = NKPTP_INITIALIZER{ 0, 0, 0, 0 }; | |||
210 | ||||
211 | const vaddr_t ptp_masks[] = PTP_MASK_INITIALIZER{ (((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL), ((0x0000ff8000000000UL|0x0000007fc0000000UL )|0x000000003fe00000UL), (0x0000ff8000000000UL|0x0000007fc0000000UL ), 0x0000ff8000000000UL }; | |||
212 | const int ptp_shifts[] = PTP_SHIFT_INITIALIZER{ 12, 21, 30, 39 }; | |||
213 | const long nkptpmax[] = NKPTPMAX_INITIALIZER{ (unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512), (unsigned long)((unsigned long)((unsigned long)1 * 512) * 512), (unsigned long)((unsigned long)1 * 512 ), (unsigned long)1 }; | |||
214 | const long nbpd[] = NBPD_INITIALIZER{ (1ULL << 12), (1ULL << 21), (1ULL << 30), (1ULL << 39) }; | |||
215 | pd_entry_t *const normal_pdes[] = PDES_INITIALIZER{ ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))), ((pd_entry_t *)((char *) ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21 ))), ((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * ( 1ULL << 12))) }; | |||
216 | ||||
217 | #define pmap_pte_set(p, n)_atomic_swap_64((p), (n)) atomic_swap_64(p, n)_atomic_swap_64((p), (n)) | |||
218 | #define pmap_pte_clearbits(p, b)x86_atomic_clearbits_u64(p, b) x86_atomic_clearbits_u64(p, b) | |||
219 | #define pmap_pte_setbits(p, b)x86_atomic_setbits_u64(p, b) x86_atomic_setbits_u64(p, b) | |||
220 | ||||
221 | /* | |||
222 | * global data structures | |||
223 | */ | |||
224 | ||||
225 | struct pmap kernel_pmap_store; /* the kernel's pmap (proc0) */ | |||
226 | ||||
227 | /* | |||
228 | * pg_nx: NX PTE bit (if CPU supports) | |||
229 | * pg_g_kern: PG_G if global pages should be used in kernel mappings, | |||
230 | * 0 otherwise (for insecure CPUs) | |||
231 | */ | |||
232 | pt_entry_t pg_nx = 0; | |||
233 | pt_entry_t pg_g_kern = 0; | |||
234 | ||||
235 | /* pg_xo: XO PTE bits, set to PKU key1 (if cpu supports PKU) */ | |||
236 | pt_entry_t pg_xo; | |||
237 | ||||
238 | /* | |||
239 | * pmap_pg_wc: if our processor supports PAT then we set this | |||
240 | * to be the pte bits for Write Combining. Else we fall back to | |||
241 | * UC- so mtrrs can override the cacheability; | |||
242 | */ | |||
243 | int pmap_pg_wc = PG_UCMINUS(0x0000000000000010UL); | |||
244 | ||||
245 | /* | |||
246 | * pmap_use_pcid: nonzero if PCID use is enabled (currently we require INVPCID) | |||
247 | * | |||
248 | * The next three are zero unless and until PCID support is enabled so code | |||
249 | * can just 'or' them in as needed without tests. | |||
250 | * cr3_pcid: CR3_REUSE_PCID | |||
251 | * cr3_pcid_proc and cr3_pcid_temp: PCID_PROC and PCID_TEMP | |||
252 | */ | |||
253 | #if PCID_KERN0 != 0 | |||
254 | # error "pmap.c assumes PCID_KERN is zero" | |||
255 | #endif | |||
256 | int pmap_use_pcid; | |||
257 | static u_int cr3_pcid_proc; | |||
258 | static u_int cr3_pcid_temp; | |||
259 | /* these two are accessed from locore.o */ | |||
260 | paddr_t cr3_reuse_pcid; | |||
261 | paddr_t cr3_pcid_proc_intel; | |||
262 | ||||
263 | /* | |||
264 | * other data structures | |||
265 | */ | |||
266 | ||||
267 | pt_entry_t protection_codes[8]; /* maps MI prot to i386 prot code */ | |||
268 | int pmap_initialized = 0; /* pmap_init done yet? */ | |||
269 | ||||
270 | /* | |||
271 | * pv management structures. | |||
272 | */ | |||
273 | struct pool pmap_pv_pool; | |||
274 | ||||
275 | /* | |||
276 | * linked list of all non-kernel pmaps | |||
277 | */ | |||
278 | ||||
279 | struct pmap_head pmaps; | |||
280 | struct mutex pmaps_lock = MUTEX_INITIALIZER(IPL_VM){ ((void *)0), ((((0xa)) > 0x0 && ((0xa)) < 0x9 ) ? 0x9 : ((0xa))), 0x0 }; | |||
281 | ||||
282 | /* | |||
283 | * pool that pmap structures are allocated from | |||
284 | */ | |||
285 | ||||
286 | struct pool pmap_pmap_pool; | |||
287 | ||||
288 | /* | |||
289 | * When we're freeing a ptp, we need to delay the freeing until all | |||
290 | * tlb shootdown has been done. This is the list of the to-be-freed pages. | |||
291 | */ | |||
292 | TAILQ_HEAD(pg_to_free, vm_page)struct pg_to_free { struct vm_page *tqh_first; struct vm_page **tqh_last; }; | |||
293 | ||||
294 | /* | |||
295 | * pool that PDPs are allocated from | |||
296 | */ | |||
297 | ||||
298 | struct pool pmap_pdp_pool; | |||
299 | void pmap_pdp_ctor(pd_entry_t *); | |||
300 | void pmap_pdp_ctor_intel(pd_entry_t *); | |||
301 | ||||
302 | extern vaddr_t msgbuf_vaddr; | |||
303 | extern paddr_t msgbuf_paddr; | |||
304 | ||||
305 | extern vaddr_t idt_vaddr; /* we allocate IDT early */ | |||
306 | extern paddr_t idt_paddr; | |||
307 | ||||
308 | extern vaddr_t lo32_vaddr; | |||
309 | extern vaddr_t lo32_paddr; | |||
310 | ||||
311 | vaddr_t virtual_avail; | |||
312 | extern int end; | |||
313 | ||||
314 | /* | |||
315 | * local prototypes | |||
316 | */ | |||
317 | ||||
318 | void pmap_enter_pv(struct vm_page *, struct pv_entry *, struct pmap *, | |||
319 | vaddr_t, struct vm_page *); | |||
320 | struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t); | |||
321 | struct vm_page *pmap_find_ptp(struct pmap *, vaddr_t, paddr_t, int); | |||
322 | int pmap_find_pte_direct(struct pmap *pm, vaddr_t va, pt_entry_t **pd, int *offs); | |||
323 | void pmap_free_ptp(struct pmap *, struct vm_page *, | |||
324 | vaddr_t, struct pg_to_free *); | |||
325 | void pmap_freepage(struct pmap *, struct vm_page *, int, struct pg_to_free *); | |||
326 | #ifdef MULTIPROCESSOR1 | |||
327 | static int pmap_is_active(struct pmap *, struct cpu_info *); | |||
328 | #endif | |||
329 | paddr_t pmap_map_ptes(struct pmap *); | |||
330 | struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t); | |||
331 | void pmap_do_remove(struct pmap *, vaddr_t, vaddr_t, int); | |||
332 | void pmap_remove_ept(struct pmap *, vaddr_t, vaddr_t); | |||
333 | void pmap_do_remove_ept(struct pmap *, vaddr_t); | |||
334 | int pmap_enter_ept(struct pmap *, vaddr_t, paddr_t, vm_prot_t); | |||
335 | int pmap_remove_pte(struct pmap *, struct vm_page *, pt_entry_t *, | |||
336 | vaddr_t, int, struct pv_entry **); | |||
337 | void pmap_remove_ptes(struct pmap *, struct vm_page *, vaddr_t, | |||
338 | vaddr_t, vaddr_t, int, struct pv_entry **); | |||
339 | #define PMAP_REMOVE_ALL0 0 /* remove all mappings */ | |||
340 | #define PMAP_REMOVE_SKIPWIRED1 1 /* skip wired mappings */ | |||
341 | ||||
342 | void pmap_unmap_ptes(struct pmap *, paddr_t); | |||
343 | int pmap_get_physpage(vaddr_t, int, paddr_t *); | |||
344 | int pmap_pdes_valid(vaddr_t, pd_entry_t *); | |||
345 | void pmap_alloc_level(vaddr_t, int, long *); | |||
346 | ||||
347 | static inline | |||
348 | void pmap_sync_flags_pte(struct vm_page *, u_long); | |||
349 | ||||
350 | void pmap_tlb_shootpage(struct pmap *, vaddr_t, int); | |||
351 | void pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t, int); | |||
352 | void pmap_tlb_shoottlb(struct pmap *, int); | |||
353 | #ifdef MULTIPROCESSOR1 | |||
354 | void pmap_tlb_shootwait(void); | |||
355 | #else | |||
356 | #define pmap_tlb_shootwait() do { } while (0) | |||
357 | #endif | |||
358 | ||||
359 | /* | |||
360 | * p m a p i n l i n e h e l p e r f u n c t i o n s | |||
361 | */ | |||
362 | ||||
363 | /* | |||
364 | * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]? | |||
365 | * of course the kernel is always loaded | |||
366 | */ | |||
367 | ||||
368 | static inline int | |||
369 | pmap_is_curpmap(struct pmap *pmap) | |||
370 | { | |||
371 | return((pmap == pmap_kernel()(&kernel_pmap_store)) || | |||
372 | (pmap->pm_pdirpa == (rcr3() & CR3_PADDR0x7ffffffffffff000ULL))); | |||
373 | } | |||
374 | ||||
375 | /* | |||
376 | * pmap_is_active: is this pmap loaded into the specified processor's %cr3? | |||
377 | */ | |||
378 | ||||
379 | #ifdef MULTIPROCESSOR1 | |||
380 | static inline int | |||
381 | pmap_is_active(struct pmap *pmap, struct cpu_info *ci) | |||
382 | { | |||
383 | return pmap == pmap_kernel()(&kernel_pmap_store) || pmap == ci->ci_proc_pmap; | |||
384 | } | |||
385 | #endif | |||
386 | ||||
387 | static inline u_int | |||
388 | pmap_pte2flags(u_long pte) | |||
389 | { | |||
390 | return (((pte & PG_U0x0000000000000020UL) ? PG_PMAP_REF0x02000000 : 0) | | |||
391 | ((pte & PG_M0x0000000000000040UL) ? PG_PMAP_MOD0x01000000 : 0)); | |||
392 | } | |||
393 | ||||
394 | static inline void | |||
395 | pmap_sync_flags_pte(struct vm_page *pg, u_long pte) | |||
396 | { | |||
397 | if (pte & (PG_U0x0000000000000020UL|PG_M0x0000000000000040UL)) { | |||
398 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, pmap_pte2flags(pte)); | |||
399 | } | |||
400 | } | |||
401 | ||||
402 | /* | |||
403 | * pmap_map_ptes: map a pmap's PTEs into KVM | |||
404 | * | |||
405 | * This should not be done for EPT pmaps | |||
406 | */ | |||
407 | paddr_t | |||
408 | pmap_map_ptes(struct pmap *pmap) | |||
409 | { | |||
410 | paddr_t cr3; | |||
411 | ||||
412 | KASSERT(pmap->pm_type != PMAP_TYPE_EPT)((pmap->pm_type != 2) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c" , 412, "pmap->pm_type != PMAP_TYPE_EPT")); | |||
413 | ||||
414 | /* the kernel's pmap is always accessible */ | |||
415 | if (pmap == pmap_kernel()(&kernel_pmap_store)) | |||
416 | return 0; | |||
417 | ||||
418 | /* | |||
419 | * Lock the target map before switching to its page tables to | |||
420 | * guarantee other CPUs have finished changing the tables before | |||
421 | * we potentially start caching table and TLB entries. | |||
422 | */ | |||
423 | mtx_enter(&pmap->pm_mtx); | |||
424 | ||||
425 | cr3 = rcr3(); | |||
426 | KASSERT((cr3 & CR3_PCID) == PCID_KERN ||(((cr3 & 0xfffULL) == 0 || (cr3 & 0xfffULL) == 1) ? ( void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c" , 427, "(cr3 & CR3_PCID) == PCID_KERN || (cr3 & CR3_PCID) == PCID_PROC" )) | |||
427 | (cr3 & CR3_PCID) == PCID_PROC)(((cr3 & 0xfffULL) == 0 || (cr3 & 0xfffULL) == 1) ? ( void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c" , 427, "(cr3 & CR3_PCID) == PCID_KERN || (cr3 & CR3_PCID) == PCID_PROC" )); | |||
428 | if (pmap->pm_pdirpa == (cr3 & CR3_PADDR0x7ffffffffffff000ULL)) | |||
429 | cr3 = 0; | |||
430 | else { | |||
431 | cr3 |= cr3_reuse_pcid; | |||
432 | lcr3(pmap->pm_pdirpa | cr3_pcid_temp); | |||
433 | } | |||
434 | ||||
435 | return cr3; | |||
436 | } | |||
437 | ||||
438 | void | |||
439 | pmap_unmap_ptes(struct pmap *pmap, paddr_t save_cr3) | |||
440 | { | |||
441 | if (pmap != pmap_kernel()(&kernel_pmap_store)) | |||
442 | mtx_leave(&pmap->pm_mtx); | |||
443 | ||||
444 | if (save_cr3 != 0) | |||
445 | lcr3(save_cr3); | |||
446 | } | |||
447 | ||||
448 | int | |||
449 | pmap_find_pte_direct(struct pmap *pm, vaddr_t va, pt_entry_t **pd, int *offs) | |||
450 | { | |||
451 | u_long mask, shift; | |||
452 | pd_entry_t pde; | |||
453 | paddr_t pdpa; | |||
454 | int lev; | |||
455 | ||||
456 | pdpa = pm->pm_pdirpa; | |||
457 | shift = L4_SHIFT39; | |||
458 | mask = L4_MASK0x0000ff8000000000UL; | |||
459 | for (lev = PTP_LEVELS4; lev > 0; lev--) { | |||
460 | *pd = (pd_entry_t *)PMAP_DIRECT_MAP(pdpa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (pdpa)); | |||
461 | *offs = (VA_SIGN_POS(va)((va) & ~0xffff000000000000) & mask) >> shift; | |||
462 | pde = (*pd)[*offs]; | |||
463 | ||||
464 | /* Large pages are different, break early if we run into one. */ | |||
465 | if ((pde & (PG_PS0x0000000000000080UL|PG_V0x0000000000000001UL)) != PG_V0x0000000000000001UL) | |||
466 | return (lev - 1); | |||
467 | ||||
468 | pdpa = ((*pd)[*offs] & PG_FRAME0x000ffffffffff000UL); | |||
469 | /* 4096/8 == 512 == 2^9 entries per level */ | |||
470 | shift -= 9; | |||
471 | mask >>= 9; | |||
472 | } | |||
473 | ||||
474 | return (0); | |||
475 | } | |||
476 | ||||
477 | /* | |||
478 | * p m a p k e n t e r f u n c t i o n s | |||
479 | * | |||
480 | * functions to quickly enter/remove pages from the kernel address | |||
481 | * space. pmap_kremove is exported to MI kernel. we make use of | |||
482 | * the recursive PTE mappings. | |||
483 | */ | |||
484 | ||||
485 | /* | |||
486 | * pmap_kenter_pa: enter a kernel mapping without R/M (pv_entry) tracking | |||
487 | * | |||
488 | * => no need to lock anything, assume va is already allocated | |||
489 | * => should be faster than normal pmap enter function | |||
490 | */ | |||
491 | ||||
492 | void | |||
493 | pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) | |||
494 | { | |||
495 | pt_entry_t *pte, opte, npte; | |||
496 | ||||
497 | pte = kvtopte(va); | |||
498 | ||||
499 | npte = (pa & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1))) | ((prot & PROT_WRITE0x02) ? PG_RW0x0000000000000002UL : PG_RO0x0000000000000000UL) | | |||
500 | ((pa & PMAP_NOCACHE0x1) ? PG_N0x0000000000000010UL : 0) | | |||
501 | ((pa & PMAP_WC0x2) ? pmap_pg_wc : 0) | PG_V0x0000000000000001UL; | |||
502 | ||||
503 | /* special 1:1 mappings in the first 2MB must not be global */ | |||
504 | if (va >= (vaddr_t)NBPD_L2(1ULL << 21)) | |||
505 | npte |= pg_g_kern; | |||
506 | ||||
507 | if (!(prot & PROT_EXEC0x04)) | |||
508 | npte |= pg_nx; | |||
509 | opte = pmap_pte_set(pte, npte)_atomic_swap_64((pte), (npte)); | |||
510 | #ifdef LARGEPAGES | |||
511 | /* XXX For now... */ | |||
512 | if (opte & PG_PS0x0000000000000080UL) | |||
513 | panic("%s: PG_PS", __func__); | |||
514 | #endif | |||
515 | if (pmap_valid_entry(opte)((opte) & 0x0000000000000001UL)) { | |||
516 | if (pa & PMAP_NOCACHE0x1 && (opte & PG_N0x0000000000000010UL) == 0) | |||
517 | wbinvd_on_all_cpus(); | |||
518 | /* This shouldn't happen */ | |||
519 | pmap_tlb_shootpage(pmap_kernel()(&kernel_pmap_store), va, 1); | |||
520 | pmap_tlb_shootwait(); | |||
521 | } | |||
522 | } | |||
523 | ||||
524 | /* | |||
525 | * pmap_kremove: remove a kernel mapping(s) without R/M (pv_entry) tracking | |||
526 | * | |||
527 | * => no need to lock anything | |||
528 | * => caller must dispose of any vm_page mapped in the va range | |||
529 | * => note: not an inline function | |||
530 | * => we assume the va is page aligned and the len is a multiple of PAGE_SIZE | |||
531 | * => we assume kernel only unmaps valid addresses and thus don't bother | |||
532 | * checking the valid bit before doing TLB flushing | |||
533 | */ | |||
534 | ||||
535 | void | |||
536 | pmap_kremove(vaddr_t sva, vsize_t len) | |||
537 | { | |||
538 | pt_entry_t *pte, opte; | |||
539 | vaddr_t va, eva; | |||
540 | ||||
541 | eva = sva + len; | |||
542 | ||||
543 | for (va = sva; va != eva; va += PAGE_SIZE(1 << 12)) { | |||
544 | pte = kvtopte(va); | |||
545 | ||||
546 | opte = pmap_pte_set(pte, 0)_atomic_swap_64((pte), (0)); | |||
547 | #ifdef LARGEPAGES | |||
548 | KASSERT((opte & PG_PS) == 0)(((opte & 0x0000000000000080UL) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c", 548, "(opte & PG_PS) == 0" )); | |||
549 | #endif | |||
550 | KASSERT((opte & PG_PVLIST) == 0)(((opte & 0x0000000000000400UL) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c", 550, "(opte & PG_PVLIST) == 0" )); | |||
551 | } | |||
552 | ||||
553 | pmap_tlb_shootrange(pmap_kernel()(&kernel_pmap_store), sva, eva, 1); | |||
554 | pmap_tlb_shootwait(); | |||
555 | } | |||
556 | ||||
557 | /* | |||
558 | * pmap_set_pml4_early | |||
559 | * | |||
560 | * Utility function to map 2GB of 2MB pages to 'pa'. The VA that is assigned | |||
561 | * is the pml4 entry for 'early mappings' (see pmap.h). This function is used | |||
562 | * by display drivers that need to map their framebuffers early, before the | |||
563 | * pmap is fully initialized (eg, to show panic messages). | |||
564 | * | |||
565 | * Users of this function must call pmap_clear_pml4_early to remove the | |||
566 | * mapping when finished. | |||
567 | * | |||
568 | * Parameters: | |||
569 | * pa: phys addr to map | |||
570 | * | |||
571 | * Return value: | |||
572 | * VA mapping to 'pa'. This mapping is 2GB in size and starts at the base | |||
573 | * of the 2MB region containing 'va'. | |||
574 | */ | |||
575 | vaddr_t | |||
576 | pmap_set_pml4_early(paddr_t pa) | |||
577 | { | |||
578 | extern paddr_t early_pte_pages; | |||
579 | pt_entry_t *pml4e, *pte; | |||
580 | int i, j, off; | |||
581 | paddr_t curpa; | |||
582 | vaddr_t va; | |||
583 | ||||
584 | pml4e = (pt_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE0xffffffff80000000); | |||
585 | pml4e[PDIR_SLOT_EARLY((511 - 4) - 1)] = (pd_entry_t)early_pte_pages | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL; | |||
586 | ||||
587 | off = pa & PAGE_MASK_L2((1ULL << 21) - 1); | |||
588 | curpa = pa & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL ); | |||
589 | ||||
590 | pte = (pt_entry_t *)PMAP_DIRECT_MAP(early_pte_pages)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (early_pte_pages)); | |||
591 | memset(pte, 0, 3 * NBPG)__builtin_memset((pte), (0), (3 * (1 << 12))); | |||
592 | ||||
593 | pte[0] = (early_pte_pages + NBPG(1 << 12)) | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL; | |||
594 | pte[1] = (early_pte_pages + 2 * NBPG(1 << 12)) | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL; | |||
595 | ||||
596 | pte = (pt_entry_t *)PMAP_DIRECT_MAP(early_pte_pages + NBPG)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (early_pte_pages + (1 << 12))); | |||
597 | for (i = 0; i < 2; i++) { | |||
598 | /* 2 early pages of mappings */ | |||
599 | for (j = 0; j < 512; j++) { | |||
600 | /* j[0..511] : 2MB mappings per page */ | |||
601 | pte[(i * 512) + j] = curpa | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL | PG_PS0x0000000000000080UL; | |||
602 | curpa += (2 * 1024 * 1024); | |||
603 | } | |||
604 | } | |||
605 | ||||
606 | va = (vaddr_t)((PDIR_SLOT_EARLY((511 - 4) - 1) * 512ULL) << L3_SHIFT30) + off; | |||
607 | return VA_SIGN_NEG(va)((va) | 0xffff000000000000); | |||
608 | } | |||
609 | ||||
610 | /* | |||
611 | * pmap_clear_pml4_early | |||
612 | * | |||
613 | * Clears the mapping previously established with pmap_set_pml4_early. | |||
614 | */ | |||
615 | void | |||
616 | pmap_clear_pml4_early(void) | |||
617 | { | |||
618 | extern paddr_t early_pte_pages; | |||
619 | pt_entry_t *pml4e, *pte; | |||
620 | ||||
621 | pte = (pt_entry_t *)PMAP_DIRECT_MAP(early_pte_pages)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (early_pte_pages)); | |||
622 | memset(pte, 0, 3 * NBPG)__builtin_memset((pte), (0), (3 * (1 << 12))); | |||
623 | ||||
624 | pml4e = (pd_entry_t *)pmap_kernel()(&kernel_pmap_store)->pm_pdir; | |||
625 | pml4e[PDIR_SLOT_EARLY((511 - 4) - 1)] = 0; | |||
626 | tlbflush(); | |||
627 | } | |||
628 | ||||
629 | /* | |||
630 | * p m a p i n i t f u n c t i o n s | |||
631 | * | |||
632 | * pmap_bootstrap and pmap_init are called during system startup | |||
633 | * to init the pmap module. pmap_bootstrap() does a low level | |||
634 | * init just to get things rolling. pmap_init() finishes the job. | |||
635 | */ | |||
636 | ||||
637 | /* | |||
638 | * pmap_bootstrap: get the system in a state where it can run with VM | |||
639 | * properly enabled (called before main()). the VM system is | |||
640 | * fully init'd later... | |||
641 | */ | |||
642 | ||||
643 | paddr_t | |||
644 | pmap_bootstrap(paddr_t first_avail, paddr_t max_pa) | |||
645 | { | |||
646 | vaddr_t kva_start = VM_MIN_KERNEL_ADDRESS0xffff800000000000; | |||
647 | struct pmap *kpm; | |||
648 | int curslot, i, j, p; | |||
649 | long ndmpdp; | |||
650 | paddr_t dmpd, dmpdp, start_cur, cur_pa; | |||
651 | vaddr_t kva, kva_end; | |||
652 | pt_entry_t *pml3, *pml2; | |||
653 | ||||
654 | /* | |||
655 | * define the boundaries of the managed kernel virtual address | |||
656 | * space. | |||
657 | */ | |||
658 | ||||
659 | virtual_avail = kva_start; /* first free KVA */ | |||
660 | ||||
661 | /* | |||
662 | * If PKU is available, initialize PROT_EXEC entry correctly, | |||
663 | * and enable the feature before it gets used | |||
664 | * XXX Some Hypervisors forget to save/restore PKU | |||
665 | */ | |||
666 | if (cpuid_level >= 0x7) { | |||
667 | uint32_t ecx, dummy; | |||
668 | ||||
669 | CPUID_LEAF(0x7, 0, dummy, dummy, ecx, dummy)__asm volatile("cpuid" : "=a" (dummy), "=b" (dummy), "=c" (ecx ), "=d" (dummy) : "a" (0x7), "c" (0)); | |||
670 | if (ecx & SEFF0ECX_PKU0x00000008) { | |||
671 | lcr4(rcr4() | CR4_PKE0x00400000); | |||
672 | pg_xo = PG_XO0x0800000000000000UL; | |||
673 | } | |||
674 | } | |||
675 | ||||
676 | /* | |||
677 | * set up protection_codes: we need to be able to convert from | |||
678 | * a MI protection code (some combo of VM_PROT...) to something | |||
679 | * we can jam into a i386 PTE. | |||
680 | */ | |||
681 | ||||
682 | protection_codes[PROT_NONE0x00] = pg_nx; /* --- */ | |||
683 | protection_codes[PROT_EXEC0x04] = pg_xo; ; /* --x */ | |||
684 | protection_codes[PROT_READ0x01] = PG_RO0x0000000000000000UL | pg_nx; /* -r- */ | |||
685 | protection_codes[PROT_READ0x01 | PROT_EXEC0x04] = PG_RO0x0000000000000000UL; /* -rx */ | |||
686 | protection_codes[PROT_WRITE0x02] = PG_RW0x0000000000000002UL | pg_nx; /* w-- */ | |||
687 | protection_codes[PROT_WRITE0x02 | PROT_EXEC0x04] = PG_RW0x0000000000000002UL; /* w-x */ | |||
688 | protection_codes[PROT_WRITE0x02 | PROT_READ0x01] = PG_RW0x0000000000000002UL | pg_nx; /* wr- */ | |||
689 | protection_codes[PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04] = PG_RW0x0000000000000002UL; /* wrx */ | |||
690 | ||||
691 | /* | |||
692 | * now we init the kernel's pmap | |||
693 | * | |||
694 | * the kernel pmap's pm_obj is not used for much. however, in | |||
695 | * user pmaps the pm_obj contains the list of active PTPs. | |||
696 | * the pm_obj currently does not have a pager. | |||
697 | */ | |||
698 | ||||
699 | kpm = pmap_kernel()(&kernel_pmap_store); | |||
700 | for (i = 0; i < PTP_LEVELS4 - 1; i++) { | |||
701 | uvm_obj_init(&kpm->pm_obj[i], &pmap_pager, 1); | |||
702 | kpm->pm_ptphint[i] = NULL((void *)0); | |||
703 | } | |||
704 | memset(&kpm->pm_list, 0, sizeof(kpm->pm_list))__builtin_memset((&kpm->pm_list), (0), (sizeof(kpm-> pm_list))); /* pm_list not used */ | |||
705 | kpm->pm_pdir = (pd_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE0xffffffff80000000); | |||
706 | kpm->pm_pdirpa = proc0.p_addr->u_pcb.pcb_cr3; | |||
707 | kpm->pm_stats.wired_count = kpm->pm_stats.resident_count = | |||
708 | atop(kva_start - VM_MIN_KERNEL_ADDRESS)((kva_start - 0xffff800000000000) >> 12); | |||
709 | /* | |||
710 | * the above is just a rough estimate and not critical to the proper | |||
711 | * operation of the system. | |||
712 | */ | |||
713 | ||||
714 | kpm->pm_type = PMAP_TYPE_NORMAL1; | |||
715 | ||||
716 | curpcb({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curpcb->pcb_pmap = kpm; /* proc0's pcb */ | |||
717 | ||||
718 | /* | |||
719 | * Configure and enable PCID use if supported. | |||
720 | * Currently we require INVPCID support. | |||
721 | */ | |||
722 | if ((cpu_ecxfeature & CPUIDECX_PCID0x00020000) && cpuid_level >= 0x07) { | |||
723 | uint32_t ebx, dummy; | |||
724 | CPUID_LEAF(0x7, 0, dummy, ebx, dummy, dummy)__asm volatile("cpuid" : "=a" (dummy), "=b" (ebx), "=c" (dummy ), "=d" (dummy) : "a" (0x7), "c" (0)); | |||
725 | if (ebx & SEFF0EBX_INVPCID0x00000400) { | |||
726 | pmap_use_pcid = 1; | |||
727 | /* | |||
728 | * We cannot use global mappings because | |||
729 | * invpcid function 0 does not invalidate global | |||
730 | * mappings. The hardware can cache kernel | |||
731 | * mappings based on PCID_KERN, i.e. there is no | |||
732 | * need for global mappings. | |||
733 | */ | |||
734 | pg_g_kern = 0; | |||
735 | lcr4( rcr4() | CR4_PCIDE0x00020000 ); | |||
736 | cr3_pcid_proc = PCID_PROC1; | |||
737 | cr3_pcid_temp = PCID_TEMP3; | |||
738 | cr3_reuse_pcid = CR3_REUSE_PCID(1ULL << 63); | |||
739 | cr3_pcid_proc_intel = PCID_PROC_INTEL2; | |||
740 | } | |||
741 | } | |||
742 | ||||
743 | /* | |||
744 | * Add PG_G attribute to already mapped kernel pages. pg_g_kern | |||
745 | * is calculated in locore0.S and may be set to: | |||
746 | * | |||
747 | * 0 if this CPU does not safely support global pages in the kernel | |||
748 | * (Intel/Meltdown) | |||
749 | * PG_G if this CPU does safely support global pages in the kernel | |||
750 | * (AMD) | |||
751 | */ | |||
752 | #if KERNBASE0xffffffff80000000 == VM_MIN_KERNEL_ADDRESS0xffff800000000000 | |||
753 | for (kva = VM_MIN_KERNEL_ADDRESS0xffff800000000000 ; kva < virtual_avail ; | |||
754 | #else | |||
755 | kva_end = roundup((vaddr_t)&end, PAGE_SIZE)(((((vaddr_t)&end)+(((1 << 12))-1))/((1 << 12 )))*((1 << 12))); | |||
756 | for (kva = KERNBASE0xffffffff80000000; kva < kva_end ; | |||
757 | #endif | |||
758 | kva += PAGE_SIZE(1 << 12)) { | |||
759 | unsigned long p1i = pl1_i(kva)(((((kva) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12); | |||
760 | if (pmap_valid_entry(PTE_BASE[p1i])((((pt_entry_t *) (255 * (1ULL << 39)))[p1i]) & 0x0000000000000001UL )) | |||
761 | PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[p1i] |= pg_g_kern; | |||
762 | } | |||
763 | ||||
764 | /* | |||
765 | * Map the direct map. The first 4GB were mapped in locore, here | |||
766 | * we map the rest if it exists. We actually use the direct map | |||
767 | * here to set up the page tables, we're assuming that we're still | |||
768 | * operating in the lower 4GB of memory. | |||
769 | * | |||
770 | * Map (up to) the first 512GB of physical memory first. This part | |||
771 | * is handled differently than physical memory > 512GB since we have | |||
772 | * already mapped part of this range in locore0. | |||
773 | */ | |||
774 | ndmpdp = (max_pa + NBPD_L3(1ULL << 30) - 1) >> L3_SHIFT30; | |||
775 | if (ndmpdp < NDML2_ENTRIES4) | |||
776 | ndmpdp = NDML2_ENTRIES4; /* At least 4GB */ | |||
777 | if (ndmpdp > 512) | |||
778 | ndmpdp = 512; /* At most 512GB */ | |||
779 | ||||
780 | dmpdp = kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4)] & PG_FRAME0x000ffffffffff000UL; | |||
781 | ||||
782 | dmpd = first_avail; first_avail += ndmpdp * PAGE_SIZE(1 << 12); | |||
783 | ||||
784 | for (i = NDML2_ENTRIES4; i < NPDPG((1 << 12) / sizeof (pd_entry_t)) * ndmpdp; i++) { | |||
785 | paddr_t pdp; | |||
786 | vaddr_t va; | |||
787 | ||||
788 | pdp = (paddr_t)&(((pd_entry_t *)dmpd)[i]); | |||
789 | va = PMAP_DIRECT_MAP(pdp)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (pdp)); | |||
790 | ||||
791 | *((pd_entry_t *)va) = ((paddr_t)i << L2_SHIFT21); | |||
792 | *((pd_entry_t *)va) |= PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_PS0x0000000000000080UL | pg_g_kern | PG_U0x0000000000000020UL | | |||
793 | PG_M0x0000000000000040UL | pg_nx; | |||
794 | } | |||
795 | ||||
796 | for (i = NDML2_ENTRIES4; i < ndmpdp; i++) { | |||
797 | paddr_t pdp; | |||
798 | vaddr_t va; | |||
799 | ||||
800 | pdp = (paddr_t)&(((pd_entry_t *)dmpdp)[i]); | |||
801 | va = PMAP_DIRECT_MAP(pdp)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (pdp)); | |||
802 | ||||
803 | *((pd_entry_t *)va) = dmpd + (i << PAGE_SHIFT12); | |||
804 | *((pd_entry_t *)va) |= PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx; | |||
805 | } | |||
806 | ||||
807 | kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4)] = dmpdp | PG_V0x0000000000000001UL | PG_KW0x0000000000000002UL | PG_U0x0000000000000020UL | | |||
808 | PG_M0x0000000000000040UL | pg_nx; | |||
809 | ||||
810 | /* Map any remaining physical memory > 512GB */ | |||
811 | for (curslot = 1 ; curslot < NUM_L4_SLOT_DIRECT4 ; curslot++) { | |||
812 | /* | |||
813 | * Start of current range starts at PA (curslot) * 512GB | |||
814 | */ | |||
815 | start_cur = (paddr_t)(curslot * NBPD_L4(1ULL << 39)); | |||
816 | if (max_pa > start_cur) { | |||
817 | /* Next 512GB, new PML4e and L3(512GB) page */ | |||
818 | dmpd = first_avail; first_avail += PAGE_SIZE(1 << 12); | |||
819 | pml3 = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (dmpd)); | |||
820 | kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4) + curslot] = dmpd | | |||
821 | PG_KW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx; | |||
822 | ||||
823 | /* Calculate full 1GB pages in this 512GB region */ | |||
824 | p = ((max_pa - start_cur) >> L3_SHIFT30); | |||
825 | ||||
826 | /* Check if a partial (<1GB) page remains */ | |||
827 | if (max_pa & L2_MASK0x000000003fe00000UL) | |||
828 | p++; | |||
829 | ||||
830 | /* | |||
831 | * Handle the case where this range is full and there | |||
832 | * is still more memory after (p would be > 512). | |||
833 | */ | |||
834 | if (p > NPDPG((1 << 12) / sizeof (pd_entry_t))) | |||
835 | p = NPDPG((1 << 12) / sizeof (pd_entry_t)); | |||
836 | ||||
837 | /* Allocate 'p' L2(1GB) pages and populate */ | |||
838 | for (i = 0; i < p; i++) { | |||
839 | dmpd = first_avail; first_avail += PAGE_SIZE(1 << 12); | |||
840 | pml2 = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (dmpd)); | |||
841 | pml3[i] = dmpd | | |||
842 | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx; | |||
843 | ||||
844 | cur_pa = start_cur + (i << L3_SHIFT30); | |||
845 | j = 0; | |||
846 | ||||
847 | while (cur_pa < max_pa && j < NPDPG((1 << 12) / sizeof (pd_entry_t))) { | |||
848 | pml2[j] = curslot * NBPD_L4(1ULL << 39) + | |||
849 | (uint64_t)i * NBPD_L3(1ULL << 30) + | |||
850 | (uint64_t)j * NBPD_L2(1ULL << 21); | |||
851 | pml2[j] |= PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | pg_g_kern | | |||
852 | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx | PG_PS0x0000000000000080UL; | |||
853 | cur_pa += NBPD_L2(1ULL << 21); | |||
854 | j++; | |||
855 | } | |||
856 | } | |||
857 | } | |||
858 | } | |||
859 | ||||
860 | tlbflush(); | |||
861 | ||||
862 | msgbuf_vaddr = virtual_avail; | |||
863 | virtual_avail += round_page(MSGBUFSIZE)((((32 * (1 << 12))) + ((1 << 12) - 1)) & ~(( 1 << 12) - 1)); | |||
864 | ||||
865 | idt_vaddr = virtual_avail; | |||
866 | virtual_avail += 2 * PAGE_SIZE(1 << 12); | |||
867 | idt_paddr = first_avail; /* steal a page */ | |||
868 | first_avail += 2 * PAGE_SIZE(1 << 12); | |||
869 | ||||
870 | #if defined(MULTIPROCESSOR1) || \ | |||
871 | (NACPI1 > 0 && !defined(SMALL_KERNEL)) | |||
872 | /* | |||
873 | * Grab a page below 4G for things that need it (i.e. | |||
874 | * having an initial %cr3 for the MP trampoline). | |||
875 | */ | |||
876 | lo32_vaddr = virtual_avail; | |||
877 | virtual_avail += PAGE_SIZE(1 << 12); | |||
878 | lo32_paddr = first_avail; | |||
879 | first_avail += PAGE_SIZE(1 << 12); | |||
880 | #endif | |||
881 | ||||
882 | /* | |||
883 | * init the global lists. | |||
884 | */ | |||
885 | LIST_INIT(&pmaps)do { ((&pmaps)->lh_first) = ((void *)0); } while (0); | |||
886 | ||||
887 | /* | |||
888 | * initialize the pmap pools. | |||
889 | */ | |||
890 | ||||
891 | pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, IPL_VM0xa, 0, | |||
892 | "pmappl", NULL((void *)0)); | |||
893 | pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, IPL_VM0xa, 0, | |||
894 | "pvpl", &pool_allocator_single); | |||
895 | pool_sethiwat(&pmap_pv_pool, 32 * 1024); | |||
896 | ||||
897 | /* | |||
898 | * initialize the PDE pool. | |||
899 | */ | |||
900 | ||||
901 | pool_init(&pmap_pdp_pool, PAGE_SIZE(1 << 12), 0, IPL_VM0xa, 0, | |||
902 | "pdppl", &pool_allocator_single); | |||
903 | ||||
904 | kpm->pm_pdir_intel = NULL((void *)0); | |||
905 | kpm->pm_pdirpa_intel = 0; | |||
906 | ||||
907 | /* | |||
908 | * ensure the TLB is sync'd with reality by flushing it... | |||
909 | */ | |||
910 | ||||
911 | tlbflush(); | |||
912 | ||||
913 | return first_avail; | |||
914 | } | |||
915 | ||||
916 | void | |||
917 | pmap_init_percpu(void) | |||
918 | { | |||
919 | pool_cache_init(&pmap_pv_pool); | |||
920 | } | |||
921 | ||||
922 | /* | |||
923 | * pmap_randomize | |||
924 | * | |||
925 | * Randomizes the location of the kernel pmap | |||
926 | */ | |||
927 | void | |||
928 | pmap_randomize(void) | |||
929 | { | |||
930 | pd_entry_t *pml4va, *oldpml4va; | |||
931 | paddr_t pml4pa; | |||
932 | int i; | |||
933 | ||||
934 | pml4va = km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero, &kd_nowait); | |||
935 | if (pml4va == NULL((void *)0)) | |||
936 | panic("%s: km_alloc failed", __func__); | |||
937 | ||||
938 | /* Copy old PML4 page to new one */ | |||
939 | oldpml4va = pmap_kernel()(&kernel_pmap_store)->pm_pdir; | |||
940 | memcpy(pml4va, oldpml4va, PAGE_SIZE)__builtin_memcpy((pml4va), (oldpml4va), ((1 << 12))); | |||
941 | ||||
942 | /* Switch to new PML4 */ | |||
943 | pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)pml4va, &pml4pa); | |||
944 | lcr3(pml4pa); | |||
945 | ||||
946 | /* Fixup pmap_kernel and proc0's %cr3 */ | |||
947 | pmap_kernel()(&kernel_pmap_store)->pm_pdirpa = pml4pa; | |||
948 | pmap_kernel()(&kernel_pmap_store)->pm_pdir = pml4va; | |||
949 | proc0.p_addr->u_pcb.pcb_cr3 = pml4pa; | |||
950 | ||||
951 | /* Fixup recursive PTE PML4E slot. We are only changing the PA */ | |||
952 | pml4va[PDIR_SLOT_PTE255] = pml4pa | (pml4va[PDIR_SLOT_PTE255] & ~PG_FRAME0x000ffffffffff000UL); | |||
953 | ||||
954 | for (i = 0; i < NPDPG((1 << 12) / sizeof (pd_entry_t)); i++) { | |||
955 | /* PTE slot already handled earlier */ | |||
956 | if (i == PDIR_SLOT_PTE255) | |||
957 | continue; | |||
958 | ||||
959 | if (pml4va[i] & PG_FRAME0x000ffffffffff000UL) | |||
960 | pmap_randomize_level(&pml4va[i], 3); | |||
961 | } | |||
962 | ||||
963 | /* Wipe out bootstrap PML4 */ | |||
964 | memset(oldpml4va, 0, PAGE_SIZE)__builtin_memset((oldpml4va), (0), ((1 << 12))); | |||
965 | tlbflush(); | |||
966 | } | |||
967 | ||||
968 | void | |||
969 | pmap_randomize_level(pd_entry_t *pde, int level) | |||
970 | { | |||
971 | pd_entry_t *new_pd_va; | |||
972 | paddr_t old_pd_pa, new_pd_pa; | |||
973 | vaddr_t old_pd_va; | |||
974 | struct vm_page *pg; | |||
975 | int i; | |||
976 | ||||
977 | if (level == 0) | |||
978 | return; | |||
979 | ||||
980 | if (level < PTP_LEVELS4 - 1 && (*pde & PG_PS0x0000000000000080UL)) | |||
981 | return; | |||
982 | ||||
983 | new_pd_va = km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero, &kd_nowait); | |||
984 | if (new_pd_va == NULL((void *)0)) | |||
985 | panic("%s: cannot allocate page for L%d page directory", | |||
986 | __func__, level); | |||
987 | ||||
988 | old_pd_pa = *pde & PG_FRAME0x000ffffffffff000UL; | |||
989 | old_pd_va = PMAP_DIRECT_MAP(old_pd_pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (old_pd_pa)); | |||
990 | pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)new_pd_va, &new_pd_pa); | |||
991 | memcpy(new_pd_va, (void *)old_pd_va, PAGE_SIZE)__builtin_memcpy((new_pd_va), ((void *)old_pd_va), ((1 << 12))); | |||
992 | *pde = new_pd_pa | (*pde & ~PG_FRAME0x000ffffffffff000UL); | |||
993 | ||||
994 | tlbflush(); | |||
995 | memset((void *)old_pd_va, 0, PAGE_SIZE)__builtin_memset(((void *)old_pd_va), (0), ((1 << 12))); | |||
996 | ||||
997 | pg = PHYS_TO_VM_PAGE(old_pd_pa); | |||
998 | if (pg != NULL((void *)0)) { | |||
999 | pg->wire_count--; | |||
1000 | pmap_kernel()(&kernel_pmap_store)->pm_stats.resident_count--; | |||
1001 | if (pg->wire_count <= 1) | |||
1002 | uvm_pagefree(pg); | |||
1003 | } | |||
1004 | ||||
1005 | for (i = 0; i < NPDPG((1 << 12) / sizeof (pd_entry_t)); i++) | |||
1006 | if (new_pd_va[i] & PG_FRAME0x000ffffffffff000UL) | |||
1007 | pmap_randomize_level(&new_pd_va[i], level - 1); | |||
1008 | } | |||
1009 | ||||
1010 | /* | |||
1011 | * Pre-allocate PTPs for low memory, so that 1:1 mappings for various | |||
1012 | * trampoline code can be entered. | |||
1013 | */ | |||
1014 | paddr_t | |||
1015 | pmap_prealloc_lowmem_ptps(paddr_t first_avail) | |||
1016 | { | |||
1017 | pd_entry_t *pdes; | |||
1018 | int level; | |||
1019 | paddr_t newp; | |||
1020 | ||||
1021 | pdes = pmap_kernel()(&kernel_pmap_store)->pm_pdir; | |||
1022 | level = PTP_LEVELS4; | |||
1023 | for (;;) { | |||
1024 | newp = first_avail; first_avail += PAGE_SIZE(1 << 12); | |||
1025 | memset((void *)PMAP_DIRECT_MAP(newp), 0, PAGE_SIZE)__builtin_memset(((void *)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (newp))), (0), ((1 << 12 ))); | |||
1026 | pdes[pl_i(0, level)(((((0) & ~0xffff000000000000)) & ptp_masks[(level)-1 ]) >> ptp_shifts[(level)-1])] = (newp & PG_FRAME0x000ffffffffff000UL) | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL; | |||
1027 | level--; | |||
1028 | if (level <= 1) | |||
1029 | break; | |||
1030 | pdes = normal_pdes[level - 2]; | |||
1031 | } | |||
1032 | ||||
1033 | return first_avail; | |||
1034 | } | |||
1035 | ||||
1036 | /* | |||
1037 | * pmap_init: no further initialization required on this platform | |||
1038 | */ | |||
1039 | void | |||
1040 | pmap_init(void) | |||
1041 | { | |||
1042 | pmap_initialized = 1; | |||
1043 | } | |||
1044 | ||||
1045 | /* | |||
1046 | * p v _ e n t r y f u n c t i o n s | |||
1047 | */ | |||
1048 | ||||
1049 | /* | |||
1050 | * main pv_entry manipulation functions: | |||
1051 | * pmap_enter_pv: enter a mapping onto a pv list | |||
1052 | * pmap_remove_pv: remove a mapping from a pv list | |||
1053 | */ | |||
1054 | ||||
1055 | /* | |||
1056 | * pmap_enter_pv: enter a mapping onto a pv list | |||
1057 | * | |||
1058 | * => caller should adjust ptp's wire_count before calling | |||
1059 | * | |||
1060 | * pve: preallocated pve for us to use | |||
1061 | * ptp: PTP in pmap that maps this VA | |||
1062 | */ | |||
1063 | ||||
1064 | void | |||
1065 | pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, | |||
1066 | vaddr_t va, struct vm_page *ptp) | |||
1067 | { | |||
1068 | pve->pv_pmap = pmap; | |||
1069 | pve->pv_va = va; | |||
1070 | pve->pv_ptp = ptp; /* NULL for kernel pmap */ | |||
1071 | mtx_enter(&pg->mdpage.pv_mtx); | |||
1072 | pve->pv_next = pg->mdpage.pv_list; /* add to ... */ | |||
1073 | pg->mdpage.pv_list = pve; /* ... list */ | |||
1074 | mtx_leave(&pg->mdpage.pv_mtx); | |||
1075 | } | |||
1076 | ||||
1077 | /* | |||
1078 | * pmap_remove_pv: try to remove a mapping from a pv_list | |||
1079 | * | |||
1080 | * => caller should adjust ptp's wire_count and free PTP if needed | |||
1081 | * => we return the removed pve | |||
1082 | */ | |||
1083 | ||||
1084 | struct pv_entry * | |||
1085 | pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) | |||
1086 | { | |||
1087 | struct pv_entry *pve, **prevptr; | |||
1088 | ||||
1089 | mtx_enter(&pg->mdpage.pv_mtx); | |||
1090 | prevptr = &pg->mdpage.pv_list; | |||
1091 | while ((pve = *prevptr) != NULL((void *)0)) { | |||
1092 | if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ | |||
1093 | *prevptr = pve->pv_next; /* remove it! */ | |||
1094 | break; | |||
1095 | } | |||
1096 | prevptr = &pve->pv_next; /* previous pointer */ | |||
1097 | } | |||
1098 | mtx_leave(&pg->mdpage.pv_mtx); | |||
1099 | return(pve); /* return removed pve */ | |||
1100 | } | |||
1101 | ||||
1102 | /* | |||
1103 | * p t p f u n c t i o n s | |||
1104 | */ | |||
1105 | ||||
1106 | struct vm_page * | |||
1107 | pmap_find_ptp(struct pmap *pmap, vaddr_t va, paddr_t pa, int level) | |||
1108 | { | |||
1109 | int lidx = level - 1; | |||
1110 | struct vm_page *pg; | |||
1111 | ||||
1112 | if (pa != (paddr_t)-1 && pmap->pm_ptphint[lidx] && | |||
1113 | pa == VM_PAGE_TO_PHYS(pmap->pm_ptphint[lidx])((pmap->pm_ptphint[lidx])->phys_addr)) | |||
1114 | return (pmap->pm_ptphint[lidx]); | |||
1115 | ||||
1116 | pg = uvm_pagelookup(&pmap->pm_obj[lidx], ptp_va2o(va, level)((((((va) & ~0xffff000000000000)) & ptp_masks[((level )+1)-1]) >> ptp_shifts[((level)+1)-1]) * (1 << 12 ))); | |||
1117 | ||||
1118 | return pg; | |||
1119 | } | |||
1120 | ||||
1121 | void | |||
1122 | pmap_freepage(struct pmap *pmap, struct vm_page *ptp, int level, | |||
1123 | struct pg_to_free *pagelist) | |||
1124 | { | |||
1125 | int lidx; | |||
1126 | struct uvm_object *obj; | |||
1127 | ||||
1128 | lidx = level - 1; | |||
1129 | ||||
1130 | obj = &pmap->pm_obj[lidx]; | |||
1131 | pmap->pm_stats.resident_count--; | |||
1132 | if (pmap->pm_ptphint[lidx] == ptp) | |||
1133 | pmap->pm_ptphint[lidx] = RBT_ROOT(uvm_objtree, &obj->memt)uvm_objtree_RBT_ROOT(&obj->memt); | |||
1134 | ptp->wire_count = 0; | |||
1135 | uvm_pagerealloc(ptp, NULL((void *)0), 0); | |||
1136 | TAILQ_INSERT_TAIL(pagelist, ptp, pageq)do { (ptp)->pageq.tqe_next = ((void *)0); (ptp)->pageq. tqe_prev = (pagelist)->tqh_last; *(pagelist)->tqh_last = (ptp); (pagelist)->tqh_last = &(ptp)->pageq.tqe_next ; } while (0); | |||
1137 | } | |||
1138 | ||||
1139 | void | |||
1140 | pmap_free_ptp(struct pmap *pmap, struct vm_page *ptp, vaddr_t va, | |||
1141 | struct pg_to_free *pagelist) | |||
1142 | { | |||
1143 | unsigned long index; | |||
1144 | int level; | |||
1145 | vaddr_t invaladdr; | |||
1146 | ||||
1147 | level = 1; | |||
1148 | do { | |||
1149 | pmap_freepage(pmap, ptp, level, pagelist); | |||
1150 | index = pl_i(va, level + 1)(((((va) & ~0xffff000000000000)) & ptp_masks[(level + 1)-1]) >> ptp_shifts[(level + 1)-1]); | |||
1151 | pmap_pte_set(&normal_pdes[level - 1][index], 0)_atomic_swap_64((&normal_pdes[level - 1][index]), (0)); | |||
1152 | if (level == PTP_LEVELS4 - 1 && pmap->pm_pdir_intel != NULL((void *)0)) { | |||
1153 | /* Zap special meltdown PML4e */ | |||
1154 | pmap_pte_set(&pmap->pm_pdir_intel[index], 0)_atomic_swap_64((&pmap->pm_pdir_intel[index]), (0)); | |||
1155 | DPRINTF("%s: cleared meltdown PML4e @ index %lu " | |||
1156 | "(va range start 0x%llx)\n", __func__, index, | |||
1157 | (uint64_t)(index << L4_SHIFT)); | |||
1158 | } | |||
1159 | invaladdr = level == 1 ? (vaddr_t)PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) : | |||
1160 | (vaddr_t)normal_pdes[level - 2]; | |||
1161 | pmap_tlb_shootpage(pmap, invaladdr + index * PAGE_SIZE(1 << 12), | |||
1162 | pmap_is_curpmap(curpcb({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curpcb->pcb_pmap)); | |||
1163 | if (level < PTP_LEVELS4 - 1) { | |||
1164 | ptp = pmap_find_ptp(pmap, va, (paddr_t)-1, level + 1); | |||
1165 | ptp->wire_count--; | |||
1166 | if (ptp->wire_count > 1) | |||
1167 | break; | |||
1168 | } | |||
1169 | } while (++level < PTP_LEVELS4); | |||
1170 | } | |||
1171 | ||||
1172 | /* | |||
1173 | * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) | |||
1174 | * | |||
1175 | * => pmap should NOT be pmap_kernel() | |||
1176 | */ | |||
1177 | ||||
1178 | struct vm_page * | |||
1179 | pmap_get_ptp(struct pmap *pmap, vaddr_t va) | |||
1180 | { | |||
1181 | struct vm_page *ptp, *pptp; | |||
1182 | int i; | |||
1183 | unsigned long index; | |||
1184 | pd_entry_t *pva, *pva_intel; | |||
1185 | paddr_t ppa, pa; | |||
1186 | struct uvm_object *obj; | |||
1187 | ||||
1188 | ptp = NULL((void *)0); | |||
1189 | pa = (paddr_t)-1; | |||
1190 | ||||
1191 | /* | |||
1192 | * Loop through all page table levels seeing if we need to | |||
1193 | * add a new page to that level. | |||
1194 | */ | |||
1195 | for (i = PTP_LEVELS4; i > 1; i--) { | |||
1196 | /* | |||
1197 | * Save values from previous round. | |||
1198 | */ | |||
1199 | pptp = ptp; | |||
1200 | ppa = pa; | |||
1201 | ||||
1202 | index = pl_i(va, i)(((((va) & ~0xffff000000000000)) & ptp_masks[(i)-1]) >> ptp_shifts[(i)-1]); | |||
1203 | pva = normal_pdes[i - 2]; | |||
1204 | ||||
1205 | if (pmap_valid_entry(pva[index])((pva[index]) & 0x0000000000000001UL)) { | |||
1206 | ppa = pva[index] & PG_FRAME0x000ffffffffff000UL; | |||
1207 | ptp = NULL((void *)0); | |||
1208 | continue; | |||
1209 | } | |||
1210 | ||||
1211 | obj = &pmap->pm_obj[i-2]; | |||
1212 | ptp = uvm_pagealloc(obj, ptp_va2o(va, i - 1)((((((va) & ~0xffff000000000000)) & ptp_masks[((i - 1 )+1)-1]) >> ptp_shifts[((i - 1)+1)-1]) * (1 << 12 )), NULL((void *)0), | |||
1213 | UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002); | |||
1214 | ||||
1215 | if (ptp == NULL((void *)0)) | |||
1216 | return NULL((void *)0); | |||
1217 | ||||
1218 | atomic_clearbits_intx86_atomic_clearbits_u32(&ptp->pg_flags, PG_BUSY0x00000001); | |||
1219 | ptp->wire_count = 1; | |||
1220 | pmap->pm_ptphint[i - 2] = ptp; | |||
1221 | pa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr); | |||
1222 | pva[index] = (pd_entry_t) (pa | PG_u0x0000000000000004UL | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL); | |||
1223 | ||||
1224 | /* | |||
1225 | * Meltdown Special case - if we are adding a new PML4e for | |||
1226 | * usermode addresses, just copy the PML4e to the U-K page | |||
1227 | * table. | |||
1228 | */ | |||
1229 | if (pmap->pm_pdir_intel != NULL((void *)0) && i == PTP_LEVELS4 && | |||
1230 | va < VM_MAXUSER_ADDRESS0x00007f7fffffc000) { | |||
1231 | pva_intel = pmap->pm_pdir_intel; | |||
1232 | pva_intel[index] = pva[index]; | |||
1233 | DPRINTF("%s: copying usermode PML4e (content=0x%llx) " | |||
1234 | "from 0x%llx -> 0x%llx\n", __func__, pva[index], | |||
1235 | (uint64_t)&pva[index], (uint64_t)&pva_intel[index]); | |||
1236 | } | |||
1237 | ||||
1238 | pmap->pm_stats.resident_count++; | |||
1239 | /* | |||
1240 | * If we're not in the top level, increase the | |||
1241 | * wire count of the parent page. | |||
1242 | */ | |||
1243 | if (i < PTP_LEVELS4) { | |||
1244 | if (pptp == NULL((void *)0)) | |||
1245 | pptp = pmap_find_ptp(pmap, va, ppa, i); | |||
1246 | #ifdef DIAGNOSTIC1 | |||
1247 | if (pptp == NULL((void *)0)) | |||
1248 | panic("%s: pde page disappeared", __func__); | |||
1249 | #endif | |||
1250 | pptp->wire_count++; | |||
1251 | } | |||
1252 | } | |||
1253 | ||||
1254 | /* | |||
1255 | * ptp is not NULL if we just allocated a new ptp. If it's | |||
1256 | * still NULL, we must look up the existing one. | |||
1257 | */ | |||
1258 | if (ptp == NULL((void *)0)) { | |||
1259 | ptp = pmap_find_ptp(pmap, va, ppa, 1); | |||
1260 | #ifdef DIAGNOSTIC1 | |||
1261 | if (ptp == NULL((void *)0)) { | |||
1262 | printf("va %lx ppa %lx\n", (unsigned long)va, | |||
1263 | (unsigned long)ppa); | |||
1264 | panic("%s: unmanaged user PTP", __func__); | |||
1265 | } | |||
1266 | #endif | |||
1267 | } | |||
1268 | ||||
1269 | pmap->pm_ptphint[0] = ptp; | |||
1270 | return(ptp); | |||
1271 | } | |||
1272 | ||||
1273 | /* | |||
1274 | * p m a p l i f e c y c l e f u n c t i o n s | |||
1275 | */ | |||
1276 | ||||
1277 | /* | |||
1278 | * pmap_pdp_ctor: constructor for the PDP cache. | |||
1279 | */ | |||
1280 | ||||
1281 | void | |||
1282 | pmap_pdp_ctor(pd_entry_t *pdir) | |||
1283 | { | |||
1284 | paddr_t pdirpa; | |||
1285 | int npde, i; | |||
1286 | struct pmap *kpm = pmap_kernel()(&kernel_pmap_store); | |||
1287 | ||||
1288 | /* fetch the physical address of the page directory. */ | |||
1289 | (void) pmap_extract(kpm, (vaddr_t) pdir, &pdirpa); | |||
1290 | ||||
1291 | /* zero init area */ | |||
1292 | memset(pdir, 0, PDIR_SLOT_PTE * sizeof(pd_entry_t))__builtin_memset((pdir), (0), (255 * sizeof(pd_entry_t))); | |||
1293 | ||||
1294 | /* put in recursive PDE to map the PTEs */ | |||
1295 | pdir[PDIR_SLOT_PTE255] = pdirpa | PG_V0x0000000000000001UL | PG_KW0x0000000000000002UL | pg_nx; | |||
| ||||
1296 | ||||
1297 | npde = nkptp[PTP_LEVELS4 - 1]; | |||
1298 | ||||
1299 | /* put in kernel VM PDEs */ | |||
1300 | memcpy(&pdir[PDIR_SLOT_KERN], &PDP_BASE[PDIR_SLOT_KERN],__builtin_memcpy((&pdir[256]), (&((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12)))[256] ), (npde * sizeof(pd_entry_t))) | |||
1301 | npde * sizeof(pd_entry_t))__builtin_memcpy((&pdir[256]), (&((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12)))[256] ), (npde * sizeof(pd_entry_t))); | |||
1302 | ||||
1303 | /* zero the rest */ | |||
1304 | memset(&pdir[PDIR_SLOT_KERN + npde], 0,__builtin_memset((&pdir[256 + npde]), (0), ((((1 << 12) / (sizeof (pd_entry_t))) - (256 + npde)) * sizeof(pd_entry_t ))) | |||
1305 | (NTOPLEVEL_PDES - (PDIR_SLOT_KERN + npde)) * sizeof(pd_entry_t))__builtin_memset((&pdir[256 + npde]), (0), ((((1 << 12) / (sizeof (pd_entry_t))) - (256 + npde)) * sizeof(pd_entry_t ))); | |||
1306 | ||||
1307 | for (i = 0; i < NUM_L4_SLOT_DIRECT4; i++) | |||
1308 | pdir[PDIR_SLOT_DIRECT(511 - 4) + i] = kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4) + i]; | |||
1309 | ||||
1310 | #if VM_MIN_KERNEL_ADDRESS0xffff800000000000 != KERNBASE0xffffffff80000000 | |||
1311 | pdir[pl4_pi(KERNBASE)(((((0xffffffff80000000) & ~0xffff000000000000)) & 0x0000ff8000000000UL ) >> 39)] = PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12)))[pl4_pi(KERNBASE)(((((0xffffffff80000000) & ~0xffff000000000000)) & 0x0000ff8000000000UL ) >> 39)]; | |||
1312 | #endif | |||
1313 | } | |||
1314 | ||||
1315 | void | |||
1316 | pmap_pdp_ctor_intel(pd_entry_t *pdir) | |||
1317 | { | |||
1318 | struct pmap *kpm = pmap_kernel()(&kernel_pmap_store); | |||
1319 | ||||
1320 | /* Copy PML4es from pmap_kernel's U-K view */ | |||
1321 | memcpy(pdir, kpm->pm_pdir_intel, PAGE_SIZE)__builtin_memcpy((pdir), (kpm->pm_pdir_intel), ((1 << 12))); | |||
1322 | } | |||
1323 | ||||
1324 | /* | |||
1325 | * pmap_create: create a pmap | |||
1326 | * | |||
1327 | * => note: old pmap interface took a "size" args which allowed for | |||
1328 | * the creation of "software only" pmaps (not in bsd). | |||
1329 | */ | |||
1330 | ||||
1331 | struct pmap * | |||
1332 | pmap_create(void) | |||
1333 | { | |||
1334 | struct pmap *pmap; | |||
1335 | int i; | |||
1336 | ||||
1337 | pmap = pool_get(&pmap_pmap_pool, PR_WAITOK0x0001); | |||
1338 | ||||
1339 | mtx_init(&pmap->pm_mtx, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&pmap-> pm_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); | |||
| ||||
1340 | ||||
1341 | /* init uvm_object */ | |||
1342 | for (i = 0; i < PTP_LEVELS4 - 1; i++) { | |||
1343 | uvm_obj_init(&pmap->pm_obj[i], &pmap_pager, 1); | |||
1344 | pmap->pm_ptphint[i] = NULL((void *)0); | |||
1345 | } | |||
1346 | pmap->pm_stats.wired_count = 0; | |||
1347 | pmap->pm_stats.resident_count = 1; /* count the PDP allocd below */ | |||
1348 | pmap->pm_type = PMAP_TYPE_NORMAL1; | |||
1349 | pmap->eptp = 0; | |||
1350 | ||||
1351 | /* allocate PDP */ | |||
1352 | ||||
1353 | /* | |||
1354 | * note that there is no need to splvm to protect us from | |||
1355 | * malloc since malloc allocates out of a submap and we should | |||
1356 | * have already allocated kernel PTPs to cover the range... | |||
1357 | */ | |||
1358 | ||||
1359 | pmap->pm_pdir = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001); | |||
1360 | pmap_pdp_ctor(pmap->pm_pdir); | |||
1361 | ||||
1362 | pmap->pm_pdirpa = pmap->pm_pdir[PDIR_SLOT_PTE255] & PG_FRAME0x000ffffffffff000UL; | |||
1363 | ||||
1364 | /* | |||
1365 | * Intel CPUs need a special page table to be used during usermode | |||
1366 | * execution, one that lacks all kernel mappings. | |||
1367 | */ | |||
1368 | if (cpu_meltdown) { | |||
1369 | pmap->pm_pdir_intel = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001); | |||
1370 | pmap_pdp_ctor_intel(pmap->pm_pdir_intel); | |||
1371 | pmap->pm_stats.resident_count++; | |||
1372 | if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)pmap->pm_pdir_intel, | |||
1373 | &pmap->pm_pdirpa_intel)) | |||
1374 | panic("%s: unknown PA mapping for meltdown PML4", | |||
1375 | __func__); | |||
1376 | } else { | |||
1377 | pmap->pm_pdir_intel = NULL((void *)0); | |||
1378 | pmap->pm_pdirpa_intel = 0; | |||
1379 | } | |||
1380 | ||||
1381 | mtx_enter(&pmaps_lock); | |||
1382 | LIST_INSERT_HEAD(&pmaps, pmap, pm_list)do { if (((pmap)->pm_list.le_next = (&pmaps)->lh_first ) != ((void *)0)) (&pmaps)->lh_first->pm_list.le_prev = &(pmap)->pm_list.le_next; (&pmaps)->lh_first = (pmap); (pmap)->pm_list.le_prev = &(&pmaps)-> lh_first; } while (0); | |||
1383 | mtx_leave(&pmaps_lock); | |||
1384 | return (pmap); | |||
1385 | } | |||
1386 | ||||
1387 | /* | |||
1388 | * pmap_destroy: drop reference count on pmap. free pmap if | |||
1389 | * reference count goes to zero. | |||
1390 | */ | |||
1391 | ||||
1392 | void | |||
1393 | pmap_destroy(struct pmap *pmap) | |||
1394 | { | |||
1395 | struct vm_page *pg; | |||
1396 | int refs; | |||
1397 | int i; | |||
1398 | ||||
1399 | /* | |||
1400 | * drop reference count | |||
1401 | */ | |||
1402 | ||||
1403 | refs = atomic_dec_int_nv(&pmap->pm_obj[0].uo_refs)_atomic_sub_int_nv((&pmap->pm_obj[0].uo_refs), 1); | |||
1404 | if (refs > 0) { | |||
1405 | return; | |||
1406 | } | |||
1407 | ||||
1408 | /* | |||
1409 | * remove it from global list of pmaps | |||
1410 | */ | |||
1411 | mtx_enter(&pmaps_lock); | |||
1412 | LIST_REMOVE(pmap, pm_list)do { if ((pmap)->pm_list.le_next != ((void *)0)) (pmap)-> pm_list.le_next->pm_list.le_prev = (pmap)->pm_list.le_prev ; *(pmap)->pm_list.le_prev = (pmap)->pm_list.le_next; ( (pmap)->pm_list.le_prev) = ((void *)-1); ((pmap)->pm_list .le_next) = ((void *)-1); } while (0); | |||
1413 | mtx_leave(&pmaps_lock); | |||
1414 | ||||
1415 | /* | |||
1416 | * free any remaining PTPs | |||
1417 | */ | |||
1418 | ||||
1419 | for (i = 0; i < PTP_LEVELS4 - 1; i++) { | |||
1420 | while ((pg = RBT_ROOT(uvm_objtree,uvm_objtree_RBT_ROOT(&pmap->pm_obj[i].memt) | |||
1421 | &pmap->pm_obj[i].memt)uvm_objtree_RBT_ROOT(&pmap->pm_obj[i].memt)) != NULL((void *)0)) { | |||
1422 | KASSERT((pg->pg_flags & PG_BUSY) == 0)(((pg->pg_flags & 0x00000001) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c", 1422, "(pg->pg_flags & PG_BUSY) == 0")); | |||
1423 | ||||
1424 | pg->wire_count = 0; | |||
1425 | pmap->pm_stats.resident_count--; | |||
1426 | ||||
1427 | uvm_pagefree(pg); | |||
1428 | } | |||
1429 | } | |||
1430 | ||||
1431 | pool_put(&pmap_pdp_pool, pmap->pm_pdir); | |||
1432 | ||||
1433 | if (pmap->pm_pdir_intel != NULL((void *)0)) { | |||
1434 | pmap->pm_stats.resident_count--; | |||
1435 | pool_put(&pmap_pdp_pool, pmap->pm_pdir_intel); | |||
1436 | } | |||
1437 | ||||
1438 | pool_put(&pmap_pmap_pool, pmap); | |||
1439 | } | |||
1440 | ||||
1441 | /* | |||
1442 | * Add a reference to the specified pmap. | |||
1443 | */ | |||
1444 | ||||
1445 | void | |||
1446 | pmap_reference(struct pmap *pmap) | |||
1447 | { | |||
1448 | atomic_inc_int(&pmap->pm_obj[0].uo_refs)_atomic_inc_int(&pmap->pm_obj[0].uo_refs); | |||
1449 | } | |||
1450 | ||||
1451 | /* | |||
1452 | * pmap_activate: activate a process' pmap (fill in %cr3) | |||
1453 | * | |||
1454 | * => called from cpu_fork() and when switching pmaps during exec | |||
1455 | * => if p is the curproc, then load it into the MMU | |||
1456 | */ | |||
1457 | ||||
1458 | void | |||
1459 | pmap_activate(struct proc *p) | |||
1460 | { | |||
1461 | struct pcb *pcb = &p->p_addr->u_pcb; | |||
1462 | struct pmap *pmap = p->p_vmspace->vm_map.pmap; | |||
1463 | ||||
1464 | pcb->pcb_pmap = pmap; | |||
1465 | pcb->pcb_cr3 = pmap->pm_pdirpa; | |||
1466 | pcb->pcb_cr3 |= (pmap != pmap_kernel()(&kernel_pmap_store)) ? cr3_pcid_proc : | |||
1467 | (PCID_KERN0 | cr3_reuse_pcid); | |||
1468 | ||||
1469 | if (p != curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc) | |||
1470 | return; | |||
1471 | ||||
1472 | if ((p->p_flag & P_SYSTEM0x00000200) == 0) { | |||
1473 | struct cpu_info *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;}); | |||
1474 | ||||
1475 | /* mark the pmap in use by this processor */ | |||
1476 | self->ci_proc_pmap = pmap; | |||
1477 | ||||
1478 | /* in case we return to userspace without context switching */ | |||
1479 | if (cpu_meltdown) { | |||
1480 | self->ci_kern_cr3 = pcb->pcb_cr3 | cr3_reuse_pcid; | |||
1481 | self->ci_user_cr3 = pmap->pm_pdirpa_intel | | |||
1482 | cr3_pcid_proc_intel; | |||
1483 | } | |||
1484 | } | |||
1485 | ||||
1486 | lcr3(pcb->pcb_cr3); | |||
1487 | } | |||
1488 | ||||
1489 | /* | |||
1490 | * pmap_deactivate: deactivate a process' pmap | |||
1491 | */ | |||
1492 | ||||
1493 | void | |||
1494 | pmap_deactivate(struct proc *p) | |||
1495 | { | |||
1496 | if ((p->p_flag & P_SYSTEM0x00000200) == 0) { | |||
1497 | struct cpu_info *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;}); | |||
1498 | ||||
1499 | /* | |||
1500 | * mark the pmap no longer in use by this processor. | |||
1501 | */ | |||
1502 | KASSERT(self->ci_proc_pmap == p->p_vmspace->vm_map.pmap)((self->ci_proc_pmap == p->p_vmspace->vm_map.pmap) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c" , 1502, "self->ci_proc_pmap == p->p_vmspace->vm_map.pmap" )); | |||
1503 | self->ci_proc_pmap = NULL((void *)0); | |||
1504 | } | |||
1505 | } | |||
1506 | ||||
1507 | /* | |||
1508 | * end of lifecycle functions | |||
1509 | */ | |||
1510 | ||||
1511 | /* | |||
1512 | * some misc. functions | |||
1513 | */ | |||
1514 | ||||
1515 | int | |||
1516 | pmap_pdes_valid(vaddr_t va, pd_entry_t *lastpde) | |||
1517 | { | |||
1518 | int i; | |||
1519 | unsigned long index; | |||
1520 | pd_entry_t pde; | |||
1521 | ||||
1522 | for (i = PTP_LEVELS4; i > 1; i--) { | |||
1523 | index = pl_i(va, i)(((((va) & ~0xffff000000000000)) & ptp_masks[(i)-1]) >> ptp_shifts[(i)-1]); | |||
1524 | pde = normal_pdes[i - 2][index]; | |||
1525 | if (!pmap_valid_entry(pde)((pde) & 0x0000000000000001UL)) | |||
1526 | return 0; | |||
1527 | } | |||
1528 | if (lastpde != NULL((void *)0)) | |||
1529 | *lastpde = pde; | |||
1530 | return 1; | |||
1531 | } | |||
1532 | ||||
1533 | /* | |||
1534 | * pmap_extract: extract a PA for the given VA | |||
1535 | */ | |||
1536 | ||||
1537 | int | |||
1538 | pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) | |||
1539 | { | |||
1540 | pt_entry_t *ptes, pte; | |||
1541 | int level, offs; | |||
1542 | ||||
1543 | if (pmap == pmap_kernel()(&kernel_pmap_store) && va >= PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) && | |||
1544 | va < PMAP_DIRECT_END((((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 ))) { | |||
1545 | *pap = va - PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)); | |||
1546 | return 1; | |||
1547 | } | |||
1548 | ||||
1549 | if (pmap != pmap_kernel()(&kernel_pmap_store)) | |||
1550 | mtx_enter(&pmap->pm_mtx); | |||
1551 | ||||
1552 | level = pmap_find_pte_direct(pmap, va, &ptes, &offs); | |||
1553 | pte = ptes[offs]; | |||
1554 | ||||
1555 | if (pmap != pmap_kernel()(&kernel_pmap_store)) | |||
1556 | mtx_leave(&pmap->pm_mtx); | |||
1557 | ||||
1558 | if (__predict_true(level == 0 && pmap_valid_entry(pte))__builtin_expect(((level == 0 && ((pte) & 0x0000000000000001UL )) != 0), 1)) { | |||
1559 | if (pap != NULL((void *)0)) | |||
1560 | *pap = (pte & PG_FRAME0x000ffffffffff000UL) | (va & PAGE_MASK((1 << 12) - 1)); | |||
1561 | return 1; | |||
1562 | } | |||
1563 | if (level == 1 && (pte & (PG_PS0x0000000000000080UL|PG_V0x0000000000000001UL)) == (PG_PS0x0000000000000080UL|PG_V0x0000000000000001UL)) { | |||
1564 | if (pap != NULL((void *)0)) | |||
1565 | *pap = (pte & PG_LGFRAME0x000fffffffe00000UL) | (va & PAGE_MASK_L2((1ULL << 21) - 1)); | |||
1566 | return 1; | |||
1567 | } | |||
1568 | ||||
1569 | return 0; | |||
1570 | } | |||
1571 | ||||
1572 | /* | |||
1573 | * pmap_zero_page: zero a page | |||
1574 | */ | |||
1575 | ||||
1576 | void | |||
1577 | pmap_zero_page(struct vm_page *pg) | |||
1578 | { | |||
1579 | pagezero(pmap_map_direct(pg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((pg)->phys_addr)))); | |||
1580 | } | |||
1581 | ||||
1582 | /* | |||
1583 | * pmap_flush_cache: flush the cache for a virtual address. | |||
1584 | */ | |||
1585 | void | |||
1586 | pmap_flush_cache(vaddr_t addr, vsize_t len) | |||
1587 | { | |||
1588 | vaddr_t i; | |||
1589 | ||||
1590 | if (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cflushsz == 0) { | |||
1591 | wbinvd_on_all_cpus(); | |||
1592 | return; | |||
1593 | } | |||
1594 | ||||
1595 | /* all cpus that have clflush also have mfence. */ | |||
1596 | mfence(); | |||
1597 | for (i = addr; i < addr + len; i += curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cflushsz) | |||
1598 | clflush(i); | |||
1599 | mfence(); | |||
1600 | } | |||
1601 | ||||
1602 | /* | |||
1603 | * pmap_copy_page: copy a page | |||
1604 | */ | |||
1605 | ||||
1606 | void | |||
1607 | pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg) | |||
1608 | { | |||
1609 | vaddr_t srcva = pmap_map_direct(srcpg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((srcpg)->phys_addr))); | |||
1610 | vaddr_t dstva = pmap_map_direct(dstpg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((dstpg)->phys_addr))); | |||
1611 | ||||
1612 | memcpy((void *)dstva, (void *)srcva, PAGE_SIZE)__builtin_memcpy(((void *)dstva), ((void *)srcva), ((1 << 12))); | |||
1613 | } | |||
1614 | ||||
1615 | /* | |||
1616 | * p m a p r e m o v e f u n c t i o n s | |||
1617 | * | |||
1618 | * functions that remove mappings | |||
1619 | */ | |||
1620 | ||||
1621 | /* | |||
1622 | * pmap_remove_ptes: remove PTEs from a PTP | |||
1623 | * | |||
1624 | * => PTP must be mapped into KVA | |||
1625 | * => PTP should be null if pmap == pmap_kernel() | |||
1626 | */ | |||
1627 | ||||
1628 | void | |||
1629 | pmap_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva, | |||
1630 | vaddr_t startva, vaddr_t endva, int flags, struct pv_entry **free_pvs) | |||
1631 | { | |||
1632 | struct pv_entry *pve; | |||
1633 | pt_entry_t *pte = (pt_entry_t *) ptpva; | |||
1634 | struct vm_page *pg; | |||
1635 | pt_entry_t opte; | |||
1636 | ||||
1637 | /* | |||
1638 | * note that ptpva points to the PTE that maps startva. this may | |||
1639 | * or may not be the first PTE in the PTP. | |||
1640 | * | |||
1641 | * we loop through the PTP while there are still PTEs to look at | |||
1642 | * and the wire_count is greater than 1 (because we use the wire_count | |||
1643 | * to keep track of the number of real PTEs in the PTP). | |||
1644 | */ | |||
1645 | ||||
1646 | for (/*null*/; startva < endva && (ptp == NULL((void *)0) || ptp->wire_count > 1) | |||
1647 | ; pte++, startva += PAGE_SIZE(1 << 12)) { | |||
1648 | if (!pmap_valid_entry(*pte)((*pte) & 0x0000000000000001UL)) | |||
1649 | continue; /* VA not mapped */ | |||
1650 | if ((flags & PMAP_REMOVE_SKIPWIRED1) && (*pte & PG_W0x0000000000000200UL)) { | |||
1651 | continue; | |||
1652 | } | |||
1653 | ||||
1654 | /* atomically save the old PTE and zap! it */ | |||
1655 | opte = pmap_pte_set(pte, 0)_atomic_swap_64((pte), (0)); | |||
1656 | ||||
1657 | if (opte & PG_W0x0000000000000200UL) | |||
1658 | pmap->pm_stats.wired_count--; | |||
1659 | pmap->pm_stats.resident_count--; | |||
1660 | ||||
1661 | if (ptp != NULL((void *)0)) | |||
1662 | ptp->wire_count--; /* dropping a PTE */ | |||
1663 | ||||
1664 | pg = PHYS_TO_VM_PAGE(opte & PG_FRAME0x000ffffffffff000UL); | |||
1665 | ||||
1666 | /* | |||
1667 | * if we are not on a pv list we are done. | |||
1668 | */ | |||
1669 | ||||
1670 | if ((opte & PG_PVLIST0x0000000000000400UL) == 0) { | |||
1671 | #ifdef DIAGNOSTIC1 | |||
1672 | if (pg != NULL((void *)0)) | |||
1673 | panic("%s: managed page without PG_PVLIST: " | |||
1674 | "va 0x%lx, opte 0x%llx", __func__, | |||
1675 | startva, opte); | |||
1676 | #endif | |||
1677 | continue; | |||
1678 | } | |||
1679 | ||||
1680 | #ifdef DIAGNOSTIC1 | |||
1681 | if (pg == NULL((void *)0)) | |||
1682 | panic("%s: unmanaged page marked PG_PVLIST: " | |||
1683 | "va 0x%lx, opte 0x%llx", __func__, | |||
1684 | startva, opte); | |||
1685 | #endif | |||
1686 | ||||
1687 | /* sync R/M bits */ | |||
1688 | pmap_sync_flags_pte(pg, opte); | |||
1689 | pve = pmap_remove_pv(pg, pmap, startva); | |||
1690 | if (pve != NULL((void *)0)) { | |||
1691 | pve->pv_next = *free_pvs; | |||
1692 | *free_pvs = pve; | |||
1693 | } | |||
1694 | ||||
1695 | /* end of "for" loop: time for next pte */ | |||
1696 | } | |||
1697 | } | |||
1698 | ||||
1699 | /* | |||
1700 | * pmap_remove_pte: remove a single PTE from a PTP | |||
1701 | * | |||
1702 | * => PTP must be mapped into KVA | |||
1703 | * => PTP should be null if pmap == pmap_kernel() | |||
1704 | * => returns true if we removed a mapping | |||
1705 | */ | |||
1706 | ||||
1707 | int | |||
1708 | pmap_remove_pte(struct pmap *pmap, struct vm_page *ptp, pt_entry_t *pte, | |||
1709 | vaddr_t va, int flags, struct pv_entry **free_pvs) | |||
1710 | { | |||
1711 | struct pv_entry *pve; | |||
1712 | struct vm_page *pg; | |||
1713 | pt_entry_t opte; | |||
1714 | ||||
1715 | if (!pmap_valid_entry(*pte)((*pte) & 0x0000000000000001UL)) | |||
1716 | return 0; /* VA not mapped */ | |||
1717 | if ((flags & PMAP_REMOVE_SKIPWIRED1) && (*pte & PG_W0x0000000000000200UL)) { | |||
1718 | return 0; | |||
1719 | } | |||
1720 | ||||
1721 | /* atomically save the old PTE and zap! it */ | |||
1722 | opte = pmap_pte_set(pte, 0)_atomic_swap_64((pte), (0)); | |||
1723 | ||||
1724 | if (opte & PG_W0x0000000000000200UL) | |||
1725 | pmap->pm_stats.wired_count--; | |||
1726 | pmap->pm_stats.resident_count--; | |||
1727 | ||||
1728 | if (ptp != NULL((void *)0)) | |||
1729 | ptp->wire_count--; /* dropping a PTE */ | |||
1730 | ||||
1731 | pg = PHYS_TO_VM_PAGE(opte & PG_FRAME0x000ffffffffff000UL); | |||
1732 | ||||
1733 | /* | |||
1734 | * if we are not on a pv list we are done. | |||
1735 | */ | |||
1736 | if ((opte & PG_PVLIST0x0000000000000400UL) == 0) { | |||
1737 | #ifdef DIAGNOSTIC1 | |||
1738 | if (pg != NULL((void *)0)) | |||
1739 | panic("%s: managed page without PG_PVLIST: " | |||
1740 | "va 0x%lx, opte 0x%llx", __func__, va, opte); | |||
1741 | #endif | |||
1742 | return 1; | |||
1743 | } | |||
1744 | ||||
1745 | #ifdef DIAGNOSTIC1 | |||
1746 | if (pg == NULL((void *)0)) | |||
1747 | panic("%s: unmanaged page marked PG_PVLIST: " | |||
1748 | "va 0x%lx, opte 0x%llx", __func__, va, opte); | |||
1749 | #endif | |||
1750 | ||||
1751 | /* sync R/M bits */ | |||
1752 | pmap_sync_flags_pte(pg, opte); | |||
1753 | pve = pmap_remove_pv(pg, pmap, va); | |||
1754 | if (pve != NULL((void *)0)) { | |||
1755 | pve->pv_next = *free_pvs; | |||
1756 | *free_pvs = pve; | |||
1757 | } | |||
1758 | ||||
1759 | return 1; | |||
1760 | } | |||
1761 | ||||
1762 | /* | |||
1763 | * pmap_remove: top level mapping removal function | |||
1764 | * | |||
1765 | * => caller should not be holding any pmap locks | |||
1766 | */ | |||
1767 | ||||
1768 | void | |||
1769 | pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) | |||
1770 | { | |||
1771 | if (pmap->pm_type == PMAP_TYPE_EPT2) | |||
1772 | pmap_remove_ept(pmap, sva, eva); | |||
1773 | else | |||
1774 | pmap_do_remove(pmap, sva, eva, PMAP_REMOVE_ALL0); | |||
1775 | } | |||
1776 | ||||
1777 | /* | |||
1778 | * pmap_do_remove: mapping removal guts | |||
1779 | * | |||
1780 | * => caller should not be holding any pmap locks | |||
1781 | */ | |||
1782 | ||||
1783 | void | |||
1784 | pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags) | |||
1785 | { | |||
1786 | pd_entry_t pde; | |||
1787 | int result; | |||
1788 | paddr_t ptppa; | |||
1789 | vaddr_t blkendva; | |||
1790 | struct vm_page *ptp; | |||
1791 | struct pv_entry *pve; | |||
1792 | struct pv_entry *free_pvs = NULL((void *)0); | |||
1793 | vaddr_t va; | |||
1794 | int shootall = 0, shootself; | |||
1795 | struct pg_to_free empty_ptps; | |||
1796 | paddr_t scr3; | |||
1797 | ||||
1798 | TAILQ_INIT(&empty_ptps)do { (&empty_ptps)->tqh_first = ((void *)0); (&empty_ptps )->tqh_last = &(&empty_ptps)->tqh_first; } while (0); | |||
1799 | ||||
1800 | scr3 = pmap_map_ptes(pmap); | |||
1801 | shootself = (scr3 == 0); | |||
1802 | ||||
1803 | /* | |||
1804 | * removing one page? take shortcut function. | |||
1805 | */ | |||
1806 | ||||
1807 | if (sva + PAGE_SIZE(1 << 12) == eva) { | |||
1808 | if (pmap_pdes_valid(sva, &pde)) { | |||
1809 | ||||
1810 | /* PA of the PTP */ | |||
1811 | ptppa = pde & PG_FRAME0x000ffffffffff000UL; | |||
1812 | ||||
1813 | /* get PTP if non-kernel mapping */ | |||
1814 | ||||
1815 | if (pmap == pmap_kernel()(&kernel_pmap_store)) { | |||
1816 | /* we never free kernel PTPs */ | |||
1817 | ptp = NULL((void *)0); | |||
1818 | } else { | |||
1819 | ptp = pmap_find_ptp(pmap, sva, ptppa, 1); | |||
1820 | #ifdef DIAGNOSTIC1 | |||
1821 | if (ptp == NULL((void *)0)) | |||
1822 | panic("%s: unmanaged PTP detected " | |||
1823 | "in shortcut path", __func__); | |||
1824 | #endif | |||
1825 | } | |||
1826 | ||||
1827 | /* do it! */ | |||
1828 | result = pmap_remove_pte(pmap, ptp, | |||
1829 | &PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(sva)(((((sva) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)], sva, flags, &free_pvs); | |||
1830 | ||||
1831 | /* | |||
1832 | * if mapping removed and the PTP is no longer | |||
1833 | * being used, free it! | |||
1834 | */ | |||
1835 | ||||
1836 | if (result && ptp && ptp->wire_count <= 1) | |||
1837 | pmap_free_ptp(pmap, ptp, sva, &empty_ptps); | |||
1838 | pmap_tlb_shootpage(pmap, sva, shootself); | |||
1839 | pmap_unmap_ptes(pmap, scr3); | |||
1840 | pmap_tlb_shootwait(); | |||
1841 | } else { | |||
1842 | pmap_unmap_ptes(pmap, scr3); | |||
1843 | } | |||
1844 | ||||
1845 | goto cleanup; | |||
1846 | } | |||
1847 | ||||
1848 | if ((eva - sva > 32 * PAGE_SIZE(1 << 12)) && sva < VM_MIN_KERNEL_ADDRESS0xffff800000000000) | |||
1849 | shootall = 1; | |||
1850 | ||||
1851 | for (va = sva; va < eva; va = blkendva) { | |||
1852 | /* determine range of block */ | |||
1853 | blkendva = x86_round_pdr(va + 1)((((unsigned long)(va + 1)) + ((1ULL << 21) - 1)) & ~((1ULL << 21) - 1)); | |||
1854 | if (blkendva > eva) | |||
1855 | blkendva = eva; | |||
1856 | ||||
1857 | /* | |||
1858 | * XXXCDC: our PTE mappings should never be removed | |||
1859 | * with pmap_remove! if we allow this (and why would | |||
1860 | * we?) then we end up freeing the pmap's page | |||
1861 | * directory page (PDP) before we are finished using | |||
1862 | * it when we hit it in the recursive mapping. this | |||
1863 | * is BAD. | |||
1864 | * | |||
1865 | * long term solution is to move the PTEs out of user | |||
1866 | * address space. and into kernel address space (up | |||
1867 | * with APTE). then we can set VM_MAXUSER_ADDRESS to | |||
1868 | * be VM_MAX_ADDRESS. | |||
1869 | */ | |||
1870 | ||||
1871 | if (pl_i(va, PTP_LEVELS)(((((va) & ~0xffff000000000000)) & ptp_masks[(4)-1]) >> ptp_shifts[(4)-1]) == PDIR_SLOT_PTE255) | |||
1872 | /* XXXCDC: ugly hack to avoid freeing PDP here */ | |||
1873 | continue; | |||
1874 | ||||
1875 | if (!pmap_pdes_valid(va, &pde)) | |||
1876 | continue; | |||
1877 | ||||
1878 | /* PA of the PTP */ | |||
1879 | ptppa = pde & PG_FRAME0x000ffffffffff000UL; | |||
1880 | ||||
1881 | /* get PTP if non-kernel mapping */ | |||
1882 | if (pmap == pmap_kernel()(&kernel_pmap_store)) { | |||
1883 | /* we never free kernel PTPs */ | |||
1884 | ptp = NULL((void *)0); | |||
1885 | } else { | |||
1886 | ptp = pmap_find_ptp(pmap, va, ptppa, 1); | |||
1887 | #ifdef DIAGNOSTIC1 | |||
1888 | if (ptp == NULL((void *)0)) | |||
1889 | panic("%s: unmanaged PTP detected", __func__); | |||
1890 | #endif | |||
1891 | } | |||
1892 | pmap_remove_ptes(pmap, ptp, (vaddr_t)&PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)], | |||
1893 | va, blkendva, flags, &free_pvs); | |||
1894 | ||||
1895 | /* if PTP is no longer being used, free it! */ | |||
1896 | if (ptp && ptp->wire_count <= 1) { | |||
1897 | pmap_free_ptp(pmap, ptp, va, &empty_ptps); | |||
1898 | } | |||
1899 | } | |||
1900 | ||||
1901 | if (shootall) | |||
1902 | pmap_tlb_shoottlb(pmap, shootself); | |||
1903 | else | |||
1904 | pmap_tlb_shootrange(pmap, sva, eva, shootself); | |||
1905 | ||||
1906 | pmap_unmap_ptes(pmap, scr3); | |||
1907 | pmap_tlb_shootwait(); | |||
1908 | ||||
1909 | cleanup: | |||
1910 | while ((pve = free_pvs) != NULL((void *)0)) { | |||
1911 | free_pvs = pve->pv_next; | |||
1912 | pool_put(&pmap_pv_pool, pve); | |||
1913 | } | |||
1914 | ||||
1915 | while ((ptp = TAILQ_FIRST(&empty_ptps)((&empty_ptps)->tqh_first)) != NULL((void *)0)) { | |||
1916 | TAILQ_REMOVE(&empty_ptps, ptp, pageq)do { if (((ptp)->pageq.tqe_next) != ((void *)0)) (ptp)-> pageq.tqe_next->pageq.tqe_prev = (ptp)->pageq.tqe_prev; else (&empty_ptps)->tqh_last = (ptp)->pageq.tqe_prev ; *(ptp)->pageq.tqe_prev = (ptp)->pageq.tqe_next; ((ptp )->pageq.tqe_prev) = ((void *)-1); ((ptp)->pageq.tqe_next ) = ((void *)-1); } while (0); | |||
1917 | uvm_pagefree(ptp); | |||
1918 | } | |||
1919 | } | |||
1920 | ||||
1921 | /* | |||
1922 | * pmap_page_remove: remove a managed vm_page from all pmaps that map it | |||
1923 | * | |||
1924 | * => R/M bits are sync'd back to attrs | |||
1925 | */ | |||
1926 | ||||
1927 | void | |||
1928 | pmap_page_remove(struct vm_page *pg) | |||
1929 | { | |||
1930 | struct pv_entry *pve; | |||
1931 | struct pmap *pm; | |||
1932 | pt_entry_t opte; | |||
1933 | #ifdef DIAGNOSTIC1 | |||
1934 | pd_entry_t pde; | |||
1935 | #endif | |||
1936 | struct pg_to_free empty_ptps; | |||
1937 | struct vm_page *ptp; | |||
1938 | paddr_t scr3; | |||
1939 | int shootself; | |||
1940 | ||||
1941 | TAILQ_INIT(&empty_ptps)do { (&empty_ptps)->tqh_first = ((void *)0); (&empty_ptps )->tqh_last = &(&empty_ptps)->tqh_first; } while (0); | |||
1942 | ||||
1943 | mtx_enter(&pg->mdpage.pv_mtx); | |||
1944 | while ((pve = pg->mdpage.pv_list) != NULL((void *)0)) { | |||
1945 | pmap_reference(pve->pv_pmap); | |||
1946 | pm = pve->pv_pmap; | |||
1947 | mtx_leave(&pg->mdpage.pv_mtx); | |||
1948 | ||||
1949 | /* XXX use direct map? */ | |||
1950 | scr3 = pmap_map_ptes(pm); /* locks pmap */ | |||
1951 | shootself = (scr3 == 0); | |||
1952 | ||||
1953 | /* | |||
1954 | * We dropped the pvlist lock before grabbing the pmap | |||
1955 | * lock to avoid lock ordering problems. This means | |||
1956 | * we have to check the pvlist again since somebody | |||
1957 | * else might have modified it. All we care about is | |||
1958 | * that the pvlist entry matches the pmap we just | |||
1959 | * locked. If it doesn't, unlock the pmap and try | |||
1960 | * again. | |||
1961 | */ | |||
1962 | mtx_enter(&pg->mdpage.pv_mtx); | |||
1963 | if ((pve = pg->mdpage.pv_list) == NULL((void *)0) || | |||
1964 | pve->pv_pmap != pm) { | |||
1965 | mtx_leave(&pg->mdpage.pv_mtx); | |||
1966 | pmap_unmap_ptes(pm, scr3); /* unlocks pmap */ | |||
1967 | pmap_destroy(pm); | |||
1968 | mtx_enter(&pg->mdpage.pv_mtx); | |||
1969 | continue; | |||
1970 | } | |||
1971 | ||||
1972 | pg->mdpage.pv_list = pve->pv_next; | |||
1973 | mtx_leave(&pg->mdpage.pv_mtx); | |||
1974 | ||||
1975 | #ifdef DIAGNOSTIC1 | |||
1976 | if (pve->pv_ptp != NULL((void *)0) && pmap_pdes_valid(pve->pv_va, &pde) && | |||
1977 | (pde & PG_FRAME0x000ffffffffff000UL) != VM_PAGE_TO_PHYS(pve->pv_ptp)((pve->pv_ptp)->phys_addr)) { | |||
1978 | printf("%s: pg=%p: va=%lx, pv_ptp=%p\n", __func__, | |||
1979 | pg, pve->pv_va, pve->pv_ptp); | |||
1980 | printf("%s: PTP's phys addr: " | |||
1981 | "actual=%lx, recorded=%lx\n", __func__, | |||
1982 | (unsigned long)(pde & PG_FRAME0x000ffffffffff000UL), | |||
1983 | VM_PAGE_TO_PHYS(pve->pv_ptp)((pve->pv_ptp)->phys_addr)); | |||
1984 | panic("%s: mapped managed page has " | |||
1985 | "invalid pv_ptp field", __func__); | |||
1986 | } | |||
1987 | #endif | |||
1988 | ||||
1989 | /* atomically save the old PTE and zap it */ | |||
1990 | opte = pmap_pte_set(&PTE_BASE[pl1_i(pve->pv_va)], 0)_atomic_swap_64((&((pt_entry_t *) (255 * (1ULL << 39 )))[(((((pve->pv_va) & ~0xffff000000000000)) & ((( 0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL)) >> 12)]), (0)); | |||
1991 | ||||
1992 | if (opte & PG_W0x0000000000000200UL) | |||
1993 | pve->pv_pmap->pm_stats.wired_count--; | |||
1994 | pve->pv_pmap->pm_stats.resident_count--; | |||
1995 | ||||
1996 | pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va, shootself); | |||
1997 | ||||
1998 | pmap_sync_flags_pte(pg, opte); | |||
1999 | ||||
2000 | /* update the PTP reference count. free if last reference. */ | |||
2001 | if (pve->pv_ptp != NULL((void *)0)) { | |||
2002 | pve->pv_ptp->wire_count--; | |||
2003 | if (pve->pv_ptp->wire_count <= 1) { | |||
2004 | pmap_free_ptp(pve->pv_pmap, pve->pv_ptp, | |||
2005 | pve->pv_va, &empty_ptps); | |||
2006 | } | |||
2007 | } | |||
2008 | pmap_unmap_ptes(pve->pv_pmap, scr3); /* unlocks pmap */ | |||
2009 | pmap_destroy(pve->pv_pmap); | |||
2010 | pool_put(&pmap_pv_pool, pve); | |||
2011 | mtx_enter(&pg->mdpage.pv_mtx); | |||
2012 | } | |||
2013 | mtx_leave(&pg->mdpage.pv_mtx); | |||
2014 | ||||
2015 | pmap_tlb_shootwait(); | |||
2016 | ||||
2017 | while ((ptp = TAILQ_FIRST(&empty_ptps)((&empty_ptps)->tqh_first)) != NULL((void *)0)) { | |||
2018 | TAILQ_REMOVE(&empty_ptps, ptp, pageq)do { if (((ptp)->pageq.tqe_next) != ((void *)0)) (ptp)-> pageq.tqe_next->pageq.tqe_prev = (ptp)->pageq.tqe_prev; else (&empty_ptps)->tqh_last = (ptp)->pageq.tqe_prev ; *(ptp)->pageq.tqe_prev = (ptp)->pageq.tqe_next; ((ptp )->pageq.tqe_prev) = ((void *)-1); ((ptp)->pageq.tqe_next ) = ((void *)-1); } while (0); | |||
2019 | uvm_pagefree(ptp); | |||
2020 | } | |||
2021 | } | |||
2022 | ||||
2023 | /* | |||
2024 | * p m a p a t t r i b u t e f u n c t i o n s | |||
2025 | * functions that test/change managed page's attributes | |||
2026 | * since a page can be mapped multiple times we must check each PTE that | |||
2027 | * maps it by going down the pv lists. | |||
2028 | */ | |||
2029 | ||||
2030 | /* | |||
2031 | * pmap_test_attrs: test a page's attributes | |||
2032 | */ | |||
2033 | ||||
2034 | int | |||
2035 | pmap_test_attrs(struct vm_page *pg, unsigned int testbits) | |||
2036 | { | |||
2037 | struct pv_entry *pve; | |||
2038 | pt_entry_t *ptes; | |||
2039 | int level, offs; | |||
2040 | u_long mybits, testflags; | |||
2041 | ||||
2042 | testflags = pmap_pte2flags(testbits); | |||
2043 | ||||
2044 | if (pg->pg_flags & testflags) | |||
2045 | return 1; | |||
2046 | ||||
2047 | mybits = 0; | |||
2048 | mtx_enter(&pg->mdpage.pv_mtx); | |||
2049 | for (pve = pg->mdpage.pv_list; pve != NULL((void *)0) && mybits == 0; | |||
2050 | pve = pve->pv_next) { | |||
2051 | level = pmap_find_pte_direct(pve->pv_pmap, pve->pv_va, &ptes, | |||
2052 | &offs); | |||
2053 | mybits |= (ptes[offs] & testbits); | |||
2054 | } | |||
2055 | mtx_leave(&pg->mdpage.pv_mtx); | |||
2056 | ||||
2057 | if (mybits == 0) | |||
2058 | return 0; | |||
2059 | ||||
2060 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, pmap_pte2flags(mybits)); | |||
2061 | ||||
2062 | return 1; | |||
2063 | } | |||
2064 | ||||
2065 | /* | |||
2066 | * pmap_clear_attrs: change a page's attributes | |||
2067 | * | |||
2068 | * => we return 1 if we cleared one of the bits we were asked to | |||
2069 | */ | |||
2070 | ||||
2071 | int | |||
2072 | pmap_clear_attrs(struct vm_page *pg, unsigned long clearbits) | |||
2073 | { | |||
2074 | struct pv_entry *pve; | |||
2075 | pt_entry_t *ptes, opte; | |||
2076 | u_long clearflags; | |||
2077 | int result, level, offs; | |||
2078 | ||||
2079 | clearflags = pmap_pte2flags(clearbits); | |||
2080 | ||||
2081 | result = pg->pg_flags & clearflags; | |||
2082 | if (result) | |||
2083 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, clearflags); | |||
2084 | ||||
2085 | mtx_enter(&pg->mdpage.pv_mtx); | |||
2086 | for (pve = pg->mdpage.pv_list; pve != NULL((void *)0); pve = pve->pv_next) { | |||
2087 | level = pmap_find_pte_direct(pve->pv_pmap, pve->pv_va, &ptes, | |||
2088 | &offs); | |||
2089 | opte = ptes[offs]; | |||
2090 | if (opte & clearbits) { | |||
2091 | result = 1; | |||
2092 | pmap_pte_clearbits(&ptes[offs], (opte & clearbits))x86_atomic_clearbits_u64(&ptes[offs], (opte & clearbits )); | |||
2093 | pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va, | |||
2094 | pmap_is_curpmap(pve->pv_pmap)); | |||
2095 | } | |||
2096 | } | |||
2097 | mtx_leave(&pg->mdpage.pv_mtx); | |||
2098 | ||||
2099 | pmap_tlb_shootwait(); | |||
2100 | ||||
2101 | return (result != 0); | |||
2102 | } | |||
2103 | ||||
2104 | /* | |||
2105 | * p m a p p r o t e c t i o n f u n c t i o n s | |||
2106 | */ | |||
2107 | ||||
2108 | /* | |||
2109 | * pmap_page_protect: change the protection of all recorded mappings | |||
2110 | * of a managed page | |||
2111 | * | |||
2112 | * => NOTE: this is an inline function in pmap.h | |||
2113 | */ | |||
2114 | ||||
2115 | /* see pmap.h */ | |||
2116 | ||||
2117 | /* | |||
2118 | * pmap_protect: set the protection in of the pages in a pmap | |||
2119 | * | |||
2120 | * => NOTE: this is an inline function in pmap.h | |||
2121 | */ | |||
2122 | ||||
2123 | /* see pmap.h */ | |||
2124 | ||||
2125 | /* | |||
2126 | * pmap_write_protect: write-protect pages in a pmap | |||
2127 | */ | |||
2128 | ||||
2129 | void | |||
2130 | pmap_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) | |||
2131 | { | |||
2132 | pt_entry_t *spte, *epte; | |||
2133 | pt_entry_t clear = 0, set = 0; | |||
2134 | vaddr_t blockend; | |||
2135 | int shootall = 0, shootself; | |||
2136 | vaddr_t va; | |||
2137 | paddr_t scr3; | |||
2138 | ||||
2139 | scr3 = pmap_map_ptes(pmap); | |||
2140 | shootself = (scr3 == 0); | |||
2141 | ||||
2142 | /* should be ok, but just in case ... */ | |||
2143 | sva &= PG_FRAME0x000ffffffffff000UL; | |||
2144 | eva &= PG_FRAME0x000ffffffffff000UL; | |||
2145 | ||||
2146 | if (!(prot & PROT_READ0x01)) | |||
2147 | set |= pg_xo; | |||
2148 | if (!(prot & PROT_WRITE0x02)) | |||
2149 | clear = PG_RW0x0000000000000002UL; | |||
2150 | if (!(prot & PROT_EXEC0x04)) | |||
2151 | set |= pg_nx; | |||
2152 | ||||
2153 | if ((eva - sva > 32 * PAGE_SIZE(1 << 12)) && sva < VM_MIN_KERNEL_ADDRESS0xffff800000000000) | |||
2154 | shootall = 1; | |||
2155 | ||||
2156 | for (va = sva; va < eva ; va = blockend) { | |||
2157 | blockend = (va & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )) + NBPD_L2(1ULL << 21); | |||
2158 | if (blockend > eva) | |||
2159 | blockend = eva; | |||
2160 | ||||
2161 | /* | |||
2162 | * XXXCDC: our PTE mappings should never be write-protected! | |||
2163 | * | |||
2164 | * long term solution is to move the PTEs out of user | |||
2165 | * address space. and into kernel address space (up | |||
2166 | * with APTE). then we can set VM_MAXUSER_ADDRESS to | |||
2167 | * be VM_MAX_ADDRESS. | |||
2168 | */ | |||
2169 | ||||
2170 | /* XXXCDC: ugly hack to avoid freeing PDP here */ | |||
2171 | if (pl_i(va, PTP_LEVELS)(((((va) & ~0xffff000000000000)) & ptp_masks[(4)-1]) >> ptp_shifts[(4)-1]) == PDIR_SLOT_PTE255) | |||
2172 | continue; | |||
2173 | ||||
2174 | /* empty block? */ | |||
2175 | if (!pmap_pdes_valid(va, NULL((void *)0))) | |||
2176 | continue; | |||
2177 | ||||
2178 | #ifdef DIAGNOSTIC1 | |||
2179 | if (va >= VM_MAXUSER_ADDRESS0x00007f7fffffc000 && va < VM_MAX_ADDRESS0x00007fbfdfeff000) | |||
2180 | panic("%s: PTE space", __func__); | |||
2181 | #endif | |||
2182 | ||||
2183 | spte = &PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)]; | |||
2184 | epte = &PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(blockend)(((((blockend) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)]; | |||
2185 | ||||
2186 | for (/*null */; spte < epte ; spte++) { | |||
2187 | if (!pmap_valid_entry(*spte)((*spte) & 0x0000000000000001UL)) | |||
2188 | continue; | |||
2189 | pmap_pte_clearbits(spte, clear)x86_atomic_clearbits_u64(spte, clear); | |||
2190 | pmap_pte_setbits(spte, set)x86_atomic_setbits_u64(spte, set); | |||
2191 | } | |||
2192 | } | |||
2193 | ||||
2194 | if (shootall) | |||
2195 | pmap_tlb_shoottlb(pmap, shootself); | |||
2196 | else | |||
2197 | pmap_tlb_shootrange(pmap, sva, eva, shootself); | |||
2198 | ||||
2199 | pmap_unmap_ptes(pmap, scr3); | |||
2200 | pmap_tlb_shootwait(); | |||
2201 | } | |||
2202 | ||||
2203 | /* | |||
2204 | * end of protection functions | |||
2205 | */ | |||
2206 | ||||
2207 | /* | |||
2208 | * pmap_unwire: clear the wired bit in the PTE | |||
2209 | * | |||
2210 | * => mapping should already be in map | |||
2211 | */ | |||
2212 | ||||
2213 | void | |||
2214 | pmap_unwire(struct pmap *pmap, vaddr_t va) | |||
2215 | { | |||
2216 | pt_entry_t *ptes; | |||
2217 | int level, offs; | |||
2218 | ||||
2219 | level = pmap_find_pte_direct(pmap, va, &ptes, &offs); | |||
2220 | ||||
2221 | if (level == 0) { | |||
2222 | ||||
2223 | #ifdef DIAGNOSTIC1 | |||
2224 | if (!pmap_valid_entry(ptes[offs])((ptes[offs]) & 0x0000000000000001UL)) | |||
2225 | panic("%s: invalid (unmapped) va 0x%lx", __func__, va); | |||
2226 | #endif | |||
2227 | if (__predict_true((ptes[offs] & PG_W) != 0)__builtin_expect((((ptes[offs] & 0x0000000000000200UL) != 0) != 0), 1)) { | |||
2228 | pmap_pte_clearbits(&ptes[offs], PG_W)x86_atomic_clearbits_u64(&ptes[offs], 0x0000000000000200UL ); | |||
2229 | pmap->pm_stats.wired_count--; | |||
2230 | } | |||
2231 | #ifdef DIAGNOSTIC1 | |||
2232 | else { | |||
2233 | printf("%s: wiring for pmap %p va 0x%lx " | |||
2234 | "didn't change!\n", __func__, pmap, va); | |||
2235 | } | |||
2236 | #endif | |||
2237 | } | |||
2238 | #ifdef DIAGNOSTIC1 | |||
2239 | else { | |||
2240 | panic("%s: invalid PDE", __func__); | |||
2241 | } | |||
2242 | #endif | |||
2243 | } | |||
2244 | ||||
2245 | #if 0 | |||
2246 | /* | |||
2247 | * pmap_collect: free resources held by a pmap | |||
2248 | * | |||
2249 | * => optional function. | |||
2250 | * => called when a process is swapped out to free memory. | |||
2251 | */ | |||
2252 | ||||
2253 | void | |||
2254 | pmap_collect(struct pmap *pmap) | |||
2255 | { | |||
2256 | /* | |||
2257 | * free all of the pt pages by removing the physical mappings | |||
2258 | * for its entire address space. | |||
2259 | */ | |||
2260 | ||||
2261 | pmap_do_remove(pmap, VM_MIN_ADDRESS(1 << 12), VM_MAX_ADDRESS0x00007fbfdfeff000, | |||
2262 | PMAP_REMOVE_SKIPWIRED1); | |||
2263 | } | |||
2264 | #endif | |||
2265 | ||||
2266 | void | |||
2267 | pmap_enter_special(vaddr_t va, paddr_t pa, vm_prot_t prot) | |||
2268 | { | |||
2269 | uint64_t l4idx, l3idx, l2idx, l1idx; | |||
2270 | pd_entry_t *pd, *ptp; | |||
2271 | paddr_t npa; | |||
2272 | struct pmap *pmap = pmap_kernel()(&kernel_pmap_store); | |||
2273 | pt_entry_t *ptes; | |||
2274 | int level, offs; | |||
2275 | ||||
2276 | /* If CPU is secure, no need to do anything */ | |||
2277 | if (!cpu_meltdown) | |||
2278 | return; | |||
2279 | ||||
2280 | /* Must be kernel VA */ | |||
2281 | if (va < VM_MIN_KERNEL_ADDRESS0xffff800000000000) | |||
2282 | panic("%s: invalid special mapping va 0x%lx requested", | |||
2283 | __func__, va); | |||
2284 | ||||
2285 | if (pmap->pm_pdir_intel == NULL((void *)0)) | |||
2286 | pmap->pm_pdir_intel = pool_get(&pmap_pdp_pool, | |||
2287 | PR_WAITOK0x0001 | PR_ZERO0x0008); | |||
2288 | ||||
2289 | l4idx = (va & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */ | |||
2290 | l3idx = (va & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */ | |||
2291 | l2idx = (va & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */ | |||
2292 | l1idx = (va & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */ | |||
2293 | ||||
2294 | DPRINTF("%s: va=0x%llx pa=0x%llx l4idx=%lld l3idx=%lld " | |||
2295 | "l2idx=%lld l1idx=%lld\n", __func__, (uint64_t)va, | |||
2296 | (uint64_t)pa, l4idx, l3idx, l2idx, l1idx); | |||
2297 | ||||
2298 | /* Start at PML4 / top level */ | |||
2299 | pd = pmap->pm_pdir_intel; | |||
2300 | ||||
2301 | if (pd == NULL((void *)0)) | |||
2302 | panic("%s: PML4 not initialized for pmap @ %p", __func__, | |||
2303 | pmap); | |||
2304 | ||||
2305 | /* npa = physaddr of PDPT */ | |||
2306 | npa = pd[l4idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2307 | ||||
2308 | /* Valid PML4e for the 512GB region containing va? */ | |||
2309 | if (!npa) { | |||
2310 | /* No valid PML4E - allocate PDPT page and set PML4E */ | |||
2311 | ||||
2312 | ptp = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001 | PR_ZERO0x0008); | |||
2313 | ||||
2314 | if (!pmap_extract(pmap, (vaddr_t)ptp, &npa)) | |||
2315 | panic("%s: can't locate PDPT page", __func__); | |||
2316 | ||||
2317 | pd[l4idx] = (npa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL); | |||
2318 | ||||
2319 | DPRINTF("%s: allocated new PDPT page at phys 0x%llx, " | |||
2320 | "setting PML4e[%lld] = 0x%llx\n", __func__, | |||
2321 | (uint64_t)npa, l4idx, pd[l4idx]); | |||
2322 | } | |||
2323 | ||||
2324 | pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa)); | |||
2325 | if (pd == NULL((void *)0)) | |||
2326 | panic("%s: can't locate PDPT @ pa=0x%llx", __func__, | |||
2327 | (uint64_t)npa); | |||
2328 | ||||
2329 | /* npa = physaddr of PD page */ | |||
2330 | npa = pd[l3idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2331 | ||||
2332 | /* Valid PDPTe for the 1GB region containing va? */ | |||
2333 | if (!npa) { | |||
2334 | /* No valid PDPTe - allocate PD page and set PDPTe */ | |||
2335 | ||||
2336 | ptp = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001 | PR_ZERO0x0008); | |||
2337 | ||||
2338 | if (!pmap_extract(pmap, (vaddr_t)ptp, &npa)) | |||
2339 | panic("%s: can't locate PD page", __func__); | |||
2340 | ||||
2341 | pd[l3idx] = (npa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL); | |||
2342 | ||||
2343 | DPRINTF("%s: allocated new PD page at phys 0x%llx, " | |||
2344 | "setting PDPTe[%lld] = 0x%llx\n", __func__, | |||
2345 | (uint64_t)npa, l3idx, pd[l3idx]); | |||
2346 | } | |||
2347 | ||||
2348 | pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa)); | |||
2349 | if (pd == NULL((void *)0)) | |||
2350 | panic("%s: can't locate PD page @ pa=0x%llx", __func__, | |||
2351 | (uint64_t)npa); | |||
2352 | ||||
2353 | /* npa = physaddr of PT page */ | |||
2354 | npa = pd[l2idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2355 | ||||
2356 | /* Valid PDE for the 2MB region containing va? */ | |||
2357 | if (!npa) { | |||
2358 | /* No valid PDE - allocate PT page and set PDE */ | |||
2359 | ||||
2360 | ptp = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001 | PR_ZERO0x0008); | |||
2361 | ||||
2362 | if (!pmap_extract(pmap, (vaddr_t)ptp, &npa)) | |||
2363 | panic("%s: can't locate PT page", __func__); | |||
2364 | ||||
2365 | pd[l2idx] = (npa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL); | |||
2366 | ||||
2367 | DPRINTF("%s: allocated new PT page at phys 0x%llx, " | |||
2368 | "setting PDE[%lld] = 0x%llx\n", __func__, | |||
2369 | (uint64_t)npa, l2idx, pd[l2idx]); | |||
2370 | } | |||
2371 | ||||
2372 | pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa)); | |||
2373 | if (pd == NULL((void *)0)) | |||
2374 | panic("%s: can't locate PT page @ pa=0x%llx", __func__, | |||
2375 | (uint64_t)npa); | |||
2376 | ||||
2377 | DPRINTF("%s: setting PTE, PT page @ phys 0x%llx virt 0x%llx prot " | |||
2378 | "0x%llx was 0x%llx\n", __func__, (uint64_t)npa, (uint64_t)pd, | |||
2379 | (uint64_t)prot, (uint64_t)pd[l1idx]); | |||
2380 | ||||
2381 | pd[l1idx] = pa | protection_codes[prot] | PG_V0x0000000000000001UL | PG_W0x0000000000000200UL; | |||
2382 | ||||
2383 | /* | |||
2384 | * Look up the corresponding U+K entry. If we're installing the | |||
2385 | * same PA into the U-K map then set the PG_G bit on both and copy | |||
2386 | * the cache-control bits from the U+K entry to the U-K entry. | |||
2387 | */ | |||
2388 | level = pmap_find_pte_direct(pmap, va, &ptes, &offs); | |||
2389 | if (__predict_true(level == 0 && pmap_valid_entry(ptes[offs]))__builtin_expect(((level == 0 && ((ptes[offs]) & 0x0000000000000001UL )) != 0), 1)) { | |||
2390 | if (((pd[l1idx] ^ ptes[offs]) & PG_FRAME0x000ffffffffff000UL) == 0) { | |||
2391 | pd[l1idx] |= PG_G0x0000000000000100UL | (ptes[offs] & (PG_N0x0000000000000010UL | PG_WT0x0000000000000008UL)); | |||
2392 | ptes[offs] |= PG_G0x0000000000000100UL; | |||
2393 | } else { | |||
2394 | DPRINTF("%s: special diffing mapping at %llx\n", | |||
2395 | __func__, (long long)va); | |||
2396 | } | |||
2397 | } else | |||
2398 | DPRINTF("%s: no U+K mapping for special mapping?\n", __func__); | |||
2399 | ||||
2400 | DPRINTF("%s: setting PTE[%lld] = 0x%llx\n", __func__, l1idx, pd[l1idx]); | |||
2401 | } | |||
2402 | ||||
2403 | void | |||
2404 | pmap_remove_ept(struct pmap *pmap, vaddr_t sgpa, vaddr_t egpa) | |||
2405 | { | |||
2406 | vaddr_t v; | |||
2407 | #if NVMM1 > 0 | |||
2408 | struct vmx_invept_descriptor vid; | |||
2409 | #endif /* NVMM > 0 */ | |||
2410 | ||||
2411 | DPRINTF("%s: sgpa=0x%llx egpa=0x%llx\n", __func__, (uint64_t)sgpa, | |||
2412 | (uint64_t)egpa); | |||
2413 | for (v = sgpa; v < egpa + PAGE_SIZE(1 << 12); v += PAGE_SIZE(1 << 12)) | |||
2414 | pmap_do_remove_ept(pmap, v); | |||
2415 | ||||
2416 | #if NVMM1 > 0 | |||
2417 | if (pmap->eptp != 0) { | |||
2418 | memset(&vid, 0, sizeof(vid))__builtin_memset((&vid), (0), (sizeof(vid))); | |||
2419 | vid.vid_eptp = pmap->eptp; | |||
2420 | DPRINTF("%s: flushing EPT TLB for EPTP 0x%llx\n", __func__, | |||
2421 | vid.vid_eptp); | |||
2422 | invept(IA32_VMX_INVEPT_SINGLE_CTX0x1, &vid); | |||
2423 | } | |||
2424 | #endif /* NVMM > 0 */ | |||
2425 | } | |||
2426 | ||||
2427 | void | |||
2428 | pmap_do_remove_ept(struct pmap *pmap, paddr_t gpa) | |||
2429 | { | |||
2430 | uint64_t l4idx, l3idx, l2idx, l1idx; | |||
2431 | struct vm_page *pg3, *pg2, *pg1; | |||
2432 | paddr_t npa3, npa2, npa1; | |||
2433 | pd_entry_t *pd4, *pd3, *pd2, *pd1; | |||
2434 | pd_entry_t *pptes; | |||
2435 | ||||
2436 | l4idx = (gpa & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */ | |||
2437 | l3idx = (gpa & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */ | |||
2438 | l2idx = (gpa & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */ | |||
2439 | l1idx = (gpa & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */ | |||
2440 | ||||
2441 | /* Start at PML4 / top level */ | |||
2442 | pd4 = (pd_entry_t *)pmap->pm_pdir; | |||
2443 | ||||
2444 | if (pd4 == NULL((void *)0)) | |||
2445 | return; | |||
2446 | ||||
2447 | /* npa3 = physaddr of PDPT */ | |||
2448 | npa3 = pd4[l4idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2449 | if (!npa3) | |||
2450 | return; | |||
2451 | pd3 = (pd_entry_t *)PMAP_DIRECT_MAP(npa3)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa3)); | |||
2452 | pg3 = PHYS_TO_VM_PAGE(npa3); | |||
2453 | ||||
2454 | /* npa2 = physaddr of PD page */ | |||
2455 | npa2 = pd3[l3idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2456 | if (!npa2) | |||
2457 | return; | |||
2458 | pd2 = (pd_entry_t *)PMAP_DIRECT_MAP(npa2)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa2)); | |||
2459 | pg2 = PHYS_TO_VM_PAGE(npa2); | |||
2460 | ||||
2461 | /* npa1 = physaddr of PT page */ | |||
2462 | npa1 = pd2[l2idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2463 | if (!npa1) | |||
2464 | return; | |||
2465 | pd1 = (pd_entry_t *)PMAP_DIRECT_MAP(npa1)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa1)); | |||
2466 | pg1 = PHYS_TO_VM_PAGE(npa1); | |||
2467 | ||||
2468 | if (pd1[l1idx] == 0) | |||
2469 | return; | |||
2470 | ||||
2471 | pd1[l1idx] = 0; | |||
2472 | pg1->wire_count--; | |||
2473 | pmap->pm_stats.resident_count--; | |||
2474 | ||||
2475 | if (pg1->wire_count > 1) | |||
2476 | return; | |||
2477 | ||||
2478 | pg1->wire_count = 0; | |||
2479 | pptes = (pd_entry_t *)PMAP_DIRECT_MAP(npa2)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa2)); | |||
2480 | pptes[l2idx] = 0; | |||
2481 | uvm_pagefree(pg1); | |||
2482 | pmap->pm_stats.resident_count--; | |||
2483 | ||||
2484 | pg2->wire_count--; | |||
2485 | if (pg2->wire_count > 1) | |||
2486 | return; | |||
2487 | ||||
2488 | pg2->wire_count = 0; | |||
2489 | pptes = (pd_entry_t *)PMAP_DIRECT_MAP(npa3)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa3)); | |||
2490 | pptes[l3idx] = 0; | |||
2491 | uvm_pagefree(pg2); | |||
2492 | pmap->pm_stats.resident_count--; | |||
2493 | ||||
2494 | pg3->wire_count--; | |||
2495 | if (pg3->wire_count > 1) | |||
2496 | return; | |||
2497 | ||||
2498 | pg3->wire_count = 0; | |||
2499 | pptes = pd4; | |||
2500 | pptes[l4idx] = 0; | |||
2501 | uvm_pagefree(pg3); | |||
2502 | pmap->pm_stats.resident_count--; | |||
2503 | } | |||
2504 | ||||
2505 | int | |||
2506 | pmap_enter_ept(struct pmap *pmap, paddr_t gpa, paddr_t hpa, vm_prot_t prot) | |||
2507 | { | |||
2508 | uint64_t l4idx, l3idx, l2idx, l1idx; | |||
2509 | pd_entry_t *pd, npte; | |||
2510 | struct vm_page *ptp, *pptp; | |||
2511 | paddr_t npa; | |||
2512 | struct uvm_object *obj; | |||
2513 | ||||
2514 | if (gpa > MAXDSIZ((paddr_t)128*1024*1024*1024)) | |||
2515 | return ENOMEM12; | |||
2516 | ||||
2517 | l4idx = (gpa & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */ | |||
2518 | l3idx = (gpa & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */ | |||
2519 | l2idx = (gpa & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */ | |||
2520 | l1idx = (gpa & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */ | |||
2521 | ||||
2522 | /* Start at PML4 / top level */ | |||
2523 | pd = (pd_entry_t *)pmap->pm_pdir; | |||
2524 | ||||
2525 | if (pd == NULL((void *)0)) | |||
2526 | return ENOMEM12; | |||
2527 | ||||
2528 | /* npa = physaddr of PDPT */ | |||
2529 | npa = pd[l4idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2530 | ||||
2531 | /* Valid PML4e for the 512GB region containing gpa? */ | |||
2532 | if (!npa) { | |||
2533 | /* No valid PML4e - allocate PDPT page and set PML4e */ | |||
2534 | obj = &pmap->pm_obj[2]; /* PML4 UVM object */ | |||
2535 | ptp = uvm_pagealloc(obj, ptp_va2o(gpa, 3)((((((gpa) & ~0xffff000000000000)) & ptp_masks[((3)+1 )-1]) >> ptp_shifts[((3)+1)-1]) * (1 << 12)), NULL((void *)0), | |||
2536 | UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002); | |||
2537 | ||||
2538 | if (ptp == NULL((void *)0)) | |||
2539 | return ENOMEM12; | |||
2540 | ||||
2541 | /* | |||
2542 | * New PDPT page - we are setting the first entry, so set | |||
2543 | * the wired count to 1 | |||
2544 | */ | |||
2545 | ptp->wire_count = 1; | |||
2546 | ||||
2547 | /* Calculate phys address of this new PDPT page */ | |||
2548 | npa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr); | |||
2549 | ||||
2550 | /* | |||
2551 | * Higher levels get full perms; specific permissions are | |||
2552 | * entered at the lowest level. | |||
2553 | */ | |||
2554 | pd[l4idx] = (npa | EPT_R(1ULL << 0) | EPT_W(1ULL << 1) | EPT_X(1ULL << 2)); | |||
2555 | ||||
2556 | pmap->pm_stats.resident_count++; | |||
2557 | ||||
2558 | pptp = ptp; | |||
2559 | } else { | |||
2560 | /* Already allocated PML4e */ | |||
2561 | pptp = PHYS_TO_VM_PAGE(npa); | |||
2562 | } | |||
2563 | ||||
2564 | pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa)); | |||
2565 | if (pd == NULL((void *)0)) | |||
2566 | panic("%s: can't locate PDPT @ pa=0x%llx", __func__, | |||
2567 | (uint64_t)npa); | |||
2568 | ||||
2569 | /* npa = physaddr of PD page */ | |||
2570 | npa = pd[l3idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2571 | ||||
2572 | /* Valid PDPTe for the 1GB region containing gpa? */ | |||
2573 | if (!npa) { | |||
2574 | /* No valid PDPTe - allocate PD page and set PDPTe */ | |||
2575 | obj = &pmap->pm_obj[1]; /* PDPT UVM object */ | |||
2576 | ptp = uvm_pagealloc(obj, ptp_va2o(gpa, 2)((((((gpa) & ~0xffff000000000000)) & ptp_masks[((2)+1 )-1]) >> ptp_shifts[((2)+1)-1]) * (1 << 12)), NULL((void *)0), | |||
2577 | UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002); | |||
2578 | ||||
2579 | if (ptp == NULL((void *)0)) | |||
2580 | return ENOMEM12; | |||
2581 | ||||
2582 | /* | |||
2583 | * New PD page - we are setting the first entry, so set | |||
2584 | * the wired count to 1 | |||
2585 | */ | |||
2586 | ptp->wire_count = 1; | |||
2587 | pptp->wire_count++; | |||
2588 | ||||
2589 | npa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr); | |||
2590 | ||||
2591 | /* | |||
2592 | * Higher levels get full perms; specific permissions are | |||
2593 | * entered at the lowest level. | |||
2594 | */ | |||
2595 | pd[l3idx] = (npa | EPT_R(1ULL << 0) | EPT_W(1ULL << 1) | EPT_X(1ULL << 2)); | |||
2596 | ||||
2597 | pmap->pm_stats.resident_count++; | |||
2598 | ||||
2599 | pptp = ptp; | |||
2600 | } else { | |||
2601 | /* Already allocated PDPTe */ | |||
2602 | pptp = PHYS_TO_VM_PAGE(npa); | |||
2603 | } | |||
2604 | ||||
2605 | pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa)); | |||
2606 | if (pd == NULL((void *)0)) | |||
2607 | panic("%s: can't locate PD page @ pa=0x%llx", __func__, | |||
2608 | (uint64_t)npa); | |||
2609 | ||||
2610 | /* npa = physaddr of PT page */ | |||
2611 | npa = pd[l2idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2612 | ||||
2613 | /* Valid PDE for the 2MB region containing gpa? */ | |||
2614 | if (!npa) { | |||
2615 | /* No valid PDE - allocate PT page and set PDE */ | |||
2616 | obj = &pmap->pm_obj[0]; /* PDE UVM object */ | |||
2617 | ptp = uvm_pagealloc(obj, ptp_va2o(gpa, 1)((((((gpa) & ~0xffff000000000000)) & ptp_masks[((1)+1 )-1]) >> ptp_shifts[((1)+1)-1]) * (1 << 12)), NULL((void *)0), | |||
2618 | UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002); | |||
2619 | ||||
2620 | if (ptp == NULL((void *)0)) | |||
2621 | return ENOMEM12; | |||
2622 | ||||
2623 | pptp->wire_count++; | |||
2624 | ||||
2625 | npa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr); | |||
2626 | ||||
2627 | /* | |||
2628 | * Higher level get full perms; specific permissions are | |||
2629 | * entered at the lowest level. | |||
2630 | */ | |||
2631 | pd[l2idx] = (npa | EPT_R(1ULL << 0) | EPT_W(1ULL << 1) | EPT_X(1ULL << 2)); | |||
2632 | ||||
2633 | pmap->pm_stats.resident_count++; | |||
2634 | ||||
2635 | } else { | |||
2636 | /* Find final ptp */ | |||
2637 | ptp = PHYS_TO_VM_PAGE(npa); | |||
2638 | if (ptp == NULL((void *)0)) | |||
2639 | panic("%s: ptp page vanished?", __func__); | |||
2640 | } | |||
2641 | ||||
2642 | pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (npa)); | |||
2643 | if (pd == NULL((void *)0)) | |||
2644 | panic("%s: can't locate PT page @ pa=0x%llx", __func__, | |||
2645 | (uint64_t)npa); | |||
2646 | ||||
2647 | npte = hpa | EPT_WB(6ULL << 3); | |||
2648 | if (prot & PROT_READ0x01) | |||
2649 | npte |= EPT_R(1ULL << 0); | |||
2650 | if (prot & PROT_WRITE0x02) | |||
2651 | npte |= EPT_W(1ULL << 1); | |||
2652 | if (prot & PROT_EXEC0x04) | |||
2653 | npte |= EPT_X(1ULL << 2); | |||
2654 | ||||
2655 | if (pd[l1idx] == 0) { | |||
2656 | ptp->wire_count++; | |||
2657 | pmap->pm_stats.resident_count++; | |||
2658 | } else { | |||
2659 | /* XXX flush ept */ | |||
2660 | } | |||
2661 | ||||
2662 | pd[l1idx] = npte; | |||
2663 | ||||
2664 | return 0; | |||
2665 | } | |||
2666 | ||||
2667 | /* | |||
2668 | * pmap_enter: enter a mapping into a pmap | |||
2669 | * | |||
2670 | * => must be done "now" ... no lazy-evaluation | |||
2671 | */ | |||
2672 | ||||
2673 | int | |||
2674 | pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) | |||
2675 | { | |||
2676 | pt_entry_t opte, npte; | |||
2677 | struct vm_page *ptp, *pg = NULL((void *)0); | |||
2678 | struct pv_entry *pve, *opve = NULL((void *)0); | |||
2679 | int ptpdelta, wireddelta, resdelta; | |||
2680 | int wired = (flags & PMAP_WIRED0x00000010) != 0; | |||
2681 | int nocache = (pa & PMAP_NOCACHE0x1) != 0; | |||
2682 | int wc = (pa & PMAP_WC0x2) != 0; | |||
2683 | int error, shootself; | |||
2684 | paddr_t scr3; | |||
2685 | ||||
2686 | if (pmap->pm_type == PMAP_TYPE_EPT2) | |||
2687 | return pmap_enter_ept(pmap, va, pa, prot); | |||
2688 | ||||
2689 | KASSERT(!(wc && nocache))((!(wc && nocache)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/arch/amd64/amd64/pmap.c", 2689, "!(wc && nocache)" )); | |||
2690 | pa &= PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)); | |||
2691 | ||||
2692 | #ifdef DIAGNOSTIC1 | |||
2693 | if (va == (vaddr_t) PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12)))) | |||
2694 | panic("%s: trying to map over PDP!", __func__); | |||
2695 | ||||
2696 | /* sanity check: kernel PTPs should already have been pre-allocated */ | |||
2697 | if (va >= VM_MIN_KERNEL_ADDRESS0xffff800000000000 && | |||
2698 | !pmap_valid_entry(pmap->pm_pdir[pl_i(va, PTP_LEVELS)])((pmap->pm_pdir[(((((va) & ~0xffff000000000000)) & ptp_masks[(4)-1]) >> ptp_shifts[(4)-1])]) & 0x0000000000000001UL )) | |||
2699 | panic("%s: missing kernel PTP for va %lx!", __func__, va); | |||
2700 | ||||
2701 | #endif | |||
2702 | ||||
2703 | pve = pool_get(&pmap_pv_pool, PR_NOWAIT0x0002); | |||
2704 | if (pve == NULL((void *)0)) { | |||
2705 | if (flags & PMAP_CANFAIL0x00000020) { | |||
2706 | error = ENOMEM12; | |||
2707 | goto out; | |||
2708 | } | |||
2709 | panic("%s: no pv entries available", __func__); | |||
2710 | } | |||
2711 | ||||
2712 | /* | |||
2713 | * map in ptes and get a pointer to our PTP (unless we are the kernel) | |||
2714 | */ | |||
2715 | ||||
2716 | scr3 = pmap_map_ptes(pmap); | |||
2717 | shootself = (scr3 == 0); | |||
2718 | if (pmap == pmap_kernel()(&kernel_pmap_store)) { | |||
2719 | ptp = NULL((void *)0); | |||
2720 | } else { | |||
2721 | ptp = pmap_get_ptp(pmap, va); | |||
2722 | if (ptp == NULL((void *)0)) { | |||
2723 | if (flags & PMAP_CANFAIL0x00000020) { | |||
2724 | pmap_unmap_ptes(pmap, scr3); | |||
2725 | error = ENOMEM12; | |||
2726 | goto out; | |||
2727 | } | |||
2728 | panic("%s: get ptp failed", __func__); | |||
2729 | } | |||
2730 | } | |||
2731 | opte = PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)]; /* old PTE */ | |||
2732 | ||||
2733 | /* | |||
2734 | * is there currently a valid mapping at our VA? | |||
2735 | */ | |||
2736 | ||||
2737 | if (pmap_valid_entry(opte)((opte) & 0x0000000000000001UL)) { | |||
2738 | /* | |||
2739 | * first, calculate pm_stats updates. resident count will not | |||
2740 | * change since we are replacing/changing a valid mapping. | |||
2741 | * wired count might change... | |||
2742 | */ | |||
2743 | ||||
2744 | resdelta = 0; | |||
2745 | if (wired && (opte & PG_W0x0000000000000200UL) == 0) | |||
2746 | wireddelta = 1; | |||
2747 | else if (!wired && (opte & PG_W0x0000000000000200UL) != 0) | |||
2748 | wireddelta = -1; | |||
2749 | else | |||
2750 | wireddelta = 0; | |||
2751 | ptpdelta = 0; | |||
2752 | ||||
2753 | /* | |||
2754 | * is the currently mapped PA the same as the one we | |||
2755 | * want to map? | |||
2756 | */ | |||
2757 | ||||
2758 | if ((opte & PG_FRAME0x000ffffffffff000UL) == pa) { | |||
2759 | ||||
2760 | /* if this is on the PVLIST, sync R/M bit */ | |||
2761 | if (opte & PG_PVLIST0x0000000000000400UL) { | |||
2762 | pg = PHYS_TO_VM_PAGE(pa); | |||
2763 | #ifdef DIAGNOSTIC1 | |||
2764 | if (pg == NULL((void *)0)) | |||
2765 | panic("%s: same pa, PG_PVLIST " | |||
2766 | "mapping with unmanaged page: " | |||
2767 | "va 0x%lx, opte 0x%llx, pa 0x%lx", | |||
2768 | __func__, va, opte, pa); | |||
2769 | #endif | |||
2770 | pmap_sync_flags_pte(pg, opte); | |||
2771 | } else { | |||
2772 | #ifdef DIAGNOSTIC1 | |||
2773 | if (PHYS_TO_VM_PAGE(pa) != NULL((void *)0)) | |||
2774 | panic("%s: same pa, no PG_PVLIST " | |||
2775 | "mapping with managed page: " | |||
2776 | "va 0x%lx, opte 0x%llx, pa 0x%lx", | |||
2777 | __func__, va, opte, pa); | |||
2778 | #endif | |||
2779 | } | |||
2780 | goto enter_now; | |||
2781 | } | |||
2782 | ||||
2783 | /* | |||
2784 | * changing PAs: we must remove the old one first | |||
2785 | */ | |||
2786 | ||||
2787 | /* | |||
2788 | * if current mapping is on a pvlist, | |||
2789 | * remove it (sync R/M bits) | |||
2790 | */ | |||
2791 | ||||
2792 | if (opte & PG_PVLIST0x0000000000000400UL) { | |||
2793 | pg = PHYS_TO_VM_PAGE(opte & PG_FRAME0x000ffffffffff000UL); | |||
2794 | #ifdef DIAGNOSTIC1 | |||
2795 | if (pg == NULL((void *)0)) | |||
2796 | panic("%s: PG_PVLIST mapping with unmanaged " | |||
2797 | "page: va 0x%lx, opte 0x%llx, pa 0x%lx", | |||
2798 | __func__, va, opte, pa); | |||
2799 | #endif | |||
2800 | pmap_sync_flags_pte(pg, opte); | |||
2801 | opve = pmap_remove_pv(pg, pmap, va); | |||
2802 | pg = NULL((void *)0); /* This is not the page we are looking for */ | |||
2803 | } | |||
2804 | } else { /* opte not valid */ | |||
2805 | resdelta = 1; | |||
2806 | if (wired) | |||
2807 | wireddelta = 1; | |||
2808 | else | |||
2809 | wireddelta = 0; | |||
2810 | if (ptp != NULL((void *)0)) | |||
2811 | ptpdelta = 1; | |||
2812 | else | |||
2813 | ptpdelta = 0; | |||
2814 | } | |||
2815 | ||||
2816 | /* | |||
2817 | * pve is either NULL or points to a now-free pv_entry structure | |||
2818 | * (the latter case is if we called pmap_remove_pv above). | |||
2819 | * | |||
2820 | * if this entry is to be on a pvlist, enter it now. | |||
2821 | */ | |||
2822 | ||||
2823 | if (pmap_initialized) | |||
2824 | pg = PHYS_TO_VM_PAGE(pa); | |||
2825 | ||||
2826 | if (pg != NULL((void *)0)) { | |||
2827 | pmap_enter_pv(pg, pve, pmap, va, ptp); | |||
2828 | pve = NULL((void *)0); | |||
2829 | } | |||
2830 | ||||
2831 | enter_now: | |||
2832 | /* | |||
2833 | * at this point pg is !NULL if we want the PG_PVLIST bit set | |||
2834 | */ | |||
2835 | ||||
2836 | pmap->pm_stats.resident_count += resdelta; | |||
2837 | pmap->pm_stats.wired_count += wireddelta; | |||
2838 | if (ptp != NULL((void *)0)) | |||
2839 | ptp->wire_count += ptpdelta; | |||
2840 | ||||
2841 | KASSERT(pg == PHYS_TO_VM_PAGE(pa))((pg == PHYS_TO_VM_PAGE(pa)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/arch/amd64/amd64/pmap.c", 2841, "pg == PHYS_TO_VM_PAGE(pa)" )); | |||
2842 | ||||
2843 | npte = pa | protection_codes[prot] | PG_V0x0000000000000001UL; | |||
2844 | if (pg != NULL((void *)0)) { | |||
2845 | npte |= PG_PVLIST0x0000000000000400UL; | |||
2846 | /* | |||
2847 | * make sure that if the page is write combined all | |||
2848 | * instances of pmap_enter make it so. | |||
2849 | */ | |||
2850 | if (pg->pg_flags & PG_PMAP_WC0x04000000) { | |||
2851 | KASSERT(nocache == 0)((nocache == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c" , 2851, "nocache == 0")); | |||
2852 | wc = 1; | |||
2853 | } | |||
2854 | } | |||
2855 | if (wc) | |||
2856 | npte |= pmap_pg_wc; | |||
2857 | if (wired) | |||
2858 | npte |= PG_W0x0000000000000200UL; | |||
2859 | if (nocache) | |||
2860 | npte |= PG_N0x0000000000000010UL; | |||
2861 | if (va < VM_MAXUSER_ADDRESS0x00007f7fffffc000) | |||
2862 | npte |= ((flags & PMAP_EFI0x00000040) ? 0 : PG_u0x0000000000000004UL); | |||
2863 | else if (va < VM_MAX_ADDRESS0x00007fbfdfeff000) | |||
2864 | npte |= (PG_u0x0000000000000004UL | PG_RW0x0000000000000002UL); /* XXXCDC: no longer needed? */ | |||
2865 | if (pmap == pmap_kernel()(&kernel_pmap_store)) | |||
2866 | npte |= pg_g_kern; | |||
2867 | ||||
2868 | /* | |||
2869 | * If the old entry wasn't valid, we can just update it and | |||
2870 | * go. If it was valid, and this isn't a read->write | |||
2871 | * transition, then we can safely just update it and flush | |||
2872 | * any old TLB entries. | |||
2873 | * | |||
2874 | * If it _was_ valid and this _is_ a read->write transition, | |||
2875 | * then this could be a CoW resolution and we need to make | |||
2876 | * sure no CPU can see the new writable mapping while another | |||
2877 | * still has the old mapping in its TLB, so insert a correct | |||
2878 | * but unwritable mapping, flush any old TLB entries, then | |||
2879 | * make it writable. | |||
2880 | */ | |||
2881 | if (! pmap_valid_entry(opte)((opte) & 0x0000000000000001UL)) { | |||
2882 | PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)] = npte; | |||
2883 | } else if ((opte | (npte ^ PG_RW0x0000000000000002UL)) & PG_RW0x0000000000000002UL) { | |||
2884 | /* previously writable or not making writable */ | |||
2885 | PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)] = npte; | |||
2886 | if (nocache && (opte & PG_N0x0000000000000010UL) == 0) | |||
2887 | wbinvd_on_all_cpus(); | |||
2888 | pmap_tlb_shootpage(pmap, va, shootself); | |||
2889 | } else { | |||
2890 | PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)] = npte ^ PG_RW0x0000000000000002UL; | |||
2891 | if (nocache && (opte & PG_N0x0000000000000010UL) == 0) /* XXX impossible? */ | |||
2892 | wbinvd_on_all_cpus(); | |||
2893 | pmap_tlb_shootpage(pmap, va, shootself); | |||
2894 | pmap_tlb_shootwait(); | |||
2895 | PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)] = npte; | |||
2896 | } | |||
2897 | ||||
2898 | pmap_unmap_ptes(pmap, scr3); | |||
2899 | pmap_tlb_shootwait(); | |||
2900 | ||||
2901 | error = 0; | |||
2902 | ||||
2903 | out: | |||
2904 | if (pve != NULL((void *)0)) | |||
2905 | pool_put(&pmap_pv_pool, pve); | |||
2906 | if (opve != NULL((void *)0)) | |||
2907 | pool_put(&pmap_pv_pool, opve); | |||
2908 | ||||
2909 | return error; | |||
2910 | } | |||
2911 | ||||
2912 | int | |||
2913 | pmap_get_physpage(vaddr_t va, int level, paddr_t *paddrp) | |||
2914 | { | |||
2915 | struct vm_page *ptp; | |||
2916 | struct pmap *kpm = pmap_kernel()(&kernel_pmap_store); | |||
2917 | ||||
2918 | if (uvm.page_init_done == 0) { | |||
2919 | vaddr_t va; | |||
2920 | ||||
2921 | /* | |||
2922 | * we're growing the kernel pmap early (from | |||
2923 | * uvm_pageboot_alloc()). this case must be | |||
2924 | * handled a little differently. | |||
2925 | */ | |||
2926 | ||||
2927 | va = pmap_steal_memory(PAGE_SIZE(1 << 12), NULL((void *)0), NULL((void *)0)); | |||
2928 | *paddrp = PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 ))); | |||
2929 | } else { | |||
2930 | ptp = uvm_pagealloc(&kpm->pm_obj[level - 1], | |||
2931 | ptp_va2o(va, level)((((((va) & ~0xffff000000000000)) & ptp_masks[((level )+1)-1]) >> ptp_shifts[((level)+1)-1]) * (1 << 12 )), NULL((void *)0), | |||
2932 | UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002); | |||
2933 | if (ptp == NULL((void *)0)) | |||
2934 | panic("%s: out of memory", __func__); | |||
2935 | atomic_clearbits_intx86_atomic_clearbits_u32(&ptp->pg_flags, PG_BUSY0x00000001); | |||
2936 | ptp->wire_count = 1; | |||
2937 | *paddrp = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr); | |||
2938 | } | |||
2939 | kpm->pm_stats.resident_count++; | |||
2940 | return 1; | |||
2941 | } | |||
2942 | ||||
2943 | /* | |||
2944 | * Allocate the amount of specified ptps for a ptp level, and populate | |||
2945 | * all levels below accordingly, mapping virtual addresses starting at | |||
2946 | * kva. | |||
2947 | * | |||
2948 | * Used by pmap_growkernel. | |||
2949 | */ | |||
2950 | void | |||
2951 | pmap_alloc_level(vaddr_t kva, int lvl, long *needed_ptps) | |||
2952 | { | |||
2953 | unsigned long i; | |||
2954 | vaddr_t va; | |||
2955 | paddr_t pa; | |||
2956 | unsigned long index, endindex; | |||
2957 | int level; | |||
2958 | pd_entry_t *pdep; | |||
2959 | ||||
2960 | for (level = lvl; level > 1; level--) { | |||
2961 | if (level == PTP_LEVELS4) | |||
2962 | pdep = pmap_kernel()(&kernel_pmap_store)->pm_pdir; | |||
2963 | else | |||
2964 | pdep = normal_pdes[level - 2]; | |||
2965 | va = kva; | |||
2966 | index = pl_i(kva, level)(((((kva) & ~0xffff000000000000)) & ptp_masks[(level) -1]) >> ptp_shifts[(level)-1]); | |||
2967 | endindex = index + needed_ptps[level - 1]; | |||
2968 | /* | |||
2969 | * XXX special case for first time call. | |||
2970 | */ | |||
2971 | if (nkptp[level - 1] != 0) | |||
2972 | index++; | |||
2973 | else | |||
2974 | endindex--; | |||
2975 | ||||
2976 | for (i = index; i <= endindex; i++) { | |||
2977 | pmap_get_physpage(va, level - 1, &pa); | |||
2978 | pdep[i] = pa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | pg_nx; | |||
2979 | nkptp[level - 1]++; | |||
2980 | va += nbpd[level - 1]; | |||
2981 | } | |||
2982 | } | |||
2983 | } | |||
2984 | ||||
2985 | /* | |||
2986 | * pmap_growkernel: increase usage of KVM space | |||
2987 | * | |||
2988 | * => we allocate new PTPs for the kernel and install them in all | |||
2989 | * the pmaps on the system. | |||
2990 | */ | |||
2991 | ||||
2992 | static vaddr_t pmap_maxkvaddr = VM_MIN_KERNEL_ADDRESS0xffff800000000000; | |||
2993 | ||||
2994 | vaddr_t | |||
2995 | pmap_growkernel(vaddr_t maxkvaddr) | |||
2996 | { | |||
2997 | struct pmap *kpm = pmap_kernel()(&kernel_pmap_store), *pm; | |||
2998 | int s, i; | |||
2999 | unsigned newpdes; | |||
3000 | long needed_kptp[PTP_LEVELS4], target_nptp, old; | |||
3001 | ||||
3002 | if (maxkvaddr <= pmap_maxkvaddr) | |||
3003 | return pmap_maxkvaddr; | |||
3004 | ||||
3005 | maxkvaddr = x86_round_pdr(maxkvaddr)((((unsigned long)(maxkvaddr)) + ((1ULL << 21) - 1)) & ~((1ULL << 21) - 1)); | |||
3006 | old = nkptp[PTP_LEVELS4 - 1]; | |||
3007 | /* | |||
3008 | * This loop could be optimized more, but pmap_growkernel() | |||
3009 | * is called infrequently. | |||
3010 | */ | |||
3011 | for (i = PTP_LEVELS4 - 1; i >= 1; i--) { | |||
3012 | target_nptp = pl_i(maxkvaddr, i + 1)(((((maxkvaddr) & ~0xffff000000000000)) & ptp_masks[( i + 1)-1]) >> ptp_shifts[(i + 1)-1]) - | |||
3013 | pl_i(VM_MIN_KERNEL_ADDRESS, i + 1)(((((0xffff800000000000) & ~0xffff000000000000)) & ptp_masks [(i + 1)-1]) >> ptp_shifts[(i + 1)-1]); | |||
3014 | /* | |||
3015 | * XXX only need to check toplevel. | |||
3016 | */ | |||
3017 | if (target_nptp > nkptpmax[i]) | |||
3018 | panic("%s: out of KVA space", __func__); | |||
3019 | needed_kptp[i] = target_nptp - nkptp[i] + 1; | |||
3020 | } | |||
3021 | ||||
3022 | ||||
3023 | s = splhigh()splraise(0xd); /* to be safe */ | |||
3024 | pmap_alloc_level(pmap_maxkvaddr, PTP_LEVELS4, needed_kptp); | |||
3025 | ||||
3026 | /* | |||
3027 | * If the number of top level entries changed, update all | |||
3028 | * pmaps. | |||
3029 | */ | |||
3030 | if (needed_kptp[PTP_LEVELS4 - 1] != 0) { | |||
3031 | newpdes = nkptp[PTP_LEVELS4 - 1] - old; | |||
3032 | mtx_enter(&pmaps_lock); | |||
3033 | LIST_FOREACH(pm, &pmaps, pm_list)for((pm) = ((&pmaps)->lh_first); (pm)!= ((void *)0); ( pm) = ((pm)->pm_list.le_next)) { | |||
3034 | memcpy(&pm->pm_pdir[PDIR_SLOT_KERN + old],__builtin_memcpy((&pm->pm_pdir[256 + old]), (&kpm-> pm_pdir[256 + old]), (newpdes * sizeof (pd_entry_t))) | |||
3035 | &kpm->pm_pdir[PDIR_SLOT_KERN + old],__builtin_memcpy((&pm->pm_pdir[256 + old]), (&kpm-> pm_pdir[256 + old]), (newpdes * sizeof (pd_entry_t))) | |||
3036 | newpdes * sizeof (pd_entry_t))__builtin_memcpy((&pm->pm_pdir[256 + old]), (&kpm-> pm_pdir[256 + old]), (newpdes * sizeof (pd_entry_t))); | |||
3037 | } | |||
3038 | mtx_leave(&pmaps_lock); | |||
3039 | } | |||
3040 | pmap_maxkvaddr = maxkvaddr; | |||
3041 | splx(s)spllower(s); | |||
3042 | ||||
3043 | return maxkvaddr; | |||
3044 | } | |||
3045 | ||||
3046 | vaddr_t | |||
3047 | pmap_steal_memory(vsize_t size, vaddr_t *start, vaddr_t *end) | |||
3048 | { | |||
3049 | int segno; | |||
3050 | u_int npg; | |||
3051 | vaddr_t va; | |||
3052 | paddr_t pa; | |||
3053 | struct vm_physseg *seg; | |||
3054 | ||||
3055 | size = round_page(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1 )); | |||
3056 | npg = atop(size)((size) >> 12); | |||
3057 | ||||
3058 | for (segno = 0, seg = vm_physmem; segno < vm_nphysseg; segno++, seg++) { | |||
3059 | if (seg->avail_end - seg->avail_start < npg) | |||
3060 | continue; | |||
3061 | /* | |||
3062 | * We can only steal at an ``unused'' segment boundary, | |||
3063 | * i.e. either at the start or at the end. | |||
3064 | */ | |||
3065 | if (seg->avail_start == seg->start || | |||
3066 | seg->avail_end == seg->end) | |||
3067 | break; | |||
3068 | } | |||
3069 | if (segno == vm_nphysseg) { | |||
3070 | panic("%s: out of memory", __func__); | |||
3071 | } else { | |||
3072 | if (seg->avail_start == seg->start) { | |||
3073 | pa = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12); | |||
3074 | seg->avail_start += npg; | |||
3075 | seg->start += npg; | |||
3076 | } else { | |||
3077 | pa = ptoa(seg->avail_end)((paddr_t)(seg->avail_end) << 12) - size; | |||
3078 | seg->avail_end -= npg; | |||
3079 | seg->end -= npg; | |||
3080 | } | |||
3081 | /* | |||
3082 | * If all the segment has been consumed now, remove it. | |||
3083 | * Note that the crash dump code still knows about it | |||
3084 | * and will dump it correctly. | |||
3085 | */ | |||
3086 | if (seg->start == seg->end) { | |||
3087 | if (vm_nphysseg-- == 1) | |||
3088 | panic("%s: out of memory", __func__); | |||
3089 | while (segno < vm_nphysseg) { | |||
3090 | seg[0] = seg[1]; /* struct copy */ | |||
3091 | seg++; | |||
3092 | segno++; | |||
3093 | } | |||
3094 | } | |||
3095 | ||||
3096 | va = PMAP_DIRECT_MAP(pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (pa)); | |||
3097 | memset((void *)va, 0, size)__builtin_memset(((void *)va), (0), (size)); | |||
3098 | } | |||
3099 | ||||
3100 | if (start != NULL((void *)0)) | |||
3101 | *start = virtual_avail; | |||
3102 | if (end != NULL((void *)0)) | |||
3103 | *end = VM_MAX_KERNEL_ADDRESS0xffff800100000000; | |||
3104 | ||||
3105 | return (va); | |||
3106 | } | |||
3107 | ||||
3108 | void | |||
3109 | pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) | |||
3110 | { | |||
3111 | *vstartp = virtual_avail; | |||
3112 | *vendp = VM_MAX_KERNEL_ADDRESS0xffff800100000000; | |||
3113 | } | |||
3114 | ||||
3115 | /* | |||
3116 | * pmap_convert | |||
3117 | * | |||
3118 | * Converts 'pmap' to the new 'mode'. | |||
3119 | * | |||
3120 | * Parameters: | |||
3121 | * pmap: the pmap to convert | |||
3122 | * mode: the new mode (see pmap.h, PMAP_TYPE_xxx) | |||
3123 | */ | |||
3124 | void | |||
3125 | pmap_convert(struct pmap *pmap, int mode) | |||
3126 | { | |||
3127 | pt_entry_t *pte; | |||
3128 | ||||
3129 | pmap->pm_type = mode; | |||
3130 | ||||
3131 | if (mode == PMAP_TYPE_EPT2) { | |||
3132 | /* Clear PML4 */ | |||
3133 | pte = (pt_entry_t *)pmap->pm_pdir; | |||
3134 | memset(pte, 0, PAGE_SIZE)__builtin_memset((pte), (0), ((1 << 12))); | |||
3135 | ||||
3136 | /* Give back the meltdown pdir */ | |||
3137 | if (pmap->pm_pdir_intel != NULL((void *)0)) { | |||
3138 | pool_put(&pmap_pdp_pool, pmap->pm_pdir_intel); | |||
3139 | pmap->pm_pdir_intel = NULL((void *)0); | |||
3140 | } | |||
3141 | } | |||
3142 | } | |||
3143 | ||||
3144 | #ifdef MULTIPROCESSOR1 | |||
3145 | /* | |||
3146 | * Locking for tlb shootdown. | |||
3147 | * | |||
3148 | * We lock by setting tlb_shoot_wait to the number of cpus that will | |||
3149 | * receive our tlb shootdown. After sending the IPIs, we don't need to | |||
3150 | * worry about locking order or interrupts spinning for the lock because | |||
3151 | * the call that grabs the "lock" isn't the one that releases it. And | |||
3152 | * there is nothing that can block the IPI that releases the lock. | |||
3153 | * | |||
3154 | * The functions are organized so that we first count the number of | |||
3155 | * cpus we need to send the IPI to, then we grab the counter, then | |||
3156 | * we send the IPIs, then we finally do our own shootdown. | |||
3157 | * | |||
3158 | * Our shootdown is last to make it parallel with the other cpus | |||
3159 | * to shorten the spin time. | |||
3160 | * | |||
3161 | * Notice that we depend on failures to send IPIs only being able to | |||
3162 | * happen during boot. If they happen later, the above assumption | |||
3163 | * doesn't hold since we can end up in situations where noone will | |||
3164 | * release the lock if we get an interrupt in a bad moment. | |||
3165 | */ | |||
3166 | #ifdef MP_LOCKDEBUG | |||
3167 | #include <ddb/db_output.h> | |||
3168 | extern int __mp_lock_spinout; | |||
3169 | #endif | |||
3170 | ||||
3171 | volatile long tlb_shoot_wait __attribute__((section(".kudata"))); | |||
3172 | ||||
3173 | volatile vaddr_t tlb_shoot_addr1 __attribute__((section(".kudata"))); | |||
3174 | volatile vaddr_t tlb_shoot_addr2 __attribute__((section(".kudata"))); | |||
3175 | volatile int tlb_shoot_first_pcid __attribute__((section(".kudata"))); | |||
3176 | ||||
3177 | ||||
3178 | /* Obtain the "lock" for TLB shooting */ | |||
3179 | static inline int | |||
3180 | pmap_start_tlb_shoot(long wait, const char *func) | |||
3181 | { | |||
3182 | int s = splvm()splraise(0xa); | |||
3183 | ||||
3184 | while (atomic_cas_ulong(&tlb_shoot_wait, 0, wait)_atomic_cas_ulong((&tlb_shoot_wait), (0), (wait)) != 0) { | |||
3185 | #ifdef MP_LOCKDEBUG | |||
3186 | int nticks = __mp_lock_spinout; | |||
3187 | #endif | |||
3188 | while (tlb_shoot_wait != 0) { | |||
3189 | CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory"); | |||
3190 | #ifdef MP_LOCKDEBUG | |||
3191 | if (--nticks <= 0) { | |||
3192 | db_printf("%s: spun out", func); | |||
3193 | db_enter(); | |||
3194 | nticks = __mp_lock_spinout; | |||
3195 | } | |||
3196 | #endif | |||
3197 | } | |||
3198 | } | |||
3199 | ||||
3200 | return s; | |||
3201 | } | |||
3202 | ||||
3203 | void | |||
3204 | pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself) | |||
3205 | { | |||
3206 | struct cpu_info *ci, *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;}); | |||
3207 | CPU_INFO_ITERATORint cii; | |||
3208 | long wait = 0; | |||
3209 | u_int64_t mask = 0; | |||
3210 | int is_kva = va >= VM_MIN_KERNEL_ADDRESS0xffff800000000000; | |||
3211 | ||||
3212 | CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci-> ci_next) { | |||
3213 | if (ci == self || !(ci->ci_flags & CPUF_RUNNING0x2000)) | |||
3214 | continue; | |||
3215 | if (!is_kva && !pmap_is_active(pm, ci)) | |||
3216 | continue; | |||
3217 | mask |= (1ULL << ci->ci_cpuid); | |||
3218 | wait++; | |||
3219 | } | |||
3220 | ||||
3221 | if (wait > 0) { | |||
3222 | int s = pmap_start_tlb_shoot(wait, __func__); | |||
3223 | ||||
3224 | tlb_shoot_first_pcid = is_kva ? PCID_KERN0 : PCID_PROC1; | |||
3225 | tlb_shoot_addr1 = va; | |||
3226 | CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci-> ci_next) { | |||
3227 | if ((mask & (1ULL << ci->ci_cpuid)) == 0) | |||
3228 | continue; | |||
3229 | if (x86_fast_ipi(ci, LAPIC_IPI_INVLPG(0xf0 + 1)) != 0) | |||
3230 | panic("%s: ipi failed", __func__); | |||
3231 | } | |||
3232 | splx(s)spllower(s); | |||
3233 | } | |||
3234 | ||||
3235 | if (!pmap_use_pcid) { | |||
3236 | if (shootself) | |||
3237 | pmap_update_pg(va); | |||
3238 | } else if (is_kva) { | |||
3239 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3240 | invpcid(INVPCID_ADDR0, PCID_KERN0, va); | |||
3241 | } else if (shootself) { | |||
3242 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3243 | if (cpu_meltdown) | |||
3244 | invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va); | |||
3245 | } | |||
3246 | } | |||
3247 | ||||
3248 | void | |||
3249 | pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself) | |||
3250 | { | |||
3251 | struct cpu_info *ci, *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;}); | |||
3252 | CPU_INFO_ITERATORint cii; | |||
3253 | long wait = 0; | |||
3254 | u_int64_t mask = 0; | |||
3255 | int is_kva = sva >= VM_MIN_KERNEL_ADDRESS0xffff800000000000; | |||
3256 | vaddr_t va; | |||
3257 | ||||
3258 | CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci-> ci_next) { | |||
3259 | if (ci == self || !(ci->ci_flags & CPUF_RUNNING0x2000)) | |||
3260 | continue; | |||
3261 | if (!is_kva && !pmap_is_active(pm, ci)) | |||
3262 | continue; | |||
3263 | mask |= (1ULL << ci->ci_cpuid); | |||
3264 | wait++; | |||
3265 | } | |||
3266 | ||||
3267 | if (wait > 0) { | |||
3268 | int s = pmap_start_tlb_shoot(wait, __func__); | |||
3269 | ||||
3270 | tlb_shoot_first_pcid = is_kva ? PCID_KERN0 : PCID_PROC1; | |||
3271 | tlb_shoot_addr1 = sva; | |||
3272 | tlb_shoot_addr2 = eva; | |||
3273 | CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci-> ci_next) { | |||
3274 | if ((mask & (1ULL << ci->ci_cpuid)) == 0) | |||
3275 | continue; | |||
3276 | if (x86_fast_ipi(ci, LAPIC_IPI_INVLRANGE(0xf0 + 2)) != 0) | |||
3277 | panic("%s: ipi failed", __func__); | |||
3278 | } | |||
3279 | splx(s)spllower(s); | |||
3280 | } | |||
3281 | ||||
3282 | if (!pmap_use_pcid) { | |||
3283 | if (shootself) { | |||
3284 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) | |||
3285 | pmap_update_pg(va); | |||
3286 | } | |||
3287 | } else if (is_kva) { | |||
3288 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) { | |||
3289 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3290 | invpcid(INVPCID_ADDR0, PCID_KERN0, va); | |||
3291 | } | |||
3292 | } else if (shootself) { | |||
3293 | if (cpu_meltdown) { | |||
3294 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) { | |||
3295 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3296 | invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va); | |||
3297 | } | |||
3298 | } else { | |||
3299 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) | |||
3300 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3301 | } | |||
3302 | } | |||
3303 | } | |||
3304 | ||||
3305 | void | |||
3306 | pmap_tlb_shoottlb(struct pmap *pm, int shootself) | |||
3307 | { | |||
3308 | struct cpu_info *ci, *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;}); | |||
3309 | CPU_INFO_ITERATORint cii; | |||
3310 | long wait = 0; | |||
3311 | u_int64_t mask = 0; | |||
3312 | ||||
3313 | KASSERT(pm != pmap_kernel())((pm != (&kernel_pmap_store)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/arch/amd64/amd64/pmap.c", 3313, "pm != pmap_kernel()" )); | |||
3314 | ||||
3315 | CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci-> ci_next) { | |||
3316 | if (ci == self || !pmap_is_active(pm, ci) || | |||
3317 | !(ci->ci_flags & CPUF_RUNNING0x2000)) | |||
3318 | continue; | |||
3319 | mask |= (1ULL << ci->ci_cpuid); | |||
3320 | wait++; | |||
3321 | } | |||
3322 | ||||
3323 | if (wait) { | |||
3324 | int s = pmap_start_tlb_shoot(wait, __func__); | |||
3325 | ||||
3326 | CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci-> ci_next) { | |||
3327 | if ((mask & (1ULL << ci->ci_cpuid)) == 0) | |||
3328 | continue; | |||
3329 | if (x86_fast_ipi(ci, LAPIC_IPI_INVLTLB(0xf0 + 0)) != 0) | |||
3330 | panic("%s: ipi failed", __func__); | |||
3331 | } | |||
3332 | splx(s)spllower(s); | |||
3333 | } | |||
3334 | ||||
3335 | if (shootself) { | |||
3336 | if (!pmap_use_pcid) | |||
3337 | tlbflush(); | |||
3338 | else { | |||
3339 | invpcid(INVPCID_PCID1, PCID_PROC1, 0); | |||
3340 | if (cpu_meltdown) | |||
3341 | invpcid(INVPCID_PCID1, PCID_PROC_INTEL2, 0); | |||
3342 | } | |||
3343 | } | |||
3344 | } | |||
3345 | ||||
3346 | void | |||
3347 | pmap_tlb_shootwait(void) | |||
3348 | { | |||
3349 | #ifdef MP_LOCKDEBUG | |||
3350 | int nticks = __mp_lock_spinout; | |||
3351 | #endif | |||
3352 | while (tlb_shoot_wait != 0) { | |||
3353 | CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory"); | |||
3354 | #ifdef MP_LOCKDEBUG | |||
3355 | if (--nticks <= 0) { | |||
3356 | db_printf("%s: spun out", __func__); | |||
3357 | db_enter(); | |||
3358 | nticks = __mp_lock_spinout; | |||
3359 | } | |||
3360 | #endif | |||
3361 | } | |||
3362 | } | |||
3363 | ||||
3364 | #else /* MULTIPROCESSOR */ | |||
3365 | ||||
3366 | void | |||
3367 | pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself) | |||
3368 | { | |||
3369 | if (!pmap_use_pcid) { | |||
3370 | if (shootself) | |||
3371 | pmap_update_pg(va); | |||
3372 | } else if (va >= VM_MIN_KERNEL_ADDRESS0xffff800000000000) { | |||
3373 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3374 | invpcid(INVPCID_ADDR0, PCID_KERN0, va); | |||
3375 | } else if (shootself) { | |||
3376 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3377 | if (cpu_meltdown) | |||
3378 | invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va); | |||
3379 | } | |||
3380 | } | |||
3381 | ||||
3382 | void | |||
3383 | pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself) | |||
3384 | { | |||
3385 | vaddr_t va; | |||
3386 | ||||
3387 | if (!pmap_use_pcid) { | |||
3388 | if (shootself) { | |||
3389 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) | |||
3390 | pmap_update_pg(va); | |||
3391 | } | |||
3392 | } else if (sva >= VM_MIN_KERNEL_ADDRESS0xffff800000000000) { | |||
3393 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) { | |||
3394 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3395 | invpcid(INVPCID_ADDR0, PCID_KERN0, va); | |||
3396 | } | |||
3397 | } else if (shootself) { | |||
3398 | if (cpu_meltdown) { | |||
3399 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) { | |||
3400 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3401 | invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va); | |||
3402 | } | |||
3403 | } else { | |||
3404 | for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) | |||
3405 | invpcid(INVPCID_ADDR0, PCID_PROC1, va); | |||
3406 | } | |||
3407 | } | |||
3408 | } | |||
3409 | ||||
3410 | void | |||
3411 | pmap_tlb_shoottlb(struct pmap *pm, int shootself) | |||
3412 | { | |||
3413 | if (shootself) { | |||
3414 | if (!pmap_use_pcid) | |||
3415 | tlbflush(); | |||
3416 | else { | |||
3417 | invpcid(INVPCID_PCID1, PCID_PROC1, 0); | |||
3418 | if (cpu_meltdown) | |||
3419 | invpcid(INVPCID_PCID1, PCID_PROC_INTEL2, 0); | |||
3420 | } | |||
3421 | } | |||
3422 | } | |||
3423 | #endif /* MULTIPROCESSOR */ |