Bug Summary

File:arch/amd64/amd64/cpu.c
Warning:line 1200, column 8
Assigned value is garbage or undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name cpu.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/arch/amd64/amd64/cpu.c
1/* $OpenBSD: cpu.c,v 1.154 2021/08/31 17:40:59 dv Exp $ */
2/* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
3
4/*-
5 * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by RedBack Networks Inc.
10 *
11 * Author: Bill Sommerfeld
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35/*
36 * Copyright (c) 1999 Stefan Grefen
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. All advertising materials mentioning features or use of this software
47 * must display the following acknowledgement:
48 * This product includes software developed by the NetBSD
49 * Foundation, Inc. and its contributors.
50 * 4. Neither the name of The NetBSD Foundation nor the names of its
51 * contributors may be used to endorse or promote products derived
52 * from this software without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
55 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 */
66
67#include "lapic.h"
68#include "ioapic.h"
69#include "vmm.h"
70#include "pctr.h"
71#include "pvbus.h"
72
73#include <sys/param.h>
74#include <sys/proc.h>
75#include <sys/timeout.h>
76#include <sys/systm.h>
77#include <sys/device.h>
78#include <sys/malloc.h>
79#include <sys/memrange.h>
80#include <sys/atomic.h>
81#include <sys/user.h>
82
83#include <uvm/uvm_extern.h>
84
85#include <machine/codepatch.h>
86#include <machine/cpu_full.h>
87#include <machine/cpufunc.h>
88#include <machine/cpuvar.h>
89#include <machine/pmap.h>
90#include <machine/vmparam.h>
91#include <machine/mpbiosvar.h>
92#include <machine/pcb.h>
93#include <machine/specialreg.h>
94#include <machine/segments.h>
95#include <machine/gdt.h>
96#include <machine/pio.h>
97#include <machine/vmmvar.h>
98
99#if NLAPIC1 > 0
100#include <machine/i82489reg.h>
101#include <machine/i82489var.h>
102#endif
103
104#if NIOAPIC1 > 0
105#include <machine/i82093var.h>
106#endif
107
108#if NPCTR1 > 0
109#include <machine/pctr.h>
110#endif
111
112#if NPVBUS1 > 0
113#include <dev/pv/pvvar.h>
114#endif
115
116#include <dev/ic/mc146818reg.h>
117#include <amd64/isa/nvram.h>
118#include <dev/isa/isareg.h>
119
120#ifdef HIBERNATE1
121#include <sys/hibernate.h>
122#include <machine/hibernate.h>
123#endif /* HIBERNATE */
124
125/* #define CPU_DEBUG */
126
127#ifdef CPU_DEBUG
128#define DPRINTF(x...) do { printf(x); } while(0)
129#else
130#define DPRINTF(x...)
131#endif /* CPU_DEBUG */
132
133int cpu_match(struct device *, void *, void *);
134void cpu_attach(struct device *, struct device *, void *);
135int cpu_activate(struct device *, int);
136void patinit(struct cpu_info *ci);
137#if NVMM1 > 0
138void cpu_init_vmm(struct cpu_info *ci);
139#endif /* NVMM > 0 */
140
141struct cpu_softc {
142 struct device sc_dev; /* device tree glue */
143 struct cpu_info *sc_info; /* pointer to CPU info */
144};
145
146void replacesmap(void);
147void replacemeltdown(void);
148void replacemds(void);
149
150extern long _stac;
151extern long _clac;
152
153void
154replacesmap(void)
155{
156 static int replacedone = 0;
157 int s;
158
159 if (replacedone)
160 return;
161 replacedone = 1;
162
163 s = splhigh()splraise(0xd);
164
165 codepatch_replace(CPTAG_STAC1, &_stac, 3);
166 codepatch_replace(CPTAG_CLAC2, &_clac, 3);
167
168 splx(s)spllower(s);
169}
170
171void
172replacemeltdown(void)
173{
174 static int replacedone = 0;
175 struct cpu_info *ci = &cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev)));
176 int swapgs_vuln = 0, s;
177
178 if (replacedone)
179 return;
180 replacedone = 1;
181
182 if (strcmp(cpu_vendor, "GenuineIntel") == 0) {
183 int family = ci->ci_family;
184 int model = ci->ci_model;
185
186 swapgs_vuln = 1;
187 if (family == 0x6 &&
188 (model == 0x37 || model == 0x4a || model == 0x4c ||
189 model == 0x4d || model == 0x5a || model == 0x5d ||
190 model == 0x6e || model == 0x65 || model == 0x75)) {
191 /* Silvermont, Airmont */
192 swapgs_vuln = 0;
193 } else if (family == 0x6 && (model == 0x85 || model == 0x57)) {
194 /* KnightsLanding */
195 swapgs_vuln = 0;
196 }
197 }
198
199 s = splhigh()splraise(0xd);
200 if (!cpu_meltdown)
201 codepatch_nop(CPTAG_MELTDOWN_NOP6);
202 else {
203 extern long alltraps_kern_meltdown;
204
205 /* eliminate conditional branch in alltraps */
206 codepatch_jmp(CPTAG_MELTDOWN_ALLTRAPS7, &alltraps_kern_meltdown);
207
208 /* enable reuse of PCID for U-K page tables */
209 if (pmap_use_pcid) {
210 extern long _pcid_set_reuse;
211 DPRINTF("%s: codepatching PCID use", __func__);
212 codepatch_replace(CPTAG_PCID_SET_REUSE8,
213 &_pcid_set_reuse, PCID_SET_REUSE_SIZE12);
214 }
215 }
216
217 /*
218 * CVE-2019-1125: if the CPU has SMAP and it's not vulnerable to
219 * Meltdown, then it's protected both from speculatively mis-skipping
220 * the swapgs during interrupts of userspace and from speculatively
221 * mis-taking a swapgs during interrupts while already in the kernel
222 * as the speculative path will fault from SMAP. Warning: enabling
223 * WRGSBASE would break this 'protection'.
224 *
225 * Otherwise, if the CPU's swapgs can't be speculated over and it
226 * _is_ vulnerable to Meltdown then the %cr3 change will serialize
227 * user->kern transitions, but we still need to mitigate the
228 * already-in-kernel cases.
229 */
230 if (!cpu_meltdown && (ci->ci_feature_sefflags_ebx & SEFF0EBX_SMAP0x00100000)) {
231 codepatch_nop(CPTAG_FENCE_SWAPGS_MIS_TAKEN11);
232 codepatch_nop(CPTAG_FENCE_NO_SAFE_SMAP12);
233 } else if (!swapgs_vuln && cpu_meltdown) {
234 codepatch_nop(CPTAG_FENCE_SWAPGS_MIS_TAKEN11);
235 }
236 splx(s)spllower(s);
237}
238
239void
240replacemds(void)
241{
242 static int replacedone = 0;
243 extern long mds_handler_bdw, mds_handler_ivb, mds_handler_skl;
244 extern long mds_handler_skl_sse, mds_handler_skl_avx;
245 extern long mds_handler_silvermont, mds_handler_knights;
246 struct cpu_info *ci = &cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev)));
247 CPU_INFO_ITERATORint cii;
248 void *handler = NULL((void *)0), *vmm_handler = NULL((void *)0);
249 const char *type;
250 int has_verw, s;
251
252 /* ci_mds_tmp must be 32byte aligned for AVX instructions */
253 CTASSERT((offsetof(struct cpu_info, ci_mds_tmp) -extern char _ctassert[((__builtin_offsetof(struct cpu_info, ci_mds_tmp
) - __builtin_offsetof(struct cpu_info, ci_dev)) % 32 == 0) ?
1 : -1 ] __attribute__((__unused__))
254 offsetof(struct cpu_info, ci_PAGEALIGN)) % 32 == 0)extern char _ctassert[((__builtin_offsetof(struct cpu_info, ci_mds_tmp
) - __builtin_offsetof(struct cpu_info, ci_dev)) % 32 == 0) ?
1 : -1 ] __attribute__((__unused__));
255
256 if (replacedone)
257 return;
258 replacedone = 1;
259
260 if (strcmp(cpu_vendor, "GenuineIntel") != 0 ||
261 ((ci->ci_feature_sefflags_edx & SEFF0EDX_ARCH_CAP0x20000000) &&
262 (rdmsr(MSR_ARCH_CAPABILITIES0x10a) & ARCH_CAPABILITIES_MDS_NO(1 << 5)))) {
263 /* Unaffected, nop out the handling code */
264 has_verw = 0;
265 } else if (ci->ci_feature_sefflags_edx & SEFF0EDX_MD_CLEAR0x00000400) {
266 /* new firmware, use VERW */
267 has_verw = 1;
268 } else {
269 int family = ci->ci_family;
270 int model = ci->ci_model;
271 int stepping = CPUID2STEPPING(ci->ci_signature)((ci->ci_signature) & 15);
272
273 has_verw = 0;
274 if (family == 0x6 &&
275 (model == 0x2e || model == 0x1e || model == 0x1f ||
276 model == 0x1a || model == 0x2f || model == 0x25 ||
277 model == 0x2c || model == 0x2d || model == 0x2a ||
278 model == 0x3e || model == 0x3a)) {
279 /* Nehalem, SandyBridge, IvyBridge */
280 handler = vmm_handler = &mds_handler_ivb;
281 type = "IvyBridge";
282 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
283 ci->ci_mds_buf = malloc(672, M_DEVBUF2,
284 M_WAITOK0x0001);
285 memset(ci->ci_mds_buf, 0, 16)__builtin_memset((ci->ci_mds_buf), (0), (16));
286 }
287 } else if (family == 0x6 &&
288 (model == 0x3f || model == 0x3c || model == 0x45 ||
289 model == 0x46 || model == 0x56 || model == 0x4f ||
290 model == 0x47 || model == 0x3d)) {
291 /* Haswell and Broadwell */
292 handler = vmm_handler = &mds_handler_bdw;
293 type = "Broadwell";
294 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
295 ci->ci_mds_buf = malloc(1536, M_DEVBUF2,
296 M_WAITOK0x0001);
297 }
298 } else if (family == 0x6 &&
299 ((model == 0x55 && stepping <= 5) || model == 0x4e ||
300 model == 0x5e || (model == 0x8e && stepping <= 0xb) ||
301 (model == 0x9e && stepping <= 0xc))) {
302 /*
303 * Skylake, KabyLake, CoffeeLake, WhiskeyLake,
304 * CascadeLake
305 */
306 /* XXX mds_handler_skl_avx512 */
307 if (xgetbv(0) & XCR0_AVX0x00000004) {
308 handler = &mds_handler_skl_avx;
309 type = "Skylake AVX";
310 } else {
311 handler = &mds_handler_skl_sse;
312 type = "Skylake SSE";
313 }
314 vmm_handler = &mds_handler_skl;
315 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
316 vaddr_t b64;
317 b64 = (vaddr_t)malloc(6 * 1024 + 64 + 63,
318 M_DEVBUF2, M_WAITOK0x0001);
319 ci->ci_mds_buf = (void *)((b64 + 63) & ~63);
320 memset(ci->ci_mds_buf, 0, 64)__builtin_memset((ci->ci_mds_buf), (0), (64));
321 }
322 } else if (family == 0x6 &&
323 (model == 0x37 || model == 0x4a || model == 0x4c ||
324 model == 0x4d || model == 0x5a || model == 0x5d ||
325 model == 0x6e || model == 0x65 || model == 0x75)) {
326 /* Silvermont, Airmont */
327 handler = vmm_handler = &mds_handler_silvermont;
328 type = "Silvermont";
329 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
330 ci->ci_mds_buf = malloc(256, M_DEVBUF2,
331 M_WAITOK0x0001);
332 memset(ci->ci_mds_buf, 0, 16)__builtin_memset((ci->ci_mds_buf), (0), (16));
333 }
334 } else if (family == 0x6 && (model == 0x85 || model == 0x57)) {
335 handler = vmm_handler = &mds_handler_knights;
336 type = "KnightsLanding";
337 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
338 vaddr_t b64;
339 b64 = (vaddr_t)malloc(1152 + 63, M_DEVBUF2,
340 M_WAITOK0x0001);
341 ci->ci_mds_buf = (void *)((b64 + 63) & ~63);
342 }
343 }
344 }
345
346 if (handler != NULL((void *)0)) {
347 printf("cpu0: using %s MDS workaround%s\n", type, "");
348 s = splhigh()splraise(0xd);
349 codepatch_call(CPTAG_MDS9, handler);
350 codepatch_call(CPTAG_MDS_VMM10, vmm_handler);
351 splx(s)spllower(s);
352 } else if (has_verw) {
353 /* The new firmware enhances L1D_FLUSH MSR to flush MDS too */
354 if (cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev))).ci_vmm_cap.vcc_vmx.vmx_has_l1_flush_msr == 1) {
355 s = splhigh()splraise(0xd);
356 codepatch_nop(CPTAG_MDS_VMM10);
357 splx(s)spllower(s);
358 type = " (except on vmm entry)";
359 } else {
360 type = "";
361 }
362 printf("cpu0: using %s MDS workaround%s\n", "VERW", type);
363 } else {
364 s = splhigh()splraise(0xd);
365 codepatch_nop(CPTAG_MDS9);
366 codepatch_nop(CPTAG_MDS_VMM10);
367 splx(s)spllower(s);
368 }
369}
370
371#ifdef MULTIPROCESSOR1
372int mp_cpu_start(struct cpu_info *);
373void mp_cpu_start_cleanup(struct cpu_info *);
374struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL((void *)0),
375 mp_cpu_start_cleanup };
376#endif /* MULTIPROCESSOR */
377
378struct cfattach cpu_ca = {
379 sizeof(struct cpu_softc), cpu_match, cpu_attach, NULL((void *)0), cpu_activate
380};
381
382struct cfdriver cpu_cd = {
383 NULL((void *)0), "cpu", DV_DULL
384};
385
386/*
387 * Statically-allocated CPU info for the primary CPU (or the only
388 * CPU, on uniprocessors). The CPU info list is initialized to
389 * point at it.
390 */
391struct cpu_info_full cpu_info_full_primary = { .cif_cpu = { .ci_self = &cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev))) } };
392
393struct cpu_info *cpu_info_list = &cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev)));
394
395#ifdef MULTIPROCESSOR1
396/*
397 * Array of CPU info structures. Must be statically-allocated because
398 * curproc, etc. are used early.
399 */
400struct cpu_info *cpu_info[MAXCPUS64] = { &cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev))) };
401
402void cpu_hatch(void *);
403void cpu_boot_secondary(struct cpu_info *ci);
404void cpu_start_secondary(struct cpu_info *ci);
405#endif
406
407int
408cpu_match(struct device *parent, void *match, void *aux)
409{
410 struct cfdata *cf = match;
411 struct cpu_attach_args *caa = aux;
412
413 if (strcmp(caa->caa_name, cf->cf_driver->cd_name) != 0)
414 return 0;
415
416 if (cf->cf_unit >= MAXCPUS64)
417 return 0;
418
419 return 1;
420}
421
422void cpu_idle_mwait_cycle(void);
423void cpu_init_mwait(struct cpu_softc *);
424
425u_int cpu_mwait_size, cpu_mwait_states;
426
427void
428cpu_idle_mwait_cycle(void)
429{
430 struct cpu_info *ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;});
431
432 if ((read_rflags() & PSL_I0x00000200) == 0)
433 panic("idle with interrupts blocked!");
434
435 /* something already queued? */
436 if (!cpu_is_idle(ci)((ci)->ci_schedstate.spc_whichqs == 0))
437 return;
438
439 /*
440 * About to idle; setting the MWAIT_IN_IDLE bit tells
441 * cpu_unidle() that it can't be a no-op and tells cpu_kick()
442 * that it doesn't need to use an IPI. We also set the
443 * MWAIT_KEEP_IDLING bit: those routines clear it to stop
444 * the mwait. Once they're set, we do a final check of the
445 * queue, in case another cpu called setrunqueue() and added
446 * something to the queue and called cpu_unidle() between
447 * the check in sched_idle() and here.
448 */
449 atomic_setbits_intx86_atomic_setbits_u32(&ci->ci_mwait, MWAIT_IDLING(0x1 | 0x2) | MWAIT_ONLY0x4);
450 if (cpu_is_idle(ci)((ci)->ci_schedstate.spc_whichqs == 0)) {
451 monitor(&ci->ci_mwait, 0, 0);
452 if ((ci->ci_mwait & MWAIT_IDLING(0x1 | 0x2)) == MWAIT_IDLING(0x1 | 0x2))
453 mwait(0, 0);
454 }
455
456 /* done idling; let cpu_kick() know that an IPI is required */
457 atomic_clearbits_intx86_atomic_clearbits_u32(&ci->ci_mwait, MWAIT_IDLING(0x1 | 0x2));
458}
459
460void
461cpu_init_mwait(struct cpu_softc *sc)
462{
463 unsigned int smallest, largest, extensions, c_substates;
464
465 if ((cpu_ecxfeature & CPUIDECX_MWAIT0x00000008) == 0 || cpuid_level < 0x5)
466 return;
467
468 /* get the monitor granularity */
469 CPUID(0x5, smallest, largest, extensions, cpu_mwait_states)__asm volatile("cpuid" : "=a" (smallest), "=b" (largest), "=c"
(extensions), "=d" (cpu_mwait_states) : "a" (0x5));
470 smallest &= 0xffff;
471 largest &= 0xffff;
472
473 printf("%s: mwait min=%u, max=%u", sc->sc_dev.dv_xname,
474 smallest, largest);
475 if (extensions & 0x1) {
476 if (cpu_mwait_states > 0) {
477 c_substates = cpu_mwait_states;
478 printf(", C-substates=%u", 0xf & c_substates);
479 while ((c_substates >>= 4) > 0)
480 printf(".%u", 0xf & c_substates);
481 }
482 if (extensions & 0x2)
483 printf(", IBE");
484 } else {
485 /* substates not supported, forge the default: just C1 */
486 cpu_mwait_states = 1 << 4;
487 }
488
489 /* paranoia: check the values */
490 if (smallest < sizeof(int) || largest < smallest ||
491 (largest & (sizeof(int)-1)))
492 printf(" (bogus)");
493 else
494 cpu_mwait_size = largest;
495 printf("\n");
496
497 /* enable use of mwait; may be overridden by acpicpu later */
498 if (cpu_mwait_size > 0)
499 cpu_idle_cycle_fcn = &cpu_idle_mwait_cycle;
500}
501
502void
503cpu_attach(struct device *parent, struct device *self, void *aux)
504{
505 struct cpu_softc *sc = (void *) self;
506 struct cpu_attach_args *caa = aux;
507 struct cpu_info *ci;
508#if defined(MULTIPROCESSOR1)
509 int cpunum = sc->sc_dev.dv_unit;
510 vaddr_t kstack;
511 struct pcb *pcb;
512#endif
513
514 /*
515 * If we're an Application Processor, allocate a cpu_info
516 * structure, otherwise use the primary's.
517 */
518 if (caa->cpu_role == CPU_ROLE_AP2) {
519 struct cpu_info_full *cif;
520
521 cif = km_alloc(sizeof *cif, &kv_any, &kp_zero, &kd_waitok);
522 ci = &cif->cif_cpu;
523#if defined(MULTIPROCESSOR1)
524 ci->ci_tss = &cif->cif_tsscif_RO.u_tssgdt.uu_tss;
525 ci->ci_gdt = &cif->cif_gdtcif_RO.u_tssgdt.uu_gdt;
526 memcpy(ci->ci_gdt, cpu_info_primary.ci_gdt, GDT_SIZE)__builtin_memcpy((ci->ci_gdt), ((*(struct cpu_info *)((char
*)&cpu_info_full_primary + 4096*2 - __builtin_offsetof(struct
cpu_info, ci_dev))).ci_gdt), (((6 << 3) + (1 << 4
))));
527 cpu_enter_pages(cif);
528 if (cpu_info[cpunum] != NULL((void *)0))
529 panic("cpu at apic id %d already attached?", cpunum);
530 cpu_info[cpunum] = ci;
531#endif
532#ifdef TRAPLOG
533 ci->ci_tlog_base = malloc(sizeof(struct tlog),
534 M_DEVBUF2, M_WAITOK0x0001);
535#endif
536 } else {
537 ci = &cpu_info_primary(*(struct cpu_info *)((char *)&cpu_info_full_primary + 4096
*2 - __builtin_offsetof(struct cpu_info, ci_dev)));
538#if defined(MULTIPROCESSOR1)
539 if (caa->cpu_apicid != lapic_cpu_number()) {
540 panic("%s: running cpu is at apic %d"
541 " instead of at expected %d",
542 sc->sc_dev.dv_xname, lapic_cpu_number(), caa->cpu_apicid);
543 }
544#endif
545 }
546
547 ci->ci_self = ci;
548 sc->sc_info = ci;
549
550 ci->ci_dev = self;
551 ci->ci_apicid = caa->cpu_apicid;
552 ci->ci_acpi_proc_id = caa->cpu_acpi_proc_id;
553#ifdef MULTIPROCESSOR1
554 ci->ci_cpuid = cpunum;
555#else
556 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */
557#endif
558 ci->ci_func = caa->cpu_func;
559 ci->ci_handled_intr_level = IPL_NONE0x0;
560
561#if defined(MULTIPROCESSOR1)
562 /*
563 * Allocate UPAGES contiguous pages for the idle PCB and stack.
564 */
565 kstack = (vaddr_t)km_alloc(USPACE(6 * (1 << 12)), &kv_any, &kp_dirty, &kd_nowait);
566 if (kstack == 0) {
567 if (caa->cpu_role != CPU_ROLE_AP2) {
568 panic("cpu_attach: unable to allocate idle stack for"
569 " primary");
570 }
571 printf("%s: unable to allocate idle stack\n",
572 sc->sc_dev.dv_xname);
573 return;
574 }
575 pcb = ci->ci_idle_pcb = (struct pcb *) kstack;
576 memset(pcb, 0, USPACE)__builtin_memset((pcb), (0), ((6 * (1 << 12))));
577
578 pcb->pcb_kstack = kstack + USPACE(6 * (1 << 12)) - 16;
579 pcb->pcb_rbp = pcb->pcb_rsp = kstack + USPACE(6 * (1 << 12)) - 16;
580 pcb->pcb_pmap = pmap_kernel()(&kernel_pmap_store);
581 pcb->pcb_cr3 = pcb->pcb_pmap->pm_pdirpa;
582#endif
583
584 /* further PCB init done later. */
585
586 printf(": ");
587
588 switch (caa->cpu_role) {
589 case CPU_ROLE_SP0:
590 printf("(uniprocessor)\n");
591 ci->ci_flags |= CPUF_PRESENT0x1000 | CPUF_SP0x0004 | CPUF_PRIMARY0x0008;
592 cpu_intr_init(ci);
593#ifndef SMALL_KERNEL
594 cpu_ucode_apply(ci);
595#endif
596 cpu_tsx_disable(ci);
597 identifycpu(ci);
598#ifdef MTRR1
599 mem_range_attach();
600#endif /* MTRR */
601 /* XXX SP fpuinit(ci) is done earlier */
602 cpu_init(ci);
603 cpu_init_mwait(sc);
604 break;
605
606 case CPU_ROLE_BP1:
607 printf("apid %d (boot processor)\n", caa->cpu_apicid);
608 ci->ci_flags |= CPUF_PRESENT0x1000 | CPUF_BSP0x0001 | CPUF_PRIMARY0x0008;
609 cpu_intr_init(ci);
610 identifycpu(ci);
611#ifdef MTRR1
612 mem_range_attach();
613#endif /* MTRR */
614
615#if NLAPIC1 > 0
616 /*
617 * Enable local apic
618 */
619 lapic_enable();
620 lapic_calibrate_timer(ci);
621#endif
622 /* XXX BP fpuinit(ci) is done earlier */
623 cpu_init(ci);
624
625#if NIOAPIC1 > 0
626 ioapic_bsp_id = caa->cpu_apicid;
627#endif
628 cpu_init_mwait(sc);
629 break;
630
631 case CPU_ROLE_AP2:
632 /*
633 * report on an AP
634 */
635 printf("apid %d (application processor)\n", caa->cpu_apicid);
636
637#if defined(MULTIPROCESSOR1)
638 cpu_intr_init(ci);
639 cpu_start_secondary(ci);
640 sched_init_cpu(ci);
641 ncpus++;
642 if (ci->ci_flags & CPUF_PRESENT0x1000) {
643 ci->ci_next = cpu_info_list->ci_next;
644 cpu_info_list->ci_next = ci;
645 }
646#else
647 printf("%s: not started\n", sc->sc_dev.dv_xname);
648#endif
649 break;
650
651 default:
652 panic("unknown processor type??");
653 }
654
655#if defined(MULTIPROCESSOR1)
656 if (mp_verbose) {
657 printf("%s: kstack at 0x%lx for %d bytes\n",
658 sc->sc_dev.dv_xname, kstack, USPACE(6 * (1 << 12)));
659 printf("%s: idle pcb at %p, idle sp at 0x%llx\n",
660 sc->sc_dev.dv_xname, pcb, pcb->pcb_rsp);
661 }
662#endif
663#if NVMM1 > 0
664 cpu_init_vmm(ci);
665#endif /* NVMM > 0 */
666}
667
668static void
669replacexsave(void)
670{
671 extern long _xrstor, _xsave, _xsaveopt;
672 u_int32_t eax, ebx, ecx, edx;
673 static int replacedone = 0;
674 int s;
675
676 if (replacedone)
677 return;
678 replacedone = 1;
679
680 /* find out whether xsaveopt is supported */
681 CPUID_LEAF(0xd, 1, eax, ebx, ecx, edx)__asm volatile("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d"
(edx) : "a" (0xd), "c" (1));
682 s = splhigh()splraise(0xd);
683 codepatch_replace(CPTAG_XRSTOR4, &_xrstor, 4);
684 codepatch_replace(CPTAG_XSAVE5,
685 (eax & XSAVE_XSAVEOPT0x1UL) ? &_xsaveopt : &_xsave, 4);
686 splx(s)spllower(s);
687}
688
689
690/*
691 * Initialize the processor appropriately.
692 */
693
694void
695cpu_init(struct cpu_info *ci)
696{
697 struct savefpu *sfp;
698 u_int cr4;
699
700 /* configure the CPU if needed */
701 if (ci->cpu_setup != NULL((void *)0))
702 (*ci->cpu_setup)(ci);
703
704 cr4 = rcr4() | CR4_DEFAULT(0x00000020|0x00000080|0x00000010|0x00000200|0x00000400);
705 if (ci->ci_feature_sefflags_ebx & SEFF0EBX_SMEP0x00000080)
706 cr4 |= CR4_SMEP0x00100000;
707 if (ci->ci_feature_sefflags_ebx & SEFF0EBX_SMAP0x00100000)
708 cr4 |= CR4_SMAP0x00200000;
709 if (ci->ci_feature_sefflags_ecx & SEFF0ECX_UMIP0x00000004)
710 cr4 |= CR4_UMIP0x00000800;
711 if ((cpu_ecxfeature & CPUIDECX_XSAVE0x04000000) && cpuid_level >= 0xd)
712 cr4 |= CR4_OSXSAVE0x00040000;
713 if (pmap_use_pcid)
714 cr4 |= CR4_PCIDE0x00020000;
715 lcr4(cr4);
716
717 if ((cpu_ecxfeature & CPUIDECX_XSAVE0x04000000) && cpuid_level >= 0xd) {
718 u_int32_t eax, ebx, ecx, edx;
719
720 xsave_mask = XCR0_X870x00000001 | XCR0_SSE0x00000002;
721 CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx)__asm volatile("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d"
(edx) : "a" (0xd), "c" (0));
722 if (eax & XCR0_AVX0x00000004)
723 xsave_mask |= XCR0_AVX0x00000004;
724 xsetbv(0, xsave_mask);
725 CPUID_LEAF(0xd, 0, eax, ebx, ecx, edx)__asm volatile("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d"
(edx) : "a" (0xd), "c" (0));
726 if (CPU_IS_PRIMARY(ci)((ci)->ci_flags & 0x0008)) {
727 fpu_save_len = ebx;
728 KASSERT(fpu_save_len <= sizeof(struct savefpu))((fpu_save_len <= sizeof(struct savefpu)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/arch/amd64/amd64/cpu.c", 728, "fpu_save_len <= sizeof(struct savefpu)"
));
729 } else {
730 KASSERT(ebx == fpu_save_len)((ebx == fpu_save_len) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/cpu.c"
, 730, "ebx == fpu_save_len"));
731 }
732
733 replacexsave();
734 }
735
736 /* Give proc0 a clean FPU save area */
737 sfp = &proc0.p_addr->u_pcb.pcb_savefpu;
738 memset(sfp, 0, fpu_save_len)__builtin_memset((sfp), (0), (fpu_save_len));
739 sfp->fp_fxsave.fx_fcw = __INITIAL_NPXCW__0x037f;
740 sfp->fp_fxsave.fx_mxcsr = __INITIAL_MXCSR__0x1f80;
741 fpureset()xrstor_user(&proc0.p_addr->u_pcb.pcb_savefpu, xsave_mask
);
742 if (xsave_mask) {
743 /* must not use xsaveopt here */
744 xsave(sfp, xsave_mask);
745 } else
746 fxsave(sfp)__asm("fxsave %0" : "=m" (*sfp));
747
748#if NVMM1 > 0
749 /* Re-enable VMM if needed */
750 if (ci->ci_flags & CPUF_VMM0x20000)
751 start_vmm_on_cpu(ci);
752#endif /* NVMM > 0 */
753
754#ifdef MULTIPROCESSOR1
755 ci->ci_flags |= CPUF_RUNNING0x2000;
756 /*
757 * Big hammer: flush all TLB entries, including ones from PTEs
758 * with the G bit set. This should only be necessary if TLB
759 * shootdown falls far behind.
760 */
761 cr4 = rcr4();
762 lcr4(cr4 & ~CR4_PGE0x00000080);
763 lcr4(cr4);
764
765 /* Synchronize TSC */
766 if (cold && !CPU_IS_PRIMARY(ci)((ci)->ci_flags & 0x0008))
767 tsc_sync_ap(ci);
768#endif
769}
770
771#if NVMM1 > 0
772/*
773 * cpu_init_vmm
774 *
775 * Initializes per-cpu VMM state
776 *
777 * Parameters:
778 * ci: the cpu for which state is being initialized
779 */
780void
781cpu_init_vmm(struct cpu_info *ci)
782{
783 /*
784 * Allocate a per-cpu VMXON region for VMX CPUs
785 */
786 if (ci->ci_vmm_flags & CI_VMM_VMX(1 << 0)) {
787 ci->ci_vmxon_region = (struct vmxon_region *)malloc(PAGE_SIZE(1 << 12),
788 M_DEVBUF2, M_WAITOK0x0001 | M_ZERO0x0008);
789 if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)ci->ci_vmxon_region,
790 &ci->ci_vmxon_region_pa))
791 panic("Can't locate VMXON region in phys mem");
792 ci->ci_vmcs_pa = VMX_VMCS_PA_CLEAR0xFFFFFFFFFFFFFFFFUL;
793 rw_init(&ci->ci_vmcs_lock, "vmcslock")_rw_init_flags(&ci->ci_vmcs_lock, "vmcslock", 0, ((void
*)0));
794 }
795}
796#endif /* NVMM > 0 */
797
798#ifdef MULTIPROCESSOR1
799void
800cpu_boot_secondary_processors(void)
801{
802 struct cpu_info *ci;
803 u_long i;
804
805 for (i=0; i < MAXCPUS64; i++) {
806 ci = cpu_info[i];
807 if (ci == NULL((void *)0))
808 continue;
809 if (ci->ci_idle_pcb == NULL((void *)0))
810 continue;
811 if ((ci->ci_flags & CPUF_PRESENT0x1000) == 0)
812 continue;
813 if (ci->ci_flags & (CPUF_BSP0x0001 | CPUF_SP0x0004 | CPUF_PRIMARY0x0008))
814 continue;
815 ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
816 cpu_boot_secondary(ci);
817 }
818}
819
820void
821cpu_start_secondary(struct cpu_info *ci)
822{
823 int i;
824 u_long s;
825
826 ci->ci_flags |= CPUF_AP0x0002;
827
828 pmap_kenter_pa(MP_TRAMPOLINE(16 * (1 << 12)), MP_TRAMPOLINE(16 * (1 << 12)), PROT_READ0x01 | PROT_EXEC0x04);
829 pmap_kenter_pa(MP_TRAMP_DATA(17 * (1 << 12)), MP_TRAMP_DATA(17 * (1 << 12)), PROT_READ0x01 | PROT_WRITE0x02);
830
831 CPU_STARTUP(ci)((ci)->ci_func->start(ci));
832
833 /*
834 * wait for it to become ready
835 */
836 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT0x1000)) && i>0;i--) {
837 delay(10)(*delay_func)(10);
838 }
839 if (! (ci->ci_flags & CPUF_PRESENT0x1000)) {
840 printf("%s: failed to become ready\n", ci->ci_dev->dv_xname);
841#if defined(MPDEBUG) && defined(DDB1)
842 printf("dropping into debugger; continue from here to resume boot\n");
843 db_enter();
844#endif
845 } else {
846 /*
847 * Synchronize time stamp counters. Invalidate cache and
848 * synchronize twice (in tsc_sync_bp) to minimize possible
849 * cache effects. Disable interrupts to try and rule out any
850 * external interference.
851 */
852 s = intr_disable();
853 wbinvd();
854 tsc_sync_bp(ci);
855 intr_restore(s);
856#ifdef TSC_DEBUG
857 printf("TSC skew=%lld\n", (long long)ci->ci_tsc_skew);
858#endif
859 }
860
861 if ((ci->ci_flags & CPUF_IDENTIFIED0x0020) == 0) {
862 atomic_setbits_intx86_atomic_setbits_u32(&ci->ci_flags, CPUF_IDENTIFY0x0010);
863
864 /* wait for it to identify */
865 for (i = 2000000; (ci->ci_flags & CPUF_IDENTIFY0x0010) && i > 0; i--)
866 delay(10)(*delay_func)(10);
867
868 if (ci->ci_flags & CPUF_IDENTIFY0x0010)
869 printf("%s: failed to identify\n",
870 ci->ci_dev->dv_xname);
871 }
872
873 CPU_START_CLEANUP(ci)((ci)->ci_func->cleanup(ci));
874
875 pmap_kremove(MP_TRAMPOLINE(16 * (1 << 12)), PAGE_SIZE(1 << 12));
876 pmap_kremove(MP_TRAMP_DATA(17 * (1 << 12)), PAGE_SIZE(1 << 12));
877}
878
879void
880cpu_boot_secondary(struct cpu_info *ci)
881{
882 int i;
883 int64_t drift;
884 u_long s;
885
886 atomic_setbits_intx86_atomic_setbits_u32(&ci->ci_flags, CPUF_GO0x8000);
887
888 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING0x2000)) && i>0;i--) {
889 delay(10)(*delay_func)(10);
890 }
891 if (! (ci->ci_flags & CPUF_RUNNING0x2000)) {
892 printf("cpu failed to start\n");
893#if defined(MPDEBUG) && defined(DDB1)
894 printf("dropping into debugger; continue from here to resume boot\n");
895 db_enter();
896#endif
897 } else if (cold) {
898 /* Synchronize TSC again, check for drift. */
899 drift = ci->ci_tsc_skew;
900 s = intr_disable();
901 wbinvd();
902 tsc_sync_bp(ci);
903 intr_restore(s);
904 drift -= ci->ci_tsc_skew;
905#ifdef TSC_DEBUG
906 printf("TSC skew=%lld drift=%lld\n",
907 (long long)ci->ci_tsc_skew, (long long)drift);
908#endif
909 tsc_sync_drift(drift);
910 }
911}
912
913/*
914 * The CPU ends up here when it's ready to run
915 * This is called from code in mptramp.s; at this point, we are running
916 * in the idle pcb/idle stack of the new cpu. When this function returns,
917 * this processor will enter the idle loop and start looking for work.
918 *
919 * XXX should share some of this with init386 in machdep.c
920 */
921void
922cpu_hatch(void *v)
923{
924 struct cpu_info *ci = (struct cpu_info *)v;
925 int s;
926
927 cpu_init_msrs(ci);
928
929#ifdef DEBUG
930 if (ci->ci_flags & CPUF_PRESENT0x1000)
931 panic("%s: already running!?", ci->ci_dev->dv_xname);
932#endif
933
934 /*
935 * Synchronize the TSC for the first time. Note that interrupts are
936 * off at this point.
937 */
938 wbinvd();
939 ci->ci_flags |= CPUF_PRESENT0x1000;
940 ci->ci_tsc_skew = 0; /* reset on resume */
941 tsc_sync_ap(ci);
942
943 lapic_enable();
944 lapic_startclock();
945 cpu_ucode_apply(ci);
946 cpu_tsx_disable(ci);
947
948 if ((ci->ci_flags & CPUF_IDENTIFIED0x0020) == 0) {
949 /*
950 * We need to wait until we can identify, otherwise dmesg
951 * output will be messy.
952 */
953 while ((ci->ci_flags & CPUF_IDENTIFY0x0010) == 0)
954 delay(10)(*delay_func)(10);
955
956 identifycpu(ci);
957
958 /* Signal we're done */
959 atomic_clearbits_intx86_atomic_clearbits_u32(&ci->ci_flags, CPUF_IDENTIFY0x0010);
960 /* Prevent identifycpu() from running again */
961 atomic_setbits_intx86_atomic_setbits_u32(&ci->ci_flags, CPUF_IDENTIFIED0x0020);
962 }
963
964 while ((ci->ci_flags & CPUF_GO0x8000) == 0)
965 delay(10)(*delay_func)(10);
966#ifdef HIBERNATE1
967 if ((ci->ci_flags & CPUF_PARK0x10000) != 0) {
968 atomic_clearbits_intx86_atomic_clearbits_u32(&ci->ci_flags, CPUF_PARK0x10000);
969 hibernate_drop_to_real_mode();
970 }
971#endif /* HIBERNATE */
972
973#ifdef DEBUG
974 if (ci->ci_flags & CPUF_RUNNING0x2000)
975 panic("%s: already running!?", ci->ci_dev->dv_xname);
976#endif
977
978 cpu_init_idt();
979 lapic_set_lvt();
980 gdt_init_cpu(ci);
981 fpuinit(ci);
982
983 lldt(0);
984
985 cpu_init(ci);
986#if NPVBUS1 > 0
987 pvbus_init_cpu();
988#endif
989
990 /* Re-initialise memory range handling on AP */
991 if (mem_range_softc.mr_op != NULL((void *)0))
992 mem_range_softc.mr_op->initAP(&mem_range_softc);
993
994 s = splhigh()splraise(0xd);
995 lcr8(0);
996 intr_enable();
997
998 nanouptime(&ci->ci_schedstate.spc_runtime);
999 splx(s)spllower(s);
1000
1001 SCHED_LOCK(s)do { s = splraise(0xc); __mp_lock(&sched_lock); } while (
0);
1002 cpu_switchto(NULL((void *)0), sched_chooseproc());
1003}
1004
1005#if defined(DDB1)
1006
1007#include <ddb/db_output.h>
1008#include <machine/db_machdep.h>
1009
1010/*
1011 * Dump cpu information from ddb.
1012 */
1013void
1014cpu_debug_dump(void)
1015{
1016 struct cpu_info *ci;
1017 CPU_INFO_ITERATORint cii;
1018
1019 db_printf("addr dev id flags ipis curproc\n");
1020 CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
1021 db_printf("%p %s %u %x %x %10p\n",
1022 ci,
1023 ci->ci_dev == NULL((void *)0) ? "BOOT" : ci->ci_dev->dv_xname,
1024 ci->ci_cpuid,
1025 ci->ci_flags, ci->ci_ipis,
1026 ci->ci_curproc);
1027 }
1028}
1029#endif
1030
1031int
1032mp_cpu_start(struct cpu_info *ci)
1033{
1034 unsigned short dwordptr[2];
1035
1036 /*
1037 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
1038 */
1039
1040 outb(IO_RTC, NVRAM_RESET)( (__builtin_constant_p((0x070)) && (0x070) < 0x100
) ? __outbc(0x070, (0xe + 1)) : __outb(0x070, (0xe + 1)));
1041 outb(IO_RTC+1, NVRAM_RESET_JUMP)( (__builtin_constant_p((0x070 +1)) && (0x070 +1) <
0x100) ? __outbc(0x070 +1, 0x0a) : __outb(0x070 +1, 0x0a));
1042
1043 /*
1044 * "and the warm reset vector (DWORD based at 40:67) to point
1045 * to the AP startup code ..."
1046 */
1047
1048 dwordptr[0] = 0;
1049 dwordptr[1] = MP_TRAMPOLINE(16 * (1 << 12)) >> 4;
1050
1051 pmap_kenter_pa(0, 0, PROT_READ0x01 | PROT_WRITE0x02);
1052 memcpy((u_int8_t *) 0x467, dwordptr, 4)__builtin_memcpy(((u_int8_t *) 0x467), (dwordptr), (4));
1053 pmap_kremove(0, PAGE_SIZE(1 << 12));
1054
1055#if NLAPIC1 > 0
1056 /*
1057 * ... prior to executing the following sequence:"
1058 */
1059
1060 if (ci->ci_flags & CPUF_AP0x0002) {
1061 x86_ipi_init(ci->ci_apicid);
1062
1063 delay(10000)(*delay_func)(10000);
1064
1065 if (cpu_feature & CPUID_APIC0x00000200) {
1066 x86_ipi(MP_TRAMPOLINE(16 * (1 << 12))/PAGE_SIZE(1 << 12), ci->ci_apicid,
1067 LAPIC_DLMODE_STARTUP0x00000600);
1068 delay(200)(*delay_func)(200);
1069
1070 x86_ipi(MP_TRAMPOLINE(16 * (1 << 12))/PAGE_SIZE(1 << 12), ci->ci_apicid,
1071 LAPIC_DLMODE_STARTUP0x00000600);
1072 delay(200)(*delay_func)(200);
1073 }
1074 }
1075#endif
1076 return 0;
1077}
1078
1079void
1080mp_cpu_start_cleanup(struct cpu_info *ci)
1081{
1082 /*
1083 * Ensure the NVRAM reset byte contains something vaguely sane.
1084 */
1085
1086 outb(IO_RTC, NVRAM_RESET)( (__builtin_constant_p((0x070)) && (0x070) < 0x100
) ? __outbc(0x070, (0xe + 1)) : __outb(0x070, (0xe + 1)));
1087 outb(IO_RTC+1, NVRAM_RESET_RST)( (__builtin_constant_p((0x070 +1)) && (0x070 +1) <
0x100) ? __outbc(0x070 +1, 0x00) : __outb(0x070 +1, 0x00));
1088}
1089#endif /* MULTIPROCESSOR */
1090
1091typedef void (vector)(void);
1092extern vector Xsyscall_meltdown, Xsyscall, Xsyscall32;
1093
1094void
1095cpu_init_msrs(struct cpu_info *ci)
1096{
1097 uint64_t msr;
1098 int family;
1099
1100 wrmsr(MSR_STAR0xc0000081,
1101 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL)(((1) << 3) | 0) << 32) |
1102 ((uint64_t)GSEL(GUCODE32_SEL, SEL_UPL)(((3) << 3) | 3) << 48));
1103 wrmsr(MSR_LSTAR0xc0000082, cpu_meltdown ? (uint64_t)Xsyscall_meltdown :
1104 (uint64_t)Xsyscall);
1105 wrmsr(MSR_CSTAR0xc0000083, (uint64_t)Xsyscall32);
1106 wrmsr(MSR_SFMASK0xc0000084, PSL_NT0x00004000|PSL_T0x00000100|PSL_I0x00000200|PSL_C0x00000001|PSL_D0x00000400|PSL_AC0x00040000);
1107
1108 wrmsr(MSR_FSBASE0xc0000100, 0);
1109 wrmsr(MSR_GSBASE0xc0000101, (u_int64_t)ci);
1110 wrmsr(MSR_KERNELGSBASE0xc0000102, 0);
1111
1112 family = ci->ci_family;
1113 if (strcmp(cpu_vendor, "GenuineIntel") == 0 &&
1114 (family > 6 || (family == 6 && ci->ci_model >= 0xd)) &&
1115 rdmsr_safe(MSR_MISC_ENABLE0x1a0, &msr) == 0 &&
1116 (msr & MISC_ENABLE_FAST_STRINGS(1 << 0)) == 0) {
1117 msr |= MISC_ENABLE_FAST_STRINGS(1 << 0);
1118 wrmsr(MSR_MISC_ENABLE0x1a0, msr);
1119 DPRINTF("%s: enabled fast strings\n", ci->ci_dev->dv_xname);
1120 }
1121
1122 patinit(ci);
1123}
1124
1125void
1126cpu_tsx_disable(struct cpu_info *ci)
1127{
1128 uint64_t msr;
1129 uint32_t dummy, sefflags_edx;
1130
1131 /* this runs before identifycpu() populates ci_feature_sefflags_edx */
1132 if (cpuid_level < 0x07)
1133 return;
1134 CPUID_LEAF(0x7, 0, dummy, dummy, dummy, sefflags_edx)__asm volatile("cpuid" : "=a" (dummy), "=b" (dummy), "=c" (dummy
), "=d" (sefflags_edx) : "a" (0x7), "c" (0));
1135
1136 if (strcmp(cpu_vendor, "GenuineIntel") == 0 &&
1137 (sefflags_edx & SEFF0EDX_ARCH_CAP0x20000000)) {
1138 msr = rdmsr(MSR_ARCH_CAPABILITIES0x10a);
1139 if (msr & ARCH_CAPABILITIES_TSX_CTRL(1 << 7)) {
1140 msr = rdmsr(MSR_TSX_CTRL0x122);
1141 msr |= TSX_CTRL_RTM_DISABLE(1ULL << 0) | TSX_CTRL_TSX_CPUID_CLEAR(1ULL << 1);
1142 wrmsr(MSR_TSX_CTRL0x122, msr);
1143 }
1144 }
1145}
1146
1147void
1148patinit(struct cpu_info *ci)
1149{
1150 extern int pmap_pg_wc;
1151 u_int64_t reg;
1152
1153 if ((cpu_feature & CPUID_PAT0x00010000) == 0)
1154 return;
1155 /*
1156 * Set up PAT bits.
1157 * The default pat table is the following:
1158 * WB, WT, UC-, UC, WB, WT, UC-, UC
1159 * We change it to:
1160 * WB, WC, UC-, UC, WB, WC, UC-, UC
1161 * i.e change the WT bit to be WC.
1162 */
1163 reg = PATENTRY(0, PAT_WB)(0x6UL << ((0) * 8)) | PATENTRY(1, PAT_WC)(0x1UL << ((1) * 8)) |
1164 PATENTRY(2, PAT_UCMINUS)(0x7UL << ((2) * 8)) | PATENTRY(3, PAT_UC)(0x0UL << ((3) * 8)) |
1165 PATENTRY(4, PAT_WB)(0x6UL << ((4) * 8)) | PATENTRY(5, PAT_WC)(0x1UL << ((5) * 8)) |
1166 PATENTRY(6, PAT_UCMINUS)(0x7UL << ((6) * 8)) | PATENTRY(7, PAT_UC)(0x0UL << ((7) * 8));
1167
1168 wrmsr(MSR_CR_PAT0x277, reg);
1169 pmap_pg_wc = PG_WC(0x0000000000000008UL);
1170}
1171
1172struct timeout rdrand_tmo;
1173void rdrand(void *);
1174
1175void
1176rdrand(void *v)
1177{
1178 struct timeout *tmo = v;
1179 extern int has_rdrand, has_rdseed;
1180 union {
1181 uint64_t u64;
1182 uint32_t u32[2];
1183 } r, t;
1184 uint64_t tsc;
1185 uint8_t valid = 0;
1186
1187 tsc = rdtsc();
1188 if (has_rdseed)
5
←
Assuming 'has_rdseed' is 0
→
6
←
Taking false branch
→
1189 __asm volatile(
1190 "rdseed %0\n\t"
1191 "setc %1\n"
1192 : "=r" (r.u64), "=qm" (valid) );
1193 if (has_rdrand && (has_rdseed == 0 || valid == 0))
7
←
Assuming 'has_rdrand' is 0
→
1194 __asm volatile(
1195 "rdrand %0\n\t"
1196 "setc %1\n"
1197 : "=r" (r.u64), "=qm" (valid) );
1198
1199 t.u64 = tsc;
1200 t.u64 ^= r.u64;
8
←
Assigned value is garbage or undefined
1201 t.u64 ^= valid; /* potential rdrand empty */
1202 if (has_rdrand)
1203 t.u64 += rdtsc(); /* potential vmexit latency */
1204
1205 enqueue_randomness(t.u32[0]);
1206 enqueue_randomness(t.u32[1]);
1207
1208 if (tmo)
1209 timeout_add_msec(tmo, 10);
1210}
1211
1212int
1213cpu_activate(struct device *self, int act)
1214{
1215 struct cpu_softc *sc = (struct cpu_softc *)self;
1216
1217 switch (act) {
1
Control jumps to 'case 4:' at line 1218
→
1218 case DVACT_RESUME4:
1219 if (sc->sc_info->ci_cpuid == 0)
2
←
Assuming field 'ci_cpuid' is equal to 0
→
3
←
Taking true branch
→
1220 rdrand(NULL((void *)0));
4
←
Calling 'rdrand'
→
1221#if NPCTR1 > 0
1222 pctr_resume(sc->sc_info);
1223#endif
1224 break;
1225 }
1226
1227 return (0);
1228}
1229
1230/*
1231 * cpu_enter_pages
1232 *
1233 * Requests mapping of various special pages required in the Intel Meltdown
1234 * case (to be entered into the U-K page table):
1235 *
1236 * 1 tss+gdt page for each CPU
1237 * 1 trampoline stack page for each CPU
1238 *
1239 * The cpu_info_full struct for each CPU straddles these pages. The offset into
1240 * 'cif' is calculated below, for each page. For more information, consult
1241 * the definition of struct cpu_info_full in cpu_full.h
1242 *
1243 * On CPUs unaffected by Meltdown, this function still configures 'cif' but
1244 * the calls to pmap_enter_special become no-ops.
1245 *
1246 * Parameters:
1247 * cif : the cpu_info_full structure describing a CPU whose pages are to be
1248 * entered into the special meltdown U-K page table.
1249 */
1250void
1251cpu_enter_pages(struct cpu_info_full *cif)
1252{
1253 vaddr_t va;
1254 paddr_t pa;
1255
1256 /* The TSS+GDT need to be readable */
1257 va = (vaddr_t)cif;
1258 pmap_extract(pmap_kernel()(&kernel_pmap_store), va, &pa);
1259 pmap_enter_special(va, pa, PROT_READ0x01);
1260 DPRINTF("%s: entered tss+gdt page at va 0x%llx pa 0x%llx\n", __func__,
1261 (uint64_t)va, (uint64_t)pa);
1262
1263 /* The trampoline stack page needs to be read/write */
1264 va = (vaddr_t)&cif->cif_tramp_stack;
1265 pmap_extract(pmap_kernel()(&kernel_pmap_store), va, &pa);
1266 pmap_enter_special(va, pa, PROT_READ0x01 | PROT_WRITE0x02);
1267 DPRINTF("%s: entered t.stack page at va 0x%llx pa 0x%llx\n", __func__,
1268 (uint64_t)va, (uint64_t)pa);
1269
1270 cif->cif_tsscif_RO.u_tssgdt.uu_tss.tss_rsp0 = va + sizeof(cif->cif_tramp_stack) - 16;
1271 DPRINTF("%s: cif_tss.tss_rsp0 = 0x%llx\n" ,__func__,
1272 (uint64_t)cif->cif_tss.tss_rsp0);
1273 cif->cif_cpu.ci_intr_rsp = cif->cif_tsscif_RO.u_tssgdt.uu_tss.tss_rsp0 -
1274 sizeof(struct iretq_frame);
1275
1276#define SETUP_IST_SPECIAL_STACK(ist, cif, member)do { (cif)->cif_RO.u_tssgdt.uu_tss.tss_ist[(ist)] = (vaddr_t
)&(cif)->member + sizeof((cif)->member) - 16; (cif)
->member[(sizeof(((cif)->member)) / sizeof(((cif)->member
)[0])) - 2] = (int64_t)&(cif)->cif_cpu; } while (0) do { \
1277 (cif)->cif_tsscif_RO.u_tssgdt.uu_tss.tss_ist[(ist)] = (vaddr_t)&(cif)->member + \
1278 sizeof((cif)->member) - 16; \
1279 (cif)->member[nitems((cif)->member)(sizeof(((cif)->member)) / sizeof(((cif)->member)[0])) - 2] = (int64_t)&(cif)->cif_cpu; \
1280} while (0)
1281
1282 SETUP_IST_SPECIAL_STACK(0, cif, cif_dblflt_stack)do { (cif)->cif_RO.u_tssgdt.uu_tss.tss_ist[(0)] = (vaddr_t
)&(cif)->cif_dblflt_stack + sizeof((cif)->cif_dblflt_stack
) - 16; (cif)->cif_dblflt_stack[(sizeof(((cif)->cif_dblflt_stack
)) / sizeof(((cif)->cif_dblflt_stack)[0])) - 2] = (int64_t
)&(cif)->cif_cpu; } while (0);
1283 SETUP_IST_SPECIAL_STACK(1, cif, cif_nmi_stack)do { (cif)->cif_RO.u_tssgdt.uu_tss.tss_ist[(1)] = (vaddr_t
)&(cif)->cif_nmi_stack + sizeof((cif)->cif_nmi_stack
) - 16; (cif)->cif_nmi_stack[(sizeof(((cif)->cif_nmi_stack
)) / sizeof(((cif)->cif_nmi_stack)[0])) - 2] = (int64_t)&
(cif)->cif_cpu; } while (0);
1284
1285 /* an empty iomap, by setting its offset to the TSS limit */
1286 cif->cif_tsscif_RO.u_tssgdt.uu_tss.tss_iobase = sizeof(cif->cif_tsscif_RO.u_tssgdt.uu_tss);
1287}
1288
1289#ifdef MULTIPROCESSOR1
1290int
1291wbinvd_on_all_cpus(void)
1292{
1293 x86_broadcast_ipi(X86_IPI_WBINVD0x00000400);
1294 wbinvd();
1295 return 0;
1296}
1297#endif