Bug Summary

File:dev/pci/drm/amd/amdgpu/amdgpu_device.c
Warning:line 4913, column 27
The left operand of '==' is a garbage value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name amdgpu_device.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c

/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c

→
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33
34#include <drm/drm_atomic_helper.h>
35#include <drm/drm_probe_helper.h>
36#include <drm/amdgpu_drm.h>
37#include <linux/vgaarb.h>
38#include <linux/vga_switcheroo.h>
39#include <linux/efi.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_i2c.h"
43#include "atom.h"
44#include "amdgpu_atombios.h"
45#include "amdgpu_atomfirmware.h"
46#include "amd_pcie.h"
47#ifdef CONFIG_DRM_AMDGPU_SI
48#include "si.h"
49#endif
50#ifdef CONFIG_DRM_AMDGPU_CIK
51#include "cik.h"
52#endif
53#include "vi.h"
54#include "soc15.h"
55#include "nv.h"
56#include "bif/bif_4_1_d.h"
57#include <linux/pci.h>
58#include <linux/firmware.h>
59#include "amdgpu_vf_error.h"
60
61#include "amdgpu_amdkfd.h"
62#include "amdgpu_pm.h"
63
64#include "amdgpu_xgmi.h"
65#include "amdgpu_ras.h"
66#include "amdgpu_pmu.h"
67#include "amdgpu_fru_eeprom.h"
68
69#include <linux/suspend.h>
70#include <drm/task_barrier.h>
71#include <linux/pm_runtime.h>
72
73MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
74MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
75MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
76MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
77MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
78MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
79MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
80MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
81MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
82MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
83
84#define AMDGPU_RESUME_MS2000 2000
85
86const char *amdgpu_asic_name[] = {
87 "TAHITI",
88 "PITCAIRN",
89 "VERDE",
90 "OLAND",
91 "HAINAN",
92 "BONAIRE",
93 "KAVERI",
94 "KABINI",
95 "HAWAII",
96 "MULLINS",
97 "TOPAZ",
98 "TONGA",
99 "FIJI",
100 "CARRIZO",
101 "STONEY",
102 "POLARIS10",
103 "POLARIS11",
104 "POLARIS12",
105 "VEGAM",
106 "VEGA10",
107 "VEGA12",
108 "VEGA20",
109 "RAVEN",
110 "ARCTURUS",
111 "RENOIR",
112 "NAVI10",
113 "NAVI14",
114 "NAVI12",
115 "SIENNA_CICHLID",
116 "NAVY_FLOUNDER",
117 "LAST",
118};
119
120/**
121 * DOC: pcie_replay_count
122 *
123 * The amdgpu driver provides a sysfs API for reporting the total number
124 * of PCIe replays (NAKs)
125 * The file pcie_replay_count is used for this and returns the total
126 * number of replays as a sum of the NAKs generated and NAKs received
127 */
128
129static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
130 struct device_attribute *attr, char *buf)
131{
132 struct drm_device *ddev = dev_get_drvdata(dev)((void *)0);
133 struct amdgpu_device *adev = drm_to_adev(ddev);
134 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev)((adev)->asic_funcs->get_pcie_replay_count((adev)));
135
136 return snprintf(buf, PAGE_SIZE(1 << 12), "%llu\n", cnt);
137}
138
139static DEVICE_ATTR(pcie_replay_count, S_IRUGO,struct device_attribute dev_attr_pcie_replay_count
140 amdgpu_device_get_pcie_replay_count, NULL)struct device_attribute dev_attr_pcie_replay_count;
141
142static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
143
144/**
145 * DOC: product_name
146 *
147 * The amdgpu driver provides a sysfs API for reporting the product name
148 * for the device
149 * The file serial_number is used for this and returns the product name
150 * as returned from the FRU.
151 * NOTE: This is only available for certain server cards
152 */
153
154static ssize_t amdgpu_device_get_product_name(struct device *dev,
155 struct device_attribute *attr, char *buf)
156{
157 struct drm_device *ddev = dev_get_drvdata(dev)((void *)0);
158 struct amdgpu_device *adev = drm_to_adev(ddev);
159
160 return snprintf(buf, PAGE_SIZE(1 << 12), "%s\n", adev->product_name);
161}
162
163static DEVICE_ATTR(product_name, S_IRUGO,struct device_attribute dev_attr_product_name
164 amdgpu_device_get_product_name, NULL)struct device_attribute dev_attr_product_name;
165
166/**
167 * DOC: product_number
168 *
169 * The amdgpu driver provides a sysfs API for reporting the part number
170 * for the device
171 * The file serial_number is used for this and returns the part number
172 * as returned from the FRU.
173 * NOTE: This is only available for certain server cards
174 */
175
176static ssize_t amdgpu_device_get_product_number(struct device *dev,
177 struct device_attribute *attr, char *buf)
178{
179 struct drm_device *ddev = dev_get_drvdata(dev)((void *)0);
180 struct amdgpu_device *adev = drm_to_adev(ddev);
181
182 return snprintf(buf, PAGE_SIZE(1 << 12), "%s\n", adev->product_number);
183}
184
185static DEVICE_ATTR(product_number, S_IRUGO,struct device_attribute dev_attr_product_number
186 amdgpu_device_get_product_number, NULL)struct device_attribute dev_attr_product_number;
187
188/**
189 * DOC: serial_number
190 *
191 * The amdgpu driver provides a sysfs API for reporting the serial number
192 * for the device
193 * The file serial_number is used for this and returns the serial number
194 * as returned from the FRU.
195 * NOTE: This is only available for certain server cards
196 */
197
198static ssize_t amdgpu_device_get_serial_number(struct device *dev,
199 struct device_attribute *attr, char *buf)
200{
201 struct drm_device *ddev = dev_get_drvdata(dev)((void *)0);
202 struct amdgpu_device *adev = drm_to_adev(ddev);
203
204 return snprintf(buf, PAGE_SIZE(1 << 12), "%s\n", adev->serial);
205}
206
207static DEVICE_ATTR(serial_number, S_IRUGO,struct device_attribute dev_attr_serial_number
208 amdgpu_device_get_serial_number, NULL)struct device_attribute dev_attr_serial_number;
209
210/**
211 * amdgpu_device_supports_boco - Is the device a dGPU with HG/PX power control
212 *
213 * @dev: drm_device pointer
214 *
215 * Returns true if the device is a dGPU with HG/PX power control,
216 * otherwise return false.
217 */
218bool_Bool amdgpu_device_supports_boco(struct drm_device *dev)
219{
220 struct amdgpu_device *adev = drm_to_adev(dev);
221
222 if (adev->flags & AMD_IS_PX)
223 return true1;
224 return false0;
225}
226
227/**
228 * amdgpu_device_supports_baco - Does the device support BACO
229 *
230 * @dev: drm_device pointer
231 *
232 * Returns true if the device supporte BACO,
233 * otherwise return false.
234 */
235bool_Bool amdgpu_device_supports_baco(struct drm_device *dev)
236{
237 struct amdgpu_device *adev = drm_to_adev(dev);
238
239 return amdgpu_asic_supports_baco(adev)(adev)->asic_funcs->supports_baco((adev));
240}
241
242/*
243 * VRAM access helper functions
244 */
245
246/**
247 * amdgpu_device_vram_access - read/write a buffer in vram
248 *
249 * @adev: amdgpu_device pointer
250 * @pos: offset of the buffer in vram
251 * @buf: virtual address of the buffer in system memory
252 * @size: read/write size, sizeof(@buf) must > @size
253 * @write: true - write to vram, otherwise - read from vram
254 */
255void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
256 uint32_t *buf, size_t size, bool_Bool write)
257{
258 unsigned long flags;
259 uint32_t hi = ~0;
260 uint64_t last;
261
262
263#ifdef CONFIG_64BIT1
264 last = min(pos + size, adev->gmc.visible_vram_size)(((pos + size)<(adev->gmc.visible_vram_size))?(pos + size
):(adev->gmc.visible_vram_size));
265 if (last > pos) {
266 void __iomem *addr = adev->mman.aper_base_kaddr + pos;
267 size_t count = last - pos;
268
269 if (write) {
270 memcpy_toio(addr, buf, count)__builtin_memcpy((addr), (buf), (count));
271 mb()do { __asm volatile("mfence" ::: "memory"); } while (0);
272 amdgpu_asic_flush_hdp(adev, NULL)(adev)->asic_funcs->flush_hdp((adev), (((void *)0)));
273 } else {
274 amdgpu_asic_invalidate_hdp(adev, NULL)(adev)->asic_funcs->invalidate_hdp((adev), (((void *)0)
));
275 mb()do { __asm volatile("mfence" ::: "memory"); } while (0);
276 memcpy_fromio(buf, addr, count)__builtin_memcpy((buf), (addr), (count));
277 }
278
279 if (count == size)
280 return;
281
282 pos += count;
283 buf += count / 4;
284 size -= count;
285 }
286#endif
287
288 spin_lock_irqsave(&adev->mmio_idx_lock, flags)do { flags = 0; mtx_enter(&adev->mmio_idx_lock); } while
(0);
289 for (last = pos + size; pos < last; pos += 4) {
290 uint32_t tmp = pos >> 31;
291
292 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000)amdgpu_device_wreg(adev, (0x0), (((uint32_t)pos) | 0x80000000
), (1<<1));
293 if (tmp != hi) {
294 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp)amdgpu_device_wreg(adev, (0x6), (tmp), (1<<1));
295 hi = tmp;
296 }
297 if (write)
298 WREG32_NO_KIQ(mmMM_DATA, *buf++)amdgpu_device_wreg(adev, (0x1), (*buf++), (1<<1));
299 else
300 *buf++ = RREG32_NO_KIQ(mmMM_DATA)amdgpu_device_rreg(adev, (0x1), (1<<1));
301 }
302 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags)do { (void)(flags); mtx_leave(&adev->mmio_idx_lock); }
while (0);
303}
304
305/*
306 * register access helper functions.
307 */
308/**
309 * amdgpu_device_rreg - read a memory mapped IO or indirect register
310 *
311 * @adev: amdgpu_device pointer
312 * @reg: dword aligned register offset
313 * @acc_flags: access flags which require special behavior
314 *
315 * Returns the 32 bit value from the offset specified.
316 */
317uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
318 uint32_t reg, uint32_t acc_flags)
319{
320 uint32_t ret;
321
322 if (adev->in_pci_err_recovery)
323 return 0;
324
325 if ((reg * 4) < adev->rmmio_size) {
326 if (!(acc_flags & AMDGPU_REGS_NO_KIQ(1<<1)) &&
327 amdgpu_sriov_runtime(adev)((adev)->virt.caps & (1 << 4)) &&
328 down_read_trylock(&adev->reset_sem)(rw_enter(&adev->reset_sem, 0x0002UL | 0x0040UL) == 0)) {
329 ret = amdgpu_kiq_rreg(adev, reg);
330 up_read(&adev->reset_sem)rw_exit_read(&adev->reset_sem);
331 } else {
332 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4))ioread32(((void *)adev->rmmio) + (reg * 4));
333 }
334 } else {
335 ret = adev->pcie_rreg(adev, reg * 4);
336 }
337
338 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
339
340 return ret;
341}
342
343/*
344 * MMIO register read with bytes helper functions
345 * @offset:bytes offset from MMIO start
346 *
347*/
348
349/**
350 * amdgpu_mm_rreg8 - read a memory mapped IO register
351 *
352 * @adev: amdgpu_device pointer
353 * @offset: byte aligned register offset
354 *
355 * Returns the 8 bit value from the offset specified.
356 */
357uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
358{
359 if (adev->in_pci_err_recovery)
360 return 0;
361
362 if (offset < adev->rmmio_size)
363 return (readb(adev->rmmio + offset)ioread8(adev->rmmio + offset));
364 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 364); } while (0);
365}
366
367/*
368 * MMIO register write with bytes helper functions
369 * @offset:bytes offset from MMIO start
370 * @value: the value want to be written to the register
371 *
372*/
373/**
374 * amdgpu_mm_wreg8 - read a memory mapped IO register
375 *
376 * @adev: amdgpu_device pointer
377 * @offset: byte aligned register offset
378 * @value: 8 bit value to write
379 *
380 * Writes the value specified to the offset specified.
381 */
382void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
383{
384 if (adev->in_pci_err_recovery)
385 return;
386
387 if (offset < adev->rmmio_size)
388 writeb(value, adev->rmmio + offset)iowrite8(value, adev->rmmio + offset);
389 else
390 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 390); } while (0);
391}
392
393/**
394 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
395 *
396 * @adev: amdgpu_device pointer
397 * @reg: dword aligned register offset
398 * @v: 32 bit value to write to the register
399 * @acc_flags: access flags which require special behavior
400 *
401 * Writes the value specified to the offset specified.
402 */
403void amdgpu_device_wreg(struct amdgpu_device *adev,
404 uint32_t reg, uint32_t v,
405 uint32_t acc_flags)
406{
407 if (adev->in_pci_err_recovery)
408 return;
409
410 if ((reg * 4) < adev->rmmio_size) {
411 if (!(acc_flags & AMDGPU_REGS_NO_KIQ(1<<1)) &&
412 amdgpu_sriov_runtime(adev)((adev)->virt.caps & (1 << 4)) &&
413 down_read_trylock(&adev->reset_sem)(rw_enter(&adev->reset_sem, 0x0002UL | 0x0040UL) == 0)) {
414 amdgpu_kiq_wreg(adev, reg, v);
415 up_read(&adev->reset_sem)rw_exit_read(&adev->reset_sem);
416 } else {
417 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4))iowrite32(v, ((void *)adev->rmmio) + (reg * 4));
418 }
419 } else {
420 adev->pcie_wreg(adev, reg * 4, v);
421 }
422
423 trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
424}
425
426/*
427 * amdgpu_mm_wreg_mmio_rlc - write register either with mmio or with RLC path if in range
428 *
429 * this function is invoked only the debugfs register access
430 * */
431void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
432 uint32_t reg, uint32_t v)
433{
434 if (adev->in_pci_err_recovery)
435 return;
436
437 if (amdgpu_sriov_fullaccess(adev)((((adev))->virt.caps & (1 << 2)) && !((
(adev))->virt.caps & (1 << 4))) &&
438 adev->gfx.rlc.funcs &&
439 adev->gfx.rlc.funcs->is_rlcg_access_range) {
440 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
441 return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
442 } else {
443 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4))iowrite32(v, ((void *)adev->rmmio) + (reg * 4));
444 }
445}
446
447/**
448 * amdgpu_io_rreg - read an IO register
449 *
450 * @adev: amdgpu_device pointer
451 * @reg: dword aligned register offset
452 *
453 * Returns the 32 bit value from the offset specified.
454 */
455u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
456{
457 u32 val;
458
459 if (adev->in_pci_err_recovery)
460 return 0;
461
462 if ((reg * 4) < adev->rio_mem_size) {
463 val = bus_space_read_4(adev->rio_mem_bst, adev->rio_mem_bsh, reg)((adev->rio_mem_bst)->read_4((adev->rio_mem_bsh), (reg
)));
464 bus_space_barrier(adev->rio_mem_bst, adev->rio_mem_bsh, 0,
465 adev->rio_mem_size, BUS_SPACE_BARRIER_READ0x01);
466 } else {
467 bus_space_barrier(adev->rio_mem_bst, adev->rio_mem_bsh, 0,
468 adev->rio_mem_size, BUS_SPACE_BARRIER_WRITE0x02);
469 bus_space_write_4(adev->rio_mem_bst, adev->rio_mem_bsh,((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
0x0 * 4), (reg * 4)))
470 mmMM_INDEX * 4, reg * 4)((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
0x0 * 4), (reg * 4)));
471 val = bus_space_read_4(adev->rio_mem_bst, adev->rio_mem_bsh,((adev->rio_mem_bst)->read_4((adev->rio_mem_bsh), (0x0
* 4)))
472 mmMM_INDEX * 4)((adev->rio_mem_bst)->read_4((adev->rio_mem_bsh), (0x0
* 4)));
473 bus_space_barrier(adev->rio_mem_bst, adev->rio_mem_bsh, 0,
474 adev->rio_mem_size, BUS_SPACE_BARRIER_READ0x01);
475 }
476
477 return val;
478}
479
480/**
481 * amdgpu_io_wreg - write to an IO register
482 *
483 * @adev: amdgpu_device pointer
484 * @reg: dword aligned register offset
485 * @v: 32 bit value to write to the register
486 *
487 * Writes the value specified to the offset specified.
488 */
489void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
490{
491 if (adev->in_pci_err_recovery)
492 return;
493
494 if ((reg * 4) < adev->rio_mem_size) {
495 bus_space_barrier(adev->rio_mem_bst, adev->rio_mem_bsh, 0,
496 adev->rio_mem_size, BUS_SPACE_BARRIER_WRITE0x02);
497 bus_space_write_4(adev->rio_mem_bst, adev->rio_mem_bsh,((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
reg * 4), (v)))
498 reg * 4, v)((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
reg * 4), (v)));
499 } else {
500 bus_space_barrier(adev->rio_mem_bst, adev->rio_mem_bsh, 0,
501 adev->rio_mem_size, BUS_SPACE_BARRIER_WRITE0x02);
502 bus_space_write_4(adev->rio_mem_bst, adev->rio_mem_bsh,((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
0x0 * 4), (reg * 4)))
503 mmMM_INDEX * 4, reg * 4)((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
0x0 * 4), (reg * 4)));
504 bus_space_barrier(adev->rio_mem_bst, adev->rio_mem_bsh, 0,
505 adev->rio_mem_size, BUS_SPACE_BARRIER_WRITE0x02);
506 bus_space_write_4(adev->rio_mem_bst, adev->rio_mem_bsh,((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
0x1 * 4), (v)))
507 mmMM_DATA * 4, v)((adev->rio_mem_bst)->write_4((adev->rio_mem_bsh), (
0x1 * 4), (v)));
508
509 }
510}
511
512/**
513 * amdgpu_mm_rdoorbell - read a doorbell dword
514 *
515 * @adev: amdgpu_device pointer
516 * @index: doorbell index
517 *
518 * Returns the value in the doorbell aperture at the
519 * requested doorbell index (CIK).
520 */
521u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
522{
523 if (adev->in_pci_err_recovery)
524 return 0;
525
526 if (index < adev->doorbell.num_doorbells) {
527 return readl(adev->doorbell.ptr + index)ioread32(adev->doorbell.ptr + index);
528 } else {
529 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index)__drm_err("reading beyond doorbell aperture: 0x%08x!\n", index
);
530 return 0;
531 }
532}
533
534/**
535 * amdgpu_mm_wdoorbell - write a doorbell dword
536 *
537 * @adev: amdgpu_device pointer
538 * @index: doorbell index
539 * @v: value to write
540 *
541 * Writes @v to the doorbell aperture at the
542 * requested doorbell index (CIK).
543 */
544void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
545{
546 if (adev->in_pci_err_recovery)
547 return;
548
549 if (index < adev->doorbell.num_doorbells) {
550 writel(v, adev->doorbell.ptr + index)iowrite32(v, adev->doorbell.ptr + index);
551 } else {
552 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index)__drm_err("writing beyond doorbell aperture: 0x%08x!\n", index
);
553 }
554}
555
556/**
557 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
558 *
559 * @adev: amdgpu_device pointer
560 * @index: doorbell index
561 *
562 * Returns the value in the doorbell aperture at the
563 * requested doorbell index (VEGA10+).
564 */
565u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
566{
567 if (adev->in_pci_err_recovery)
568 return 0;
569
570 if (index < adev->doorbell.num_doorbells) {
571 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index))({ typeof(*((atomic64_t *)(adev->doorbell.ptr + index))) __tmp
= *(volatile typeof(*((atomic64_t *)(adev->doorbell.ptr +
index))) *)&(*((atomic64_t *)(adev->doorbell.ptr + index
))); membar_datadep_consumer(); __tmp; });
572 } else {
573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index)__drm_err("reading beyond doorbell aperture: 0x%08x!\n", index
);
574 return 0;
575 }
576}
577
578/**
579 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
580 *
581 * @adev: amdgpu_device pointer
582 * @index: doorbell index
583 * @v: value to write
584 *
585 * Writes @v to the doorbell aperture at the
586 * requested doorbell index (VEGA10+).
587 */
588void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
589{
590 if (adev->in_pci_err_recovery)
591 return;
592
593 if (index < adev->doorbell.num_doorbells) {
594 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v)({ typeof(*((atomic64_t *)(adev->doorbell.ptr + index))) __tmp
= ((v)); *(volatile typeof(*((atomic64_t *)(adev->doorbell
.ptr + index))) *)&(*((atomic64_t *)(adev->doorbell.ptr
+ index))) = __tmp; __tmp; });
595 } else {
596 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index)__drm_err("writing beyond doorbell aperture: 0x%08x!\n", index
);
597 }
598}
599
600/**
601 * amdgpu_device_indirect_rreg - read an indirect register
602 *
603 * @adev: amdgpu_device pointer
604 * @pcie_index: mmio register offset
605 * @pcie_data: mmio register offset
606 *
607 * Returns the value of indirect register @reg_addr
608 */
609u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
610 u32 pcie_index, u32 pcie_data,
611 u32 reg_addr)
612{
613 unsigned long flags;
614 u32 r;
615 void __iomem *pcie_index_offset;
616 void __iomem *pcie_data_offset;
617
618 spin_lock_irqsave(&adev->pcie_idx_lock, flags)do { flags = 0; mtx_enter(&adev->pcie_idx_lock); } while
(0);
619 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
620 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
621
622 writel(reg_addr, pcie_index_offset)iowrite32(reg_addr, pcie_index_offset);
623 readl(pcie_index_offset)ioread32(pcie_index_offset);
624 r = readl(pcie_data_offset)ioread32(pcie_data_offset);
625 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags)do { (void)(flags); mtx_leave(&adev->pcie_idx_lock); }
while (0);
626
627 return r;
628}
629
630/**
631 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
632 *
633 * @adev: amdgpu_device pointer
634 * @pcie_index: mmio register offset
635 * @pcie_data: mmio register offset
636 *
637 * Returns the value of indirect register @reg_addr
638 */
639u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
640 u32 pcie_index, u32 pcie_data,
641 u32 reg_addr)
642{
643 unsigned long flags;
644 u64 r;
645 void __iomem *pcie_index_offset;
646 void __iomem *pcie_data_offset;
647
648 spin_lock_irqsave(&adev->pcie_idx_lock, flags)do { flags = 0; mtx_enter(&adev->pcie_idx_lock); } while
(0);
649 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
650 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
651
652 /* read low 32 bits */
653 writel(reg_addr, pcie_index_offset)iowrite32(reg_addr, pcie_index_offset);
654 readl(pcie_index_offset)ioread32(pcie_index_offset);
655 r = readl(pcie_data_offset)ioread32(pcie_data_offset);
656 /* read high 32 bits */
657 writel(reg_addr + 4, pcie_index_offset)iowrite32(reg_addr + 4, pcie_index_offset);
658 readl(pcie_index_offset)ioread32(pcie_index_offset);
659 r |= ((u64)readl(pcie_data_offset)ioread32(pcie_data_offset) << 32);
660 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags)do { (void)(flags); mtx_leave(&adev->pcie_idx_lock); }
while (0);
661
662 return r;
663}
664
665/**
666 * amdgpu_device_indirect_wreg - write an indirect register address
667 *
668 * @adev: amdgpu_device pointer
669 * @pcie_index: mmio register offset
670 * @pcie_data: mmio register offset
671 * @reg_addr: indirect register offset
672 * @reg_data: indirect register data
673 *
674 */
675void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
676 u32 pcie_index, u32 pcie_data,
677 u32 reg_addr, u32 reg_data)
678{
679 unsigned long flags;
680 void __iomem *pcie_index_offset;
681 void __iomem *pcie_data_offset;
682
683 spin_lock_irqsave(&adev->pcie_idx_lock, flags)do { flags = 0; mtx_enter(&adev->pcie_idx_lock); } while
(0);
684 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
685 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
686
687 writel(reg_addr, pcie_index_offset)iowrite32(reg_addr, pcie_index_offset);
688 readl(pcie_index_offset)ioread32(pcie_index_offset);
689 writel(reg_data, pcie_data_offset)iowrite32(reg_data, pcie_data_offset);
690 readl(pcie_data_offset)ioread32(pcie_data_offset);
691 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags)do { (void)(flags); mtx_leave(&adev->pcie_idx_lock); }
while (0);
692}
693
694/**
695 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
696 *
697 * @adev: amdgpu_device pointer
698 * @pcie_index: mmio register offset
699 * @pcie_data: mmio register offset
700 * @reg_addr: indirect register offset
701 * @reg_data: indirect register data
702 *
703 */
704void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
705 u32 pcie_index, u32 pcie_data,
706 u32 reg_addr, u64 reg_data)
707{
708 unsigned long flags;
709 void __iomem *pcie_index_offset;
710 void __iomem *pcie_data_offset;
711
712 spin_lock_irqsave(&adev->pcie_idx_lock, flags)do { flags = 0; mtx_enter(&adev->pcie_idx_lock); } while
(0);
713 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
714 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
715
716 /* write low 32 bits */
717 writel(reg_addr, pcie_index_offset)iowrite32(reg_addr, pcie_index_offset);
718 readl(pcie_index_offset)ioread32(pcie_index_offset);
719 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset)iowrite32((u32)(reg_data & 0xffffffffULL), pcie_data_offset
);
720 readl(pcie_data_offset)ioread32(pcie_data_offset);
721 /* write high 32 bits */
722 writel(reg_addr + 4, pcie_index_offset)iowrite32(reg_addr + 4, pcie_index_offset);
723 readl(pcie_index_offset)ioread32(pcie_index_offset);
724 writel((u32)(reg_data >> 32), pcie_data_offset)iowrite32((u32)(reg_data >> 32), pcie_data_offset);
725 readl(pcie_data_offset)ioread32(pcie_data_offset);
726 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags)do { (void)(flags); mtx_leave(&adev->pcie_idx_lock); }
while (0);
727}
728
729/**
730 * amdgpu_invalid_rreg - dummy reg read function
731 *
732 * @adev: amdgpu_device pointer
733 * @reg: offset of register
734 *
735 * Dummy register read function. Used for register blocks
736 * that certain asics don't have (all asics).
737 * Returns the value in the register.
738 */
739static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
740{
741 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg)__drm_err("Invalid callback to read register 0x%04X\n", reg);
742 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 742); } while (0);
743 return 0;
744}
745
746/**
747 * amdgpu_invalid_wreg - dummy reg write function
748 *
749 * @adev: amdgpu_device pointer
750 * @reg: offset of register
751 * @v: value to write to the register
752 *
753 * Dummy register read function. Used for register blocks
754 * that certain asics don't have (all asics).
755 */
756static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
757{
758 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",__drm_err("Invalid callback to write register 0x%04X with 0x%08X\n"
, reg, v)
759 reg, v)__drm_err("Invalid callback to write register 0x%04X with 0x%08X\n"
, reg, v);
760 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 760); } while (0);
761}
762
763/**
764 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
765 *
766 * @adev: amdgpu_device pointer
767 * @reg: offset of register
768 *
769 * Dummy register read function. Used for register blocks
770 * that certain asics don't have (all asics).
771 * Returns the value in the register.
772 */
773static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
774{
775 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg)__drm_err("Invalid callback to read 64 bit register 0x%04X\n"
, reg);
776 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 776); } while (0);
777 return 0;
778}
779
780/**
781 * amdgpu_invalid_wreg64 - dummy reg write function
782 *
783 * @adev: amdgpu_device pointer
784 * @reg: offset of register
785 * @v: value to write to the register
786 *
787 * Dummy register read function. Used for register blocks
788 * that certain asics don't have (all asics).
789 */
790static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
791{
792 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",__drm_err("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n"
, reg, v)
793 reg, v)__drm_err("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n"
, reg, v);
794 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 794); } while (0);
795}
796
797/**
798 * amdgpu_block_invalid_rreg - dummy reg read function
799 *
800 * @adev: amdgpu_device pointer
801 * @block: offset of instance
802 * @reg: offset of register
803 *
804 * Dummy register read function. Used for register blocks
805 * that certain asics don't have (all asics).
806 * Returns the value in the register.
807 */
808static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
809 uint32_t block, uint32_t reg)
810{
811 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",__drm_err("Invalid callback to read register 0x%04X in block 0x%04X\n"
, reg, block)
812 reg, block)__drm_err("Invalid callback to read register 0x%04X in block 0x%04X\n"
, reg, block);
813 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 813); } while (0);
814 return 0;
815}
816
817/**
818 * amdgpu_block_invalid_wreg - dummy reg write function
819 *
820 * @adev: amdgpu_device pointer
821 * @block: offset of instance
822 * @reg: offset of register
823 * @v: value to write to the register
824 *
825 * Dummy register read function. Used for register blocks
826 * that certain asics don't have (all asics).
827 */
828static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
829 uint32_t block,
830 uint32_t reg, uint32_t v)
831{
832 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",__drm_err("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n"
, reg, block, v)
833 reg, block, v)__drm_err("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n"
, reg, block, v);
834 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 834); } while (0);
835}
836
837/**
838 * amdgpu_device_asic_init - Wrapper for atom asic_init
839 *
840 * @adev: amdgpu_device pointer
841 *
842 * Does any asic specific work and then calls atom asic init.
843 */
844static int amdgpu_device_asic_init(struct amdgpu_device *adev)
845{
846 amdgpu_asic_pre_asic_init(adev)(adev)->asic_funcs->pre_asic_init((adev));
847
848 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
849}
850
851/**
852 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
853 *
854 * @adev: amdgpu_device pointer
855 *
856 * Allocates a scratch page of VRAM for use by various things in the
857 * driver.
858 */
859static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
860{
861 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE4096,
862 PAGE_SIZE(1 << 12), AMDGPU_GEM_DOMAIN_VRAM0x4,
863 &adev->vram_scratch.robj,
864 &adev->vram_scratch.gpu_addr,
865 (void **)&adev->vram_scratch.ptr);
866}
867
868/**
869 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
870 *
871 * @adev: amdgpu_device pointer
872 *
873 * Frees the VRAM scratch page.
874 */
875static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
876{
877 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL((void *)0), NULL((void *)0));
878}
879
880/**
881 * amdgpu_device_program_register_sequence - program an array of registers.
882 *
883 * @adev: amdgpu_device pointer
884 * @registers: pointer to the register array
885 * @array_size: size of the register array
886 *
887 * Programs an array or registers with and and or masks.
888 * This is a helper for setting golden registers.
889 */
890void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
891 const u32 *registers,
892 const u32 array_size)
893{
894 u32 tmp, reg, and_mask, or_mask;
895 int i;
896
897 if (array_size % 3)
898 return;
899
900 for (i = 0; i < array_size; i +=3) {
901 reg = registers[i + 0];
902 and_mask = registers[i + 1];
903 or_mask = registers[i + 2];
904
905 if (and_mask == 0xffffffff) {
906 tmp = or_mask;
907 } else {
908 tmp = RREG32(reg)amdgpu_device_rreg(adev, (reg), 0);
909 tmp &= ~and_mask;
910 if (adev->family >= AMDGPU_FAMILY_AI141)
911 tmp |= (or_mask & and_mask);
912 else
913 tmp |= or_mask;
914 }
915 WREG32(reg, tmp)amdgpu_device_wreg(adev, (reg), (tmp), 0);
916 }
917}
918
919/**
920 * amdgpu_device_pci_config_reset - reset the GPU
921 *
922 * @adev: amdgpu_device pointer
923 *
924 * Resets the GPU using the pci config reset sequence.
925 * Only applicable to asics prior to vega10.
926 */
927void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
928{
929 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA0x39d5e86b);
930}
931
932/*
933 * GPU doorbell aperture helpers function.
934 */
935/**
936 * amdgpu_device_doorbell_init - Init doorbell driver information.
937 *
938 * @adev: amdgpu_device pointer
939 *
940 * Init doorbell driver information (CIK)
941 * Returns 0 on success, error on failure.
942 */
943static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
944{
945
946 /* No doorbell on SI hardware generation */
947 if (adev->asic_type < CHIP_BONAIRE) {
948 adev->doorbell.base = 0;
949 adev->doorbell.size = 0;
950 adev->doorbell.num_doorbells = 0;
951 adev->doorbell.ptr = NULL((void *)0);
952 return 0;
953 }
954
955#ifdef __linux
956 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
957 return -EINVAL22;
958#endif
959
960 amdgpu_asic_init_doorbell_index(adev)(adev)->asic_funcs->init_doorbell_index((adev));
961
962 /* doorbell bar mapping */
963#ifdef __linux__
964 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
965 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
966#endif
967
968 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),({ u32 __min_a = (adev->doorbell.size / sizeof(u32)); u32 __min_b
= (adev->doorbell_index.max_assignment+1); __min_a < __min_b
? __min_a : __min_b; })
969 adev->doorbell_index.max_assignment+1)({ u32 __min_a = (adev->doorbell.size / sizeof(u32)); u32 __min_b
= (adev->doorbell_index.max_assignment+1); __min_a < __min_b
? __min_a : __min_b; });
970 if (adev->doorbell.num_doorbells == 0)
971 return -EINVAL22;
972
973 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
974 * paging queue doorbell use the second page. The
975 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
976 * doorbells are in the first page. So with paging queue enabled,
977 * the max num_doorbells should + 1 page (0x400 in dword)
978 */
979 if (adev->asic_type >= CHIP_VEGA10)
980 adev->doorbell.num_doorbells += 0x400;
981
982#ifdef __linux__
983 adev->doorbell.ptr = ioremap(adev->doorbell.base,
984 adev->doorbell.num_doorbells *
985 sizeof(u32));
986 if (adev->doorbell.ptr == NULL((void *)0))
987 return -ENOMEM12;
988#endif
989
990 return 0;
991}
992
993/**
994 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
995 *
996 * @adev: amdgpu_device pointer
997 *
998 * Tear down doorbell driver information (CIK)
999 */
1000static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1001{
1002#ifdef __linux__
1003 iounmap(adev->doorbell.ptr);
1004#else
1005 if (adev->doorbell.size > 0)
1006 bus_space_unmap(adev->doorbell.bst, adev->doorbell.bsh,
1007 adev->doorbell.size);
1008#endif
1009 adev->doorbell.ptr = NULL((void *)0);
1010}
1011
1012
1013
1014/*
1015 * amdgpu_device_wb_*()
1016 * Writeback is the method by which the GPU updates special pages in memory
1017 * with the status of certain GPU events (fences, ring pointers,etc.).
1018 */
1019
1020/**
1021 * amdgpu_device_wb_fini - Disable Writeback and free memory
1022 *
1023 * @adev: amdgpu_device pointer
1024 *
1025 * Disables Writeback and frees the Writeback memory (all asics).
1026 * Used at driver shutdown.
1027 */
1028static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1029{
1030 if (adev->wb.wb_obj) {
1031 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1032 &adev->wb.gpu_addr,
1033 (void **)&adev->wb.wb);
1034 adev->wb.wb_obj = NULL((void *)0);
1035 }
1036}
1037
1038/**
1039 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1040 *
1041 * @adev: amdgpu_device pointer
1042 *
1043 * Initializes writeback and allocates writeback memory (all asics).
1044 * Used at driver startup.
1045 * Returns 0 on success or an -error on failure.
1046 */
1047static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1048{
1049 int r;
1050
1051 if (adev->wb.wb_obj == NULL((void *)0)) {
1052 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1053 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB256 * sizeof(uint32_t) * 8,
1054 PAGE_SIZE(1 << 12), AMDGPU_GEM_DOMAIN_GTT0x2,
1055 &adev->wb.wb_obj, &adev->wb.gpu_addr,
1056 (void **)&adev->wb.wb);
1057 if (r) {
1058 dev_warn(adev->dev, "(%d) create WB bo failed\n", r)printf("drm:pid%d:%s *WARNING* " "(%d) create WB bo failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
1059 return r;
1060 }
1061
1062 adev->wb.num_wb = AMDGPU_MAX_WB256;
1063 memset(&adev->wb.used, 0, sizeof(adev->wb.used))__builtin_memset((&adev->wb.used), (0), (sizeof(adev->
wb.used)));
1064
1065 /* clear wb memory */
1066 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8)__builtin_memset(((char *)adev->wb.wb), (0), (256 * sizeof
(uint32_t) * 8));
1067 }
1068
1069 return 0;
1070}
1071
1072/**
1073 * amdgpu_device_wb_get - Allocate a wb entry
1074 *
1075 * @adev: amdgpu_device pointer
1076 * @wb: wb index
1077 *
1078 * Allocate a wb slot for use by the driver (all asics).
1079 * Returns 0 on success or -EINVAL on failure.
1080 */
1081int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1082{
1083 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1084
1085 if (offset < adev->wb.num_wb) {
1086 __set_bit(offset, adev->wb.used);
1087 *wb = offset << 3; /* convert to dw offset */
1088 return 0;
1089 } else {
1090 return -EINVAL22;
1091 }
1092}
1093
1094/**
1095 * amdgpu_device_wb_free - Free a wb entry
1096 *
1097 * @adev: amdgpu_device pointer
1098 * @wb: wb index
1099 *
1100 * Free a wb slot allocated for use by the driver (all asics)
1101 */
1102void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1103{
1104 wb >>= 3;
1105 if (wb < adev->wb.num_wb)
1106 __clear_bit(wb, adev->wb.used);
1107}
1108
1109/**
1110 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1111 *
1112 * @adev: amdgpu_device pointer
1113 *
1114 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1115 * to fail, but if any of the BARs is not accessible after the size we abort
1116 * driver loading by returning -ENODEV.
1117 */
1118int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1119{
1120#ifdef __linux__
1121 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
1122 u32 rbar_size = order_base_2(((space_needed >> 20) | 1))drm_order(((space_needed >> 20) | 1)) - 1;
1123 struct pci_bus *root;
1124 struct resource *res;
1125 unsigned i;
1126 u16 cmd;
1127 int r;
1128
1129 /* Bypass for VF */
1130 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
1131 return 0;
1132
1133 /* skip if the bios has already enabled large BAR */
1134 if (adev->gmc.real_vram_size &&
1135 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1136 return 0;
1137
1138 /* Check if the root BUS has 64bit memory resources */
1139 root = adev->pdev->bus;
1140 while (root->parent)
1141 root = root->parent;
1142
1143 pci_bus_for_each_resource(root, res, i) {
1144 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1145 res->start > 0x100000000ull)
1146 break;
1147 }
1148
1149 /* Trying to resize is pointless without a root hub window above 4GB */
1150 if (!res)
1151 return 0;
1152
1153 /* Disable memory decoding while we change the BAR addresses and size */
1154 pci_read_config_word(adev->pdev, PCI_COMMAND0x04, &cmd);
1155 pci_write_config_word(adev->pdev, PCI_COMMAND0x04,
1156 cmd & ~PCI_COMMAND_MEMORY0x00000002);
1157
1158 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1159 amdgpu_device_doorbell_fini(adev);
1160 if (adev->asic_type >= CHIP_BONAIRE)
1161 pci_release_resource(adev->pdev, 2);
1162
1163 pci_release_resource(adev->pdev, 0);
1164
1165 r = pci_resize_resource(adev->pdev, 0, rbar_size);
1166 if (r == -ENOSPC28)
1167 DRM_INFO("Not enough PCI address space for a large BAR.")printk("\0016" "[" "drm" "] " "Not enough PCI address space for a large BAR."
);
1168 else if (r && r != -ENOTSUPP91)
1169 DRM_ERROR("Problem resizing BAR0 (%d).", r)__drm_err("Problem resizing BAR0 (%d).", r);
1170
1171 pci_assign_unassigned_bus_resources(adev->pdev->bus);
1172
1173 /* When the doorbell or fb BAR isn't available we have no chance of
1174 * using the device.
1175 */
1176 r = amdgpu_device_doorbell_init(adev);
1177 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1178 return -ENODEV19;
1179
1180 pci_write_config_word(adev->pdev, PCI_COMMAND0x04, cmd);
1181#endif /* __linux__ */
1182
1183 return 0;
1184}
1185
1186/*
1187 * GPU helpers function.
1188 */
1189/**
1190 * amdgpu_device_need_post - check if the hw need post or not
1191 *
1192 * @adev: amdgpu_device pointer
1193 *
1194 * Check if the asic has been initialized (all asics) at driver startup
1195 * or post is needed if hw reset is performed.
1196 * Returns true if need or false if not.
1197 */
1198bool_Bool amdgpu_device_need_post(struct amdgpu_device *adev)
1199{
1200 uint32_t reg;
1201
1202 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
1203 return false0;
1204
1205 if (amdgpu_passthrough(adev)((adev)->virt.caps & (1 << 3))) {
1206 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1207 * some old smc fw still need driver do vPost otherwise gpu hang, while
1208 * those smc fw version above 22.15 doesn't have this flaw, so we force
1209 * vpost executed for smc version below 22.15
1210 */
1211 if (adev->asic_type == CHIP_FIJI) {
1212 int err;
1213 uint32_t fw_ver;
1214 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1215 /* force vPost if error occured */
1216 if (err)
1217 return true1;
1218
1219 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1220 if (fw_ver < 0x00160e00)
1221 return true1;
1222 }
1223 }
1224
1225 if (adev->has_hw_reset) {
1226 adev->has_hw_reset = false0;
1227 return true1;
1228 }
1229
1230 /* bios scratch used on CIK+ */
1231 if (adev->asic_type >= CHIP_BONAIRE)
1232 return amdgpu_atombios_scratch_need_asic_init(adev);
1233
1234 /* check MEM_SIZE for older asics */
1235 reg = amdgpu_asic_get_config_memsize(adev)(adev)->asic_funcs->get_config_memsize((adev));
1236
1237 if ((reg != 0) && (reg != 0xffffffff))
1238 return false0;
1239
1240 return true1;
1241}
1242
1243/* if we get transitioned to only one device, take VGA back */
1244/**
1245 * amdgpu_device_vga_set_decode - enable/disable vga decode
1246 *
1247 * @cookie: amdgpu_device pointer
1248 * @state: enable/disable vga decode
1249 *
1250 * Enable/disable vga decode (all asics).
1251 * Returns VGA resource flags.
1252 */
1253#ifdef notyet
1254static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool_Bool state)
1255{
1256 struct amdgpu_device *adev = cookie;
1257 amdgpu_asic_set_vga_state(adev, state)(adev)->asic_funcs->set_vga_state((adev), (state));
1258 if (state)
1259 return VGA_RSRC_LEGACY_IO0x01 | VGA_RSRC_LEGACY_MEM |
1260 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1261 else
1262 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1263}
1264#endif
1265
1266/**
1267 * amdgpu_device_check_block_size - validate the vm block size
1268 *
1269 * @adev: amdgpu_device pointer
1270 *
1271 * Validates the vm block size specified via module parameter.
1272 * The vm block size defines number of bits in page table versus page directory,
1273 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1274 * page table and the remaining bits are in the page directory.
1275 */
1276static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1277{
1278 /* defines number of bits in page table versus page directory,
1279 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1280 * page table and the remaining bits are in the page directory */
1281 if (amdgpu_vm_block_size == -1)
1282 return;
1283
1284 if (amdgpu_vm_block_size < 9) {
1285 dev_warn(adev->dev, "VM page table size (%d) too small\n",printf("drm:pid%d:%s *WARNING* " "VM page table size (%d) too small\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_vm_block_size
)
1286 amdgpu_vm_block_size)printf("drm:pid%d:%s *WARNING* " "VM page table size (%d) too small\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_vm_block_size
);
1287 amdgpu_vm_block_size = -1;
1288 }
1289}
1290
1291/**
1292 * amdgpu_device_check_vm_size - validate the vm size
1293 *
1294 * @adev: amdgpu_device pointer
1295 *
1296 * Validates the vm size in GB specified via module parameter.
1297 * The VM size is the size of the GPU virtual memory space in GB.
1298 */
1299static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1300{
1301 /* no need to check the default value */
1302 if (amdgpu_vm_size == -1)
25
←
Assuming the condition is true
→
26
←
Taking true branch
→
1303 return;
27
←
Returning without writing to 'adev->pm.pcie_gen_mask', which participates in a condition later
→
28
←
Returning without writing to 'adev->pm.pcie_mlw_mask', which participates in a condition later
→
1304
1305 if (amdgpu_vm_size < 1) {
1306 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",printf("drm:pid%d:%s *WARNING* " "VM size (%d) too small, min is 1GB\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_vm_size
)
1307 amdgpu_vm_size)printf("drm:pid%d:%s *WARNING* " "VM size (%d) too small, min is 1GB\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_vm_size
);
1308 amdgpu_vm_size = -1;
1309 }
1310}
1311
1312static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1313{
1314#ifdef __linux__
1315 struct sysinfo si;
1316#endif
1317 bool_Bool is_os_64 = (sizeof(void *) == 8);
1318 uint64_t total_memory;
1319 uint64_t dram_size_seven_GB = 0x1B8000000;
1320 uint64_t dram_size_three_GB = 0xB8000000;
1321
1322 if (amdgpu_smu_memory_pool_size == 0)
19
←
Assuming 'amdgpu_smu_memory_pool_size' is equal to 0
→
20
←
Taking true branch
→
1323 return;
21
←
Returning without writing to 'adev->pm.pcie_gen_mask', which participates in a condition later
→
22
←
Returning without writing to 'adev->pm.pcie_mlw_mask', which participates in a condition later
→
1324
1325 if (!is_os_64) {
1326 DRM_WARN("Not 64-bit OS, feature not supported\n")printk("\0014" "[" "drm" "] " "Not 64-bit OS, feature not supported\n"
);
1327 goto def_value;
1328 }
1329#ifdef __linux__
1330 si_meminfo(&si);
1331 total_memory = (uint64_t)si.totalram * si.mem_unit;
1332#else
1333 total_memory = ptoa(physmem)((paddr_t)(physmem) << 12);
1334#endif
1335
1336 if ((amdgpu_smu_memory_pool_size == 1) ||
1337 (amdgpu_smu_memory_pool_size == 2)) {
1338 if (total_memory < dram_size_three_GB)
1339 goto def_value1;
1340 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1341 (amdgpu_smu_memory_pool_size == 8)) {
1342 if (total_memory < dram_size_seven_GB)
1343 goto def_value1;
1344 } else {
1345 DRM_WARN("Smu memory pool size not supported\n")printk("\0014" "[" "drm" "] " "Smu memory pool size not supported\n"
);
1346 goto def_value;
1347 }
1348 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1349
1350 return;
1351
1352def_value1:
1353 DRM_WARN("No enough system memory\n")printk("\0014" "[" "drm" "] " "No enough system memory\n");
1354def_value:
1355 adev->pm.smu_prv_buffer_size = 0;
1356}
1357
1358/**
1359 * amdgpu_device_check_arguments - validate module params
1360 *
1361 * @adev: amdgpu_device pointer
1362 *
1363 * Validates certain module parameters and updates
1364 * the associated values used by the driver (all asics).
1365 */
1366static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1367{
1368 if (amdgpu_sched_jobs < 4) {
9
←
Assuming 'amdgpu_sched_jobs' is >= 4
→
10
←
Taking false branch
→
1369 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",printf("drm:pid%d:%s *WARNING* " "sched jobs (%d) must be at least 4\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_jobs
)
1370 amdgpu_sched_jobs)printf("drm:pid%d:%s *WARNING* " "sched jobs (%d) must be at least 4\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_jobs
);
1371 amdgpu_sched_jobs = 4;
1372 } else if (!is_power_of_2(amdgpu_sched_jobs)(((amdgpu_sched_jobs) != 0) && (((amdgpu_sched_jobs) -
1) & (amdgpu_sched_jobs)) == 0)){
11
←
Assuming the condition is true
→
12
←
Taking false branch
→
1373 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",printf("drm:pid%d:%s *WARNING* " "sched jobs (%d) must be a power of 2\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_jobs
)
1374 amdgpu_sched_jobs)printf("drm:pid%d:%s *WARNING* " "sched jobs (%d) must be a power of 2\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_jobs
);
1375 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1376 }
1377
1378 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
13
←
Assuming the condition is false
→
1379 /* gart size must be greater or equal to 32M */
1380 dev_warn(adev->dev, "gart size (%d) too small\n",printf("drm:pid%d:%s *WARNING* " "gart size (%d) too small\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_gart_size
)
1381 amdgpu_gart_size)printf("drm:pid%d:%s *WARNING* " "gart size (%d) too small\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_gart_size
);
1382 amdgpu_gart_size = -1;
1383 }
1384
1385 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
14
←
Assuming the condition is false
→
1386 /* gtt size must be greater or equal to 32M */
1387 dev_warn(adev->dev, "gtt size (%d) too small\n",printf("drm:pid%d:%s *WARNING* " "gtt size (%d) too small\n",
({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_gtt_size
)
1388 amdgpu_gtt_size)printf("drm:pid%d:%s *WARNING* " "gtt size (%d) too small\n",
({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_gtt_size
);
1389 amdgpu_gtt_size = -1;
1390 }
1391
1392 /* valid range is between 4 and 9 inclusive */
1393 if (amdgpu_vm_fragment_size != -1 &&
15
←
Assuming the condition is false
→
1394 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1395 dev_warn(adev->dev, "valid range is between 4 and 9\n")printf("drm:pid%d:%s *WARNING* " "valid range is between 4 and 9\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
1396 amdgpu_vm_fragment_size = -1;
1397 }
1398
1399 if (amdgpu_sched_hw_submission < 2) {
16
←
Assuming 'amdgpu_sched_hw_submission' is < 2
→
17
←
Taking true branch
→
1400 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",printf("drm:pid%d:%s *WARNING* " "sched hw submission jobs (%d) must be at least 2\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_hw_submission
)
1401 amdgpu_sched_hw_submission)printf("drm:pid%d:%s *WARNING* " "sched hw submission jobs (%d) must be at least 2\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_hw_submission
);
1402 amdgpu_sched_hw_submission = 2;
1403 } else if (!is_power_of_2(amdgpu_sched_hw_submission)(((amdgpu_sched_hw_submission) != 0) && (((amdgpu_sched_hw_submission
) - 1) & (amdgpu_sched_hw_submission)) == 0)) {
1404 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",printf("drm:pid%d:%s *WARNING* " "sched hw submission jobs (%d) must be a power of 2\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_hw_submission
)
1405 amdgpu_sched_hw_submission)printf("drm:pid%d:%s *WARNING* " "sched hw submission jobs (%d) must be a power of 2\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , amdgpu_sched_hw_submission
);
1406 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1407 }
1408
1409 amdgpu_device_check_smu_prv_buffer_size(adev);
18
←
Calling 'amdgpu_device_check_smu_prv_buffer_size'
→
23
←
Returning from 'amdgpu_device_check_smu_prv_buffer_size'
→
1410
1411 amdgpu_device_check_vm_size(adev);
24
←
Calling 'amdgpu_device_check_vm_size'
→
29
←
Returning from 'amdgpu_device_check_vm_size'
→
1412
1413 amdgpu_device_check_block_size(adev);
1414
1415 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1416
1417 amdgpu_gmc_tmz_set(adev);
1418
1419 if (amdgpu_num_kcq == -1) {
30
←
Assuming the condition is true
→
31
←
Taking true branch
→
1420 amdgpu_num_kcq = 8;
1421 } else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) {
1422 amdgpu_num_kcq = 8;
1423 dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n")printf("drm:pid%d:%s *WARNING* " "set kernel compute queue number to 8 due to invalid parameter provided by user\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
1424 }
1425
1426 amdgpu_gmc_noretry_set(adev);
1427
1428 return 0;
1429}
1430
1431#ifdef __linux__
1432/**
1433 * amdgpu_switcheroo_set_state - set switcheroo state
1434 *
1435 * @pdev: pci dev pointer
1436 * @state: vga_switcheroo state
1437 *
1438 * Callback for the switcheroo driver. Suspends or resumes the
1439 * the asics before or after it is powered up using ACPI methods.
1440 */
1441static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1442 enum vga_switcheroo_state state)
1443{
1444 struct drm_device *dev = pci_get_drvdata(pdev);
1445 int r;
1446
1447 if (amdgpu_device_supports_boco(dev) && state == VGA_SWITCHEROO_OFF)
1448 return;
1449
1450 if (state == VGA_SWITCHEROO_ON) {
1451 pr_info("switched on\n")do { } while(0);
1452 /* don't suspend or resume card normally */
1453 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1454
1455 pci_set_power_state(dev->pdev, PCI_D0);
1456 amdgpu_device_load_pci_state(dev->pdev);
1457 r = pci_enable_device(dev->pdev)0;
1458 if (r)
1459 DRM_WARN("pci_enable_device failed (%d)\n", r)printk("\0014" "[" "drm" "] " "pci_enable_device failed (%d)\n"
, r);
1460 amdgpu_device_resume(dev, true1);
1461
1462 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1463 drm_kms_helper_poll_enable(dev);
1464 } else {
1465 pr_info("switched off\n")do { } while(0);
1466 drm_kms_helper_poll_disable(dev);
1467 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1468 amdgpu_device_suspend(dev, true1);
1469 amdgpu_device_cache_pci_state(dev->pdev);
1470 /* Shut down the device */
1471 pci_disable_device(dev->pdev);
1472 pci_set_power_state(dev->pdev, PCI_D3cold);
1473 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1474 }
1475}
1476
1477/**
1478 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1479 *
1480 * @pdev: pci dev pointer
1481 *
1482 * Callback for the switcheroo driver. Check of the switcheroo
1483 * state can be changed.
1484 * Returns true if the state can be changed, false if not.
1485 */
1486static bool_Bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1487{
1488 struct drm_device *dev = pci_get_drvdata(pdev);
1489
1490 /*
1491 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1492 * locking inversion with the driver load path. And the access here is
1493 * completely racy anyway. So don't bother with locking for now.
1494 */
1495 return atomic_read(&dev->open_count)({ typeof(*(&dev->open_count)) __tmp = *(volatile typeof
(*(&dev->open_count)) *)&(*(&dev->open_count
)); membar_datadep_consumer(); __tmp; }) == 0;
1496}
1497
1498static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1499 .set_gpu_state = amdgpu_switcheroo_set_state,
1500 .reprobe = NULL((void *)0),
1501 .can_switch = amdgpu_switcheroo_can_switch,
1502};
1503#endif /* __linux__ */
1504
1505/**
1506 * amdgpu_device_ip_set_clockgating_state - set the CG state
1507 *
1508 * @dev: amdgpu_device pointer
1509 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1510 * @state: clockgating state (gate or ungate)
1511 *
1512 * Sets the requested clockgating state for all instances of
1513 * the hardware IP specified.
1514 * Returns the error code from the last instance.
1515 */
1516int amdgpu_device_ip_set_clockgating_state(void *dev,
1517 enum amd_ip_block_type block_type,
1518 enum amd_clockgating_state state)
1519{
1520 struct amdgpu_device *adev = dev;
1521 int i, r = 0;
1522
1523 for (i = 0; i < adev->num_ip_blocks; i++) {
1524 if (!adev->ip_blocks[i].status.valid)
1525 continue;
1526 if (adev->ip_blocks[i].version->type != block_type)
1527 continue;
1528 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1529 continue;
1530 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1531 (void *)adev, state);
1532 if (r)
1533 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",__drm_err("set_clockgating_state of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
1534 adev->ip_blocks[i].version->funcs->name, r)__drm_err("set_clockgating_state of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
1535 }
1536 return r;
1537}
1538
1539/**
1540 * amdgpu_device_ip_set_powergating_state - set the PG state
1541 *
1542 * @dev: amdgpu_device pointer
1543 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1544 * @state: powergating state (gate or ungate)
1545 *
1546 * Sets the requested powergating state for all instances of
1547 * the hardware IP specified.
1548 * Returns the error code from the last instance.
1549 */
1550int amdgpu_device_ip_set_powergating_state(void *dev,
1551 enum amd_ip_block_type block_type,
1552 enum amd_powergating_state state)
1553{
1554 struct amdgpu_device *adev = dev;
1555 int i, r = 0;
1556
1557 for (i = 0; i < adev->num_ip_blocks; i++) {
1558 if (!adev->ip_blocks[i].status.valid)
1559 continue;
1560 if (adev->ip_blocks[i].version->type != block_type)
1561 continue;
1562 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1563 continue;
1564 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1565 (void *)adev, state);
1566 if (r)
1567 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",__drm_err("set_powergating_state of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
1568 adev->ip_blocks[i].version->funcs->name, r)__drm_err("set_powergating_state of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
1569 }
1570 return r;
1571}
1572
1573/**
1574 * amdgpu_device_ip_get_clockgating_state - get the CG state
1575 *
1576 * @adev: amdgpu_device pointer
1577 * @flags: clockgating feature flags
1578 *
1579 * Walks the list of IPs on the device and updates the clockgating
1580 * flags for each IP.
1581 * Updates @flags with the feature flags for each hardware IP where
1582 * clockgating is enabled.
1583 */
1584void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1585 u32 *flags)
1586{
1587 int i;
1588
1589 for (i = 0; i < adev->num_ip_blocks; i++) {
1590 if (!adev->ip_blocks[i].status.valid)
1591 continue;
1592 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1593 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1594 }
1595}
1596
1597/**
1598 * amdgpu_device_ip_wait_for_idle - wait for idle
1599 *
1600 * @adev: amdgpu_device pointer
1601 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1602 *
1603 * Waits for the request hardware IP to be idle.
1604 * Returns 0 for success or a negative error code on failure.
1605 */
1606int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1607 enum amd_ip_block_type block_type)
1608{
1609 int i, r;
1610
1611 for (i = 0; i < adev->num_ip_blocks; i++) {
1612 if (!adev->ip_blocks[i].status.valid)
1613 continue;
1614 if (adev->ip_blocks[i].version->type == block_type) {
1615 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1616 if (r)
1617 return r;
1618 break;
1619 }
1620 }
1621 return 0;
1622
1623}
1624
1625/**
1626 * amdgpu_device_ip_is_idle - is the hardware IP idle
1627 *
1628 * @adev: amdgpu_device pointer
1629 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1630 *
1631 * Check if the hardware IP is idle or not.
1632 * Returns true if it the IP is idle, false if not.
1633 */
1634bool_Bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1635 enum amd_ip_block_type block_type)
1636{
1637 int i;
1638
1639 for (i = 0; i < adev->num_ip_blocks; i++) {
1640 if (!adev->ip_blocks[i].status.valid)
1641 continue;
1642 if (adev->ip_blocks[i].version->type == block_type)
1643 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1644 }
1645 return true1;
1646
1647}
1648
1649/**
1650 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1651 *
1652 * @adev: amdgpu_device pointer
1653 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1654 *
1655 * Returns a pointer to the hardware IP block structure
1656 * if it exists for the asic, otherwise NULL.
1657 */
1658struct amdgpu_ip_block *
1659amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1660 enum amd_ip_block_type type)
1661{
1662 int i;
1663
1664 for (i = 0; i < adev->num_ip_blocks; i++)
1665 if (adev->ip_blocks[i].version->type == type)
1666 return &adev->ip_blocks[i];
1667
1668 return NULL((void *)0);
1669}
1670
1671/**
1672 * amdgpu_device_ip_block_version_cmp
1673 *
1674 * @adev: amdgpu_device pointer
1675 * @type: enum amd_ip_block_type
1676 * @major: major version
1677 * @minor: minor version
1678 *
1679 * return 0 if equal or greater
1680 * return 1 if smaller or the ip_block doesn't exist
1681 */
1682int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1683 enum amd_ip_block_type type,
1684 u32 major, u32 minor)
1685{
1686 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1687
1688 if (ip_block && ((ip_block->version->major > major) ||
1689 ((ip_block->version->major == major) &&
1690 (ip_block->version->minor >= minor))))
1691 return 0;
1692
1693 return 1;
1694}
1695
1696/**
1697 * amdgpu_device_ip_block_add
1698 *
1699 * @adev: amdgpu_device pointer
1700 * @ip_block_version: pointer to the IP to add
1701 *
1702 * Adds the IP block driver information to the collection of IPs
1703 * on the asic.
1704 */
1705int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1706 const struct amdgpu_ip_block_version *ip_block_version)
1707{
1708 if (!ip_block_version)
1709 return -EINVAL22;
1710
1711 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,printk("\0016" "[" "drm" "] " "add ip block number %d <%s>\n"
, adev->num_ip_blocks, ip_block_version->funcs->name
)
1712 ip_block_version->funcs->name)printk("\0016" "[" "drm" "] " "add ip block number %d <%s>\n"
, adev->num_ip_blocks, ip_block_version->funcs->name
);
1713
1714 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1715
1716 return 0;
1717}
1718
1719/**
1720 * amdgpu_device_enable_virtual_display - enable virtual display feature
1721 *
1722 * @adev: amdgpu_device pointer
1723 *
1724 * Enabled the virtual display feature if the user has enabled it via
1725 * the module parameter virtual_display. This feature provides a virtual
1726 * display hardware on headless boards or in virtualized environments.
1727 * This function parses and validates the configuration string specified by
1728 * the user and configues the virtual display configuration (number of
1729 * virtual connectors, crtcs, etc.) specified.
1730 */
1731static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1732{
1733 adev->enable_virtual_display = false0;
1734
1735#ifdef notyet
1736 if (amdgpu_virtual_display) {
1737 struct drm_device *ddev = adev_to_drm(adev);
1738 const char *pci_address_name = pci_name(ddev->pdev);
1739 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1740
1741 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL(0x0001 | 0x0004));
1742 pciaddstr_tmp = pciaddstr;
1743 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1744 pciaddname = strsep(&pciaddname_tmp, ",");
1745 if (!strcmp("all", pciaddname)
1746 || !strcmp(pci_address_name, pciaddname)) {
1747 long num_crtc;
1748 int res = -1;
1749
1750 adev->enable_virtual_display = true1;
1751
1752 if (pciaddname_tmp)
1753 res = kstrtol(pciaddname_tmp, 10,
1754 &num_crtc);
1755
1756 if (!res) {
1757 if (num_crtc < 1)
1758 num_crtc = 1;
1759 if (num_crtc > 6)
1760 num_crtc = 6;
1761 adev->mode_info.num_crtc = num_crtc;
1762 } else {
1763 adev->mode_info.num_crtc = 1;
1764 }
1765 break;
1766 }
1767 }
1768
1769 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",printk("\0016" "[" "drm" "] " "virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n"
, amdgpu_virtual_display, pci_address_name, adev->enable_virtual_display
, adev->mode_info.num_crtc)
1770 amdgpu_virtual_display, pci_address_name,printk("\0016" "[" "drm" "] " "virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n"
, amdgpu_virtual_display, pci_address_name, adev->enable_virtual_display
, adev->mode_info.num_crtc)
1771 adev->enable_virtual_display, adev->mode_info.num_crtc)printk("\0016" "[" "drm" "] " "virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n"
, amdgpu_virtual_display, pci_address_name, adev->enable_virtual_display
, adev->mode_info.num_crtc);
1772
1773 kfree(pciaddstr);
1774 }
1775#endif
1776}
1777
1778/**
1779 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1780 *
1781 * @adev: amdgpu_device pointer
1782 *
1783 * Parses the asic configuration parameters specified in the gpu info
1784 * firmware and makes them availale to the driver for use in configuring
1785 * the asic.
1786 * Returns 0 on success, -EINVAL on failure.
1787 */
1788static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1789{
1790 const char *chip_name;
1791 char fw_name[40];
1792 int err;
1793 const struct gpu_info_firmware_header_v1_0 *hdr;
1794
1795 adev->firmware.gpu_info_fw = NULL((void *)0);
1796
1797 if (adev->mman.discovery_bin) {
1798 amdgpu_discovery_get_gfx_info(adev);
1799
1800 /*
1801 * FIXME: The bounding box is still needed by Navi12, so
1802 * temporarily read it from gpu_info firmware. Should be droped
1803 * when DAL no longer needs it.
1804 */
1805 if (adev->asic_type != CHIP_NAVI12)
1806 return 0;
1807 }
1808
1809 switch (adev->asic_type) {
1810#ifdef CONFIG_DRM_AMDGPU_SI
1811 case CHIP_VERDE:
1812 case CHIP_TAHITI:
1813 case CHIP_PITCAIRN:
1814 case CHIP_OLAND:
1815 case CHIP_HAINAN:
1816#endif
1817#ifdef CONFIG_DRM_AMDGPU_CIK
1818 case CHIP_BONAIRE:
1819 case CHIP_HAWAII:
1820 case CHIP_KAVERI:
1821 case CHIP_KABINI:
1822 case CHIP_MULLINS:
1823#endif
1824 case CHIP_TOPAZ:
1825 case CHIP_TONGA:
1826 case CHIP_FIJI:
1827 case CHIP_POLARIS10:
1828 case CHIP_POLARIS11:
1829 case CHIP_POLARIS12:
1830 case CHIP_VEGAM:
1831 case CHIP_CARRIZO:
1832 case CHIP_STONEY:
1833 case CHIP_VEGA20:
1834 case CHIP_SIENNA_CICHLID:
1835 case CHIP_NAVY_FLOUNDER:
1836 default:
1837 return 0;
1838 case CHIP_VEGA10:
1839 chip_name = "vega10";
1840 break;
1841 case CHIP_VEGA12:
1842 chip_name = "vega12";
1843 break;
1844 case CHIP_RAVEN:
1845 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1846 chip_name = "raven2";
1847 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1848 chip_name = "picasso";
1849 else
1850 chip_name = "raven";
1851 break;
1852 case CHIP_ARCTURUS:
1853 chip_name = "arcturus";
1854 break;
1855 case CHIP_RENOIR:
1856 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1857 chip_name = "renoir";
1858 else
1859 chip_name = "green_sardine";
1860 break;
1861 case CHIP_NAVI10:
1862 chip_name = "navi10";
1863 break;
1864 case CHIP_NAVI14:
1865 chip_name = "navi14";
1866 break;
1867 case CHIP_NAVI12:
1868 chip_name = "navi12";
1869 break;
1870 }
1871
1872 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1873 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1874 if (err) {
1875 dev_err(adev->dev,printf("drm:pid%d:%s *ERROR* " "Failed to load gpu_info firmware \"%s\"\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , fw_name
)
1876 "Failed to load gpu_info firmware \"%s\"\n",printf("drm:pid%d:%s *ERROR* " "Failed to load gpu_info firmware \"%s\"\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , fw_name
)
1877 fw_name)printf("drm:pid%d:%s *ERROR* " "Failed to load gpu_info firmware \"%s\"\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , fw_name
);
1878 goto out;
1879 }
1880 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1881 if (err) {
1882 dev_err(adev->dev,printf("drm:pid%d:%s *ERROR* " "Failed to validate gpu_info firmware \"%s\"\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , fw_name
)
1883 "Failed to validate gpu_info firmware \"%s\"\n",printf("drm:pid%d:%s *ERROR* " "Failed to validate gpu_info firmware \"%s\"\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , fw_name
)
1884 fw_name)printf("drm:pid%d:%s *ERROR* " "Failed to validate gpu_info firmware \"%s\"\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , fw_name
);
1885 goto out;
1886 }
1887
1888 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1889 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1890
1891 switch (hdr->version_major) {
1892 case 1:
1893 {
1894 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1895 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1896 le32_to_cpu(hdr->header.ucode_array_offset_bytes)((__uint32_t)(hdr->header.ucode_array_offset_bytes)));
1897
1898 /*
1899 * Should be droped when DAL no longer needs it.
1900 */
1901 if (adev->asic_type == CHIP_NAVI12)
1902 goto parse_soc_bounding_box;
1903
1904 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se)((__uint32_t)(gpu_info_fw->gc_num_se));
1905 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh)((__uint32_t)(gpu_info_fw->gc_num_cu_per_sh));
1906 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se)((__uint32_t)(gpu_info_fw->gc_num_sh_per_se));
1907 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se)((__uint32_t)(gpu_info_fw->gc_num_rb_per_se));
1908 adev->gfx.config.max_texture_channel_caches =
1909 le32_to_cpu(gpu_info_fw->gc_num_tccs)((__uint32_t)(gpu_info_fw->gc_num_tccs));
1910 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs)((__uint32_t)(gpu_info_fw->gc_num_gprs));
1911 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds)((__uint32_t)(gpu_info_fw->gc_num_max_gs_thds));
1912 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth)((__uint32_t)(gpu_info_fw->gc_gs_table_depth));
1913 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth)((__uint32_t)(gpu_info_fw->gc_gsprim_buff_depth));
1914 adev->gfx.config.double_offchip_lds_buf =
1915 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer)((__uint32_t)(gpu_info_fw->gc_double_offchip_lds_buffer));
1916 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size)((__uint32_t)(gpu_info_fw->gc_wave_size));
1917 adev->gfx.cu_info.max_waves_per_simd =
1918 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd)((__uint32_t)(gpu_info_fw->gc_max_waves_per_simd));
1919 adev->gfx.cu_info.max_scratch_slots_per_cu =
1920 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu)((__uint32_t)(gpu_info_fw->gc_max_scratch_slots_per_cu));
1921 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size)((__uint32_t)(gpu_info_fw->gc_lds_size));
1922 if (hdr->version_minor >= 1) {
1923 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1924 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1925 le32_to_cpu(hdr->header.ucode_array_offset_bytes)((__uint32_t)(hdr->header.ucode_array_offset_bytes)));
1926 adev->gfx.config.num_sc_per_sh =
1927 le32_to_cpu(gpu_info_fw->num_sc_per_sh)((__uint32_t)(gpu_info_fw->num_sc_per_sh));
1928 adev->gfx.config.num_packer_per_sc =
1929 le32_to_cpu(gpu_info_fw->num_packer_per_sc)((__uint32_t)(gpu_info_fw->num_packer_per_sc));
1930 }
1931
1932parse_soc_bounding_box:
1933 /*
1934 * soc bounding box info is not integrated in disocovery table,
1935 * we always need to parse it from gpu info firmware if needed.
1936 */
1937 if (hdr->version_minor == 2) {
1938 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1939 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1940 le32_to_cpu(hdr->header.ucode_array_offset_bytes)((__uint32_t)(hdr->header.ucode_array_offset_bytes)));
1941 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1942 }
1943 break;
1944 }
1945 default:
1946 dev_err(adev->dev,printf("drm:pid%d:%s *ERROR* " "Unsupported gpu_info table %d\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , hdr->
header.ucode_version)
1947 "Unsupported gpu_info table %d\n", hdr->header.ucode_version)printf("drm:pid%d:%s *ERROR* " "Unsupported gpu_info table %d\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , hdr->
header.ucode_version);
1948 err = -EINVAL22;
1949 goto out;
1950 }
1951out:
1952 return err;
1953}
1954
1955/**
1956 * amdgpu_device_ip_early_init - run early init for hardware IPs
1957 *
1958 * @adev: amdgpu_device pointer
1959 *
1960 * Early initialization pass for hardware IPs. The hardware IPs that make
1961 * up each asic are discovered each IP's early_init callback is run. This
1962 * is the first stage in initializing the asic.
1963 * Returns 0 on success, negative error code on failure.
1964 */
1965static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1966{
1967 int i, r;
1968
1969 amdgpu_device_enable_virtual_display(adev);
1970
1971 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
1972 r = amdgpu_virt_request_full_gpu(adev, true1);
1973 if (r)
1974 return r;
1975 }
1976
1977 switch (adev->asic_type) {
1978#ifdef CONFIG_DRM_AMDGPU_SI
1979 case CHIP_VERDE:
1980 case CHIP_TAHITI:
1981 case CHIP_PITCAIRN:
1982 case CHIP_OLAND:
1983 case CHIP_HAINAN:
1984 adev->family = AMDGPU_FAMILY_SI110;
1985 r = si_set_ip_blocks(adev);
1986 if (r)
1987 return r;
1988 break;
1989#endif
1990#ifdef CONFIG_DRM_AMDGPU_CIK
1991 case CHIP_BONAIRE:
1992 case CHIP_HAWAII:
1993 case CHIP_KAVERI:
1994 case CHIP_KABINI:
1995 case CHIP_MULLINS:
1996 if (adev->flags & AMD_IS_APU)
1997 adev->family = AMDGPU_FAMILY_KV125;
1998 else
1999 adev->family = AMDGPU_FAMILY_CI120;
2000
2001 r = cik_set_ip_blocks(adev);
2002 if (r)
2003 return r;
2004 break;
2005#endif
2006 case CHIP_TOPAZ:
2007 case CHIP_TONGA:
2008 case CHIP_FIJI:
2009 case CHIP_POLARIS10:
2010 case CHIP_POLARIS11:
2011 case CHIP_POLARIS12:
2012 case CHIP_VEGAM:
2013 case CHIP_CARRIZO:
2014 case CHIP_STONEY:
2015 if (adev->flags & AMD_IS_APU)
2016 adev->family = AMDGPU_FAMILY_CZ135;
2017 else
2018 adev->family = AMDGPU_FAMILY_VI130;
2019
2020 r = vi_set_ip_blocks(adev);
2021 if (r)
2022 return r;
2023 break;
2024 case CHIP_VEGA10:
2025 case CHIP_VEGA12:
2026 case CHIP_VEGA20:
2027 case CHIP_RAVEN:
2028 case CHIP_ARCTURUS:
2029 case CHIP_RENOIR:
2030 if (adev->flags & AMD_IS_APU)
2031 adev->family = AMDGPU_FAMILY_RV142;
2032 else
2033 adev->family = AMDGPU_FAMILY_AI141;
2034
2035 r = soc15_set_ip_blocks(adev);
2036 if (r)
2037 return r;
2038 break;
2039 case CHIP_NAVI10:
2040 case CHIP_NAVI14:
2041 case CHIP_NAVI12:
2042 case CHIP_SIENNA_CICHLID:
2043 case CHIP_NAVY_FLOUNDER:
2044 adev->family = AMDGPU_FAMILY_NV143;
2045
2046 r = nv_set_ip_blocks(adev);
2047 if (r)
2048 return r;
2049 break;
2050 default:
2051 /* FIXME: not supported yet */
2052 return -EINVAL22;
2053 }
2054
2055 amdgpu_amdkfd_device_probe(adev);
2056
2057 adev->pm.pp_feature = amdgpu_pp_feature_mask;
2058 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2059 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2060
2061 for (i = 0; i < adev->num_ip_blocks; i++) {
2062 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2063 DRM_ERROR("disabled ip block: %d <%s>\n",__drm_err("disabled ip block: %d <%s>\n", i, adev->ip_blocks
[i].version->funcs->name)
2064 i, adev->ip_blocks[i].version->funcs->name)__drm_err("disabled ip block: %d <%s>\n", i, adev->ip_blocks
[i].version->funcs->name);
2065 adev->ip_blocks[i].status.valid = false0;
2066 } else {
2067 if (adev->ip_blocks[i].version->funcs->early_init) {
2068 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2069 if (r == -ENOENT2) {
2070 adev->ip_blocks[i].status.valid = false0;
2071 } else if (r) {
2072 DRM_ERROR("early_init of IP block <%s> failed %d\n",__drm_err("early_init of IP block <%s> failed %d\n", adev
->ip_blocks[i].version->funcs->name, r)
2073 adev->ip_blocks[i].version->funcs->name, r)__drm_err("early_init of IP block <%s> failed %d\n", adev
->ip_blocks[i].version->funcs->name, r);
2074 return r;
2075 } else {
2076 adev->ip_blocks[i].status.valid = true1;
2077 }
2078 } else {
2079 adev->ip_blocks[i].status.valid = true1;
2080 }
2081 }
2082 /* get the vbios after the asic_funcs are set up */
2083 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2084 r = amdgpu_device_parse_gpu_info_fw(adev);
2085 if (r)
2086 return r;
2087
2088 /* Read BIOS */
2089 if (!amdgpu_get_bios(adev))
2090 return -EINVAL22;
2091
2092 r = amdgpu_atombios_init(adev);
2093 if (r) {
2094 dev_err(adev->dev, "amdgpu_atombios_init failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_atombios_init failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
2095 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2096 return r;
2097 }
2098 }
2099 }
2100
2101 adev->cg_flags &= amdgpu_cg_mask;
2102 adev->pg_flags &= amdgpu_pg_mask;
2103
2104 return 0;
2105}
2106
2107static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2108{
2109 int i, r;
2110
2111 for (i = 0; i < adev->num_ip_blocks; i++) {
2112 if (!adev->ip_blocks[i].status.sw)
2113 continue;
2114 if (adev->ip_blocks[i].status.hw)
2115 continue;
2116 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2117 (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2118 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2119 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2120 if (r) {
2121 DRM_ERROR("hw_init of IP block <%s> failed %d\n",__drm_err("hw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2122 adev->ip_blocks[i].version->funcs->name, r)__drm_err("hw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2123 return r;
2124 }
2125 adev->ip_blocks[i].status.hw = true1;
2126 }
2127 }
2128
2129 return 0;
2130}
2131
2132static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2133{
2134 int i, r;
2135
2136 for (i = 0; i < adev->num_ip_blocks; i++) {
2137 if (!adev->ip_blocks[i].status.sw)
2138 continue;
2139 if (adev->ip_blocks[i].status.hw)
2140 continue;
2141 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2142 if (r) {
2143 DRM_ERROR("hw_init of IP block <%s> failed %d\n",__drm_err("hw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2144 adev->ip_blocks[i].version->funcs->name, r)__drm_err("hw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2145 return r;
2146 }
2147 adev->ip_blocks[i].status.hw = true1;
2148 }
2149
2150 return 0;
2151}
2152
2153static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2154{
2155 int r = 0;
2156 int i;
2157 uint32_t smu_version;
2158
2159 if (adev->asic_type >= CHIP_VEGA10) {
2160 for (i = 0; i < adev->num_ip_blocks; i++) {
2161 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2162 continue;
2163
2164 /* no need to do the fw loading again if already done*/
2165 if (adev->ip_blocks[i].status.hw == true1)
2166 break;
2167
2168 if (amdgpu_in_reset(adev) || adev->in_suspend) {
2169 r = adev->ip_blocks[i].version->funcs->resume(adev);
2170 if (r) {
2171 DRM_ERROR("resume of IP block <%s> failed %d\n",__drm_err("resume of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2172 adev->ip_blocks[i].version->funcs->name, r)__drm_err("resume of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2173 return r;
2174 }
2175 } else {
2176 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2177 if (r) {
2178 DRM_ERROR("hw_init of IP block <%s> failed %d\n",__drm_err("hw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2179 adev->ip_blocks[i].version->funcs->name, r)__drm_err("hw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2180 return r;
2181 }
2182 }
2183
2184 adev->ip_blocks[i].status.hw = true1;
2185 break;
2186 }
2187 }
2188
2189 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) || adev->asic_type == CHIP_TONGA)
2190 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2191
2192 return r;
2193}
2194
2195/**
2196 * amdgpu_device_ip_init - run init for hardware IPs
2197 *
2198 * @adev: amdgpu_device pointer
2199 *
2200 * Main initialization pass for hardware IPs. The list of all the hardware
2201 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2202 * are run. sw_init initializes the software state associated with each IP
2203 * and hw_init initializes the hardware associated with each IP.
2204 * Returns 0 on success, negative error code on failure.
2205 */
2206static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2207{
2208 int i, r;
2209
2210 r = amdgpu_ras_init(adev);
2211 if (r)
2212 return r;
2213
2214 for (i = 0; i < adev->num_ip_blocks; i++) {
2215 if (!adev->ip_blocks[i].status.valid)
2216 continue;
2217 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2218 if (r) {
2219 DRM_ERROR("sw_init of IP block <%s> failed %d\n",__drm_err("sw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2220 adev->ip_blocks[i].version->funcs->name, r)__drm_err("sw_init of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2221 goto init_failed;
2222 }
2223 adev->ip_blocks[i].status.sw = true1;
2224
2225 /* need to do gmc hw init early so we can allocate gpu mem */
2226 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2227 r = amdgpu_device_vram_scratch_init(adev);
2228 if (r) {
2229 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r)__drm_err("amdgpu_vram_scratch_init failed %d\n", r);
2230 goto init_failed;
2231 }
2232 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2233 if (r) {
2234 DRM_ERROR("hw_init %d failed %d\n", i, r)__drm_err("hw_init %d failed %d\n", i, r);
2235 goto init_failed;
2236 }
2237 r = amdgpu_device_wb_init(adev);
2238 if (r) {
2239 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r)__drm_err("amdgpu_device_wb_init failed %d\n", r);
2240 goto init_failed;
2241 }
2242 adev->ip_blocks[i].status.hw = true1;
2243
2244 /* right after GMC hw init, we create CSA */
2245 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
2246 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2247 AMDGPU_GEM_DOMAIN_VRAM0x4,
2248 AMDGPU_CSA_SIZE(128 * 1024));
2249 if (r) {
2250 DRM_ERROR("allocate CSA failed %d\n", r)__drm_err("allocate CSA failed %d\n", r);
2251 goto init_failed;
2252 }
2253 }
2254 }
2255 }
2256
2257 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
2258 amdgpu_virt_init_data_exchange(adev);
2259
2260 r = amdgpu_ib_pool_init(adev);
2261 if (r) {
2262 dev_err(adev->dev, "IB initialization failed (%d).\n", r)printf("drm:pid%d:%s *ERROR* " "IB initialization failed (%d).\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
2263 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2264 goto init_failed;
2265 }
2266
2267 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2268 if (r)
2269 goto init_failed;
2270
2271 r = amdgpu_device_ip_hw_init_phase1(adev);
2272 if (r)
2273 goto init_failed;
2274
2275 r = amdgpu_device_fw_loading(adev);
2276 if (r)
2277 goto init_failed;
2278
2279 r = amdgpu_device_ip_hw_init_phase2(adev);
2280 if (r)
2281 goto init_failed;
2282
2283 /*
2284 * retired pages will be loaded from eeprom and reserved here,
2285 * it should be called after amdgpu_device_ip_hw_init_phase2 since
2286 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2287 * for I2C communication which only true at this point.
2288 *
2289 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2290 * failure from bad gpu situation and stop amdgpu init process
2291 * accordingly. For other failed cases, it will still release all
2292 * the resource and print error message, rather than returning one
2293 * negative value to upper level.
2294 *
2295 * Note: theoretically, this should be called before all vram allocations
2296 * to protect retired page from abusing
2297 */
2298 r = amdgpu_ras_recovery_init(adev);
2299 if (r)
2300 goto init_failed;
2301
2302 if (adev->gmc.xgmi.num_physical_nodes > 1)
2303 amdgpu_xgmi_add_device(adev);
2304 amdgpu_amdkfd_device_init(adev);
2305
2306 amdgpu_fru_get_product_info(adev);
2307
2308init_failed:
2309 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
2310 amdgpu_virt_release_full_gpu(adev, true1);
2311
2312 return r;
2313}
2314
2315/**
2316 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2317 *
2318 * @adev: amdgpu_device pointer
2319 *
2320 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
2321 * this function before a GPU reset. If the value is retained after a
2322 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2323 */
2324static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2325{
2326 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM)__builtin_memcpy((adev->reset_magic), (adev->gart.ptr),
(64));
2327}
2328
2329/**
2330 * amdgpu_device_check_vram_lost - check if vram is valid
2331 *
2332 * @adev: amdgpu_device pointer
2333 *
2334 * Checks the reset magic value written to the gart pointer in VRAM.
2335 * The driver calls this after a GPU reset to see if the contents of
2336 * VRAM is lost or now.
2337 * returns true if vram is lost, false if not.
2338 */
2339static bool_Bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2340{
2341 if (memcmp(adev->gart.ptr, adev->reset_magic,__builtin_memcmp((adev->gart.ptr), (adev->reset_magic),
(64))
2342 AMDGPU_RESET_MAGIC_NUM)__builtin_memcmp((adev->gart.ptr), (adev->reset_magic),
(64)))
2343 return true1;
2344
2345 if (!amdgpu_in_reset(adev))
2346 return false0;
2347
2348 /*
2349 * For all ASICs with baco/mode1 reset, the VRAM is
2350 * always assumed to be lost.
2351 */
2352 switch (amdgpu_asic_reset_method(adev)(adev)->asic_funcs->reset_method((adev))) {
2353 case AMD_RESET_METHOD_BACO:
2354 case AMD_RESET_METHOD_MODE1:
2355 return true1;
2356 default:
2357 return false0;
2358 }
2359}
2360
2361/**
2362 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2363 *
2364 * @adev: amdgpu_device pointer
2365 * @state: clockgating state (gate or ungate)
2366 *
2367 * The list of all the hardware IPs that make up the asic is walked and the
2368 * set_clockgating_state callbacks are run.
2369 * Late initialization pass enabling clockgating for hardware IPs.
2370 * Fini or suspend, pass disabling clockgating for hardware IPs.
2371 * Returns 0 on success, negative error code on failure.
2372 */
2373
2374static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2375 enum amd_clockgating_state state)
2376{
2377 int i, j, r;
2378
2379 if (amdgpu_emu_mode == 1)
2380 return 0;
2381
2382 for (j = 0; j < adev->num_ip_blocks; j++) {
2383 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2384 if (!adev->ip_blocks[i].status.late_initialized)
2385 continue;
2386 /* skip CG for VCE/UVD, it's handled specially */
2387 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2388 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2389 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2390 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2391 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2392 /* enable clockgating to save power */
2393 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2394 state);
2395 if (r) {
2396 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",__drm_err("set_clockgating_state(gate) of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
2397 adev->ip_blocks[i].version->funcs->name, r)__drm_err("set_clockgating_state(gate) of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
2398 return r;
2399 }
2400 }
2401 }
2402
2403 return 0;
2404}
2405
2406static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
2407{
2408 int i, j, r;
2409
2410 if (amdgpu_emu_mode == 1)
2411 return 0;
2412
2413 for (j = 0; j < adev->num_ip_blocks; j++) {
2414 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2415 if (!adev->ip_blocks[i].status.late_initialized)
2416 continue;
2417 /* skip CG for VCE/UVD, it's handled specially */
2418 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2419 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2420 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2421 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2422 adev->ip_blocks[i].version->funcs->set_powergating_state) {
2423 /* enable powergating to save power */
2424 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2425 state);
2426 if (r) {
2427 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",__drm_err("set_powergating_state(gate) of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
2428 adev->ip_blocks[i].version->funcs->name, r)__drm_err("set_powergating_state(gate) of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
2429 return r;
2430 }
2431 }
2432 }
2433 return 0;
2434}
2435
2436static int amdgpu_device_enable_mgpu_fan_boost(void)
2437{
2438 struct amdgpu_gpu_instance *gpu_ins;
2439 struct amdgpu_device *adev;
2440 int i, ret = 0;
2441
2442 mutex_lock(&mgpu_info.mutex)rw_enter_write(&mgpu_info.mutex);
2443
2444 /*
2445 * MGPU fan boost feature should be enabled
2446 * only when there are two or more dGPUs in
2447 * the system
2448 */
2449 if (mgpu_info.num_dgpu < 2)
2450 goto out;
2451
2452 for (i = 0; i < mgpu_info.num_dgpu; i++) {
2453 gpu_ins = &(mgpu_info.gpu_ins[i]);
2454 adev = gpu_ins->adev;
2455 if (!(adev->flags & AMD_IS_APU) &&
2456 !gpu_ins->mgpu_fan_enabled) {
2457 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2458 if (ret)
2459 break;
2460
2461 gpu_ins->mgpu_fan_enabled = 1;
2462 }
2463 }
2464
2465out:
2466 mutex_unlock(&mgpu_info.mutex)rw_exit_write(&mgpu_info.mutex);
2467
2468 return ret;
2469}
2470
2471/**
2472 * amdgpu_device_ip_late_init - run late init for hardware IPs
2473 *
2474 * @adev: amdgpu_device pointer
2475 *
2476 * Late initialization pass for hardware IPs. The list of all the hardware
2477 * IPs that make up the asic is walked and the late_init callbacks are run.
2478 * late_init covers any special initialization that an IP requires
2479 * after all of the have been initialized or something that needs to happen
2480 * late in the init process.
2481 * Returns 0 on success, negative error code on failure.
2482 */
2483static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2484{
2485 struct amdgpu_gpu_instance *gpu_instance;
2486 int i = 0, r;
2487
2488 for (i = 0; i < adev->num_ip_blocks; i++) {
2489 if (!adev->ip_blocks[i].status.hw)
2490 continue;
2491 if (adev->ip_blocks[i].version->funcs->late_init) {
2492 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2493 if (r) {
2494 DRM_ERROR("late_init of IP block <%s> failed %d\n",__drm_err("late_init of IP block <%s> failed %d\n", adev
->ip_blocks[i].version->funcs->name, r)
2495 adev->ip_blocks[i].version->funcs->name, r)__drm_err("late_init of IP block <%s> failed %d\n", adev
->ip_blocks[i].version->funcs->name, r);
2496 return r;
2497 }
2498 }
2499 adev->ip_blocks[i].status.late_initialized = true1;
2500 }
2501
2502 amdgpu_ras_set_error_query_ready(adev, true1);
2503
2504 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2505 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2506
2507 amdgpu_device_fill_reset_magic(adev);
2508
2509 r = amdgpu_device_enable_mgpu_fan_boost();
2510 if (r)
2511 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r)__drm_err("enable mgpu fan boost failed (%d).\n", r);
2512
2513
2514 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2515 mutex_lock(&mgpu_info.mutex)rw_enter_write(&mgpu_info.mutex);
2516
2517 /*
2518 * Reset device p-state to low as this was booted with high.
2519 *
2520 * This should be performed only after all devices from the same
2521 * hive get initialized.
2522 *
2523 * However, it's unknown how many device in the hive in advance.
2524 * As this is counted one by one during devices initializations.
2525 *
2526 * So, we wait for all XGMI interlinked devices initialized.
2527 * This may bring some delays as those devices may come from
2528 * different hives. But that should be OK.
2529 */
2530 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2531 for (i = 0; i < mgpu_info.num_gpu; i++) {
2532 gpu_instance = &(mgpu_info.gpu_ins[i]);
2533 if (gpu_instance->adev->flags & AMD_IS_APU)
2534 continue;
2535
2536 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2537 AMDGPU_XGMI_PSTATE_MIN);
2538 if (r) {
2539 DRM_ERROR("pstate setting failed (%d).\n", r)__drm_err("pstate setting failed (%d).\n", r);
2540 break;
2541 }
2542 }
2543 }
2544
2545 mutex_unlock(&mgpu_info.mutex)rw_exit_write(&mgpu_info.mutex);
2546 }
2547
2548 return 0;
2549}
2550
2551/**
2552 * amdgpu_device_ip_fini - run fini for hardware IPs
2553 *
2554 * @adev: amdgpu_device pointer
2555 *
2556 * Main teardown pass for hardware IPs. The list of all the hardware
2557 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2558 * are run. hw_fini tears down the hardware associated with each IP
2559 * and sw_fini tears down any software state associated with each IP.
2560 * Returns 0 on success, negative error code on failure.
2561 */
2562static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2563{
2564 int i, r;
2565
2566 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) && adev->virt.ras_init_done)
2567 amdgpu_virt_release_ras_err_handler_data(adev);
2568
2569 amdgpu_ras_pre_fini(adev);
2570
2571 if (adev->gmc.xgmi.num_physical_nodes > 1)
2572 amdgpu_xgmi_remove_device(adev);
2573
2574 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2575 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2576
2577 amdgpu_amdkfd_device_fini(adev);
2578
2579 /* need to disable SMC first */
2580 for (i = 0; i < adev->num_ip_blocks; i++) {
2581 if (!adev->ip_blocks[i].status.hw)
2582 continue;
2583 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2584 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2585 /* XXX handle errors */
2586 if (r) {
2587 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",__drm_dbg(DRM_UT_CORE, "hw_fini of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
2588 adev->ip_blocks[i].version->funcs->name, r)__drm_dbg(DRM_UT_CORE, "hw_fini of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
2589 }
2590 adev->ip_blocks[i].status.hw = false0;
2591 break;
2592 }
2593 }
2594
2595 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2596 if (!adev->ip_blocks[i].status.hw)
2597 continue;
2598
2599 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2600 /* XXX handle errors */
2601 if (r) {
2602 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",__drm_dbg(DRM_UT_CORE, "hw_fini of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
2603 adev->ip_blocks[i].version->funcs->name, r)__drm_dbg(DRM_UT_CORE, "hw_fini of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
2604 }
2605
2606 adev->ip_blocks[i].status.hw = false0;
2607 }
2608
2609
2610 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2611 if (!adev->ip_blocks[i].status.sw)
2612 continue;
2613
2614 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2615 amdgpu_ucode_free_bo(adev);
2616 amdgpu_free_static_csa(&adev->virt.csa_obj);
2617 amdgpu_device_wb_fini(adev);
2618 amdgpu_device_vram_scratch_fini(adev);
2619 amdgpu_ib_pool_fini(adev);
2620 }
2621
2622 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2623 /* XXX handle errors */
2624 if (r) {
2625 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",__drm_dbg(DRM_UT_CORE, "sw_fini of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r)
2626 adev->ip_blocks[i].version->funcs->name, r)__drm_dbg(DRM_UT_CORE, "sw_fini of IP block <%s> failed %d\n"
, adev->ip_blocks[i].version->funcs->name, r);
2627 }
2628 adev->ip_blocks[i].status.sw = false0;
2629 adev->ip_blocks[i].status.valid = false0;
2630 }
2631
2632 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2633 if (!adev->ip_blocks[i].status.late_initialized)
2634 continue;
2635 if (adev->ip_blocks[i].version->funcs->late_fini)
2636 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2637 adev->ip_blocks[i].status.late_initialized = false0;
2638 }
2639
2640 amdgpu_ras_fini(adev);
2641
2642 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
2643 if (amdgpu_virt_release_full_gpu(adev, false0))
2644 DRM_ERROR("failed to release exclusive mode on fini\n")__drm_err("failed to release exclusive mode on fini\n");
2645
2646 return 0;
2647}
2648
2649/**
2650 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2651 *
2652 * @work: work_struct.
2653 */
2654static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2655{
2656 struct amdgpu_device *adev =
2657 container_of(work, struct amdgpu_device, delayed_init_work.work)({ const __typeof( ((struct amdgpu_device *)0)->delayed_init_work
.work ) *__mptr = (work); (struct amdgpu_device *)( (char *)__mptr
- __builtin_offsetof(struct amdgpu_device, delayed_init_work
.work) );});
2658 int r;
2659
2660 r = amdgpu_ib_ring_tests(adev);
2661 if (r)
2662 DRM_ERROR("ib ring test failed (%d).\n", r)__drm_err("ib ring test failed (%d).\n", r);
2663}
2664
2665static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2666{
2667 struct amdgpu_device *adev =
2668 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work)({ const __typeof( ((struct amdgpu_device *)0)->gfx.gfx_off_delay_work
.work ) *__mptr = (work); (struct amdgpu_device *)( (char *)__mptr
- __builtin_offsetof(struct amdgpu_device, gfx.gfx_off_delay_work
.work) );});
2669
2670 WARN_ON_ONCE(adev->gfx.gfx_off_state)({ static int __warned; int __ret = !!(adev->gfx.gfx_off_state
); if (__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "adev->gfx.gfx_off_state", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 2670); __warned = 1; } __builtin_expect(!!(__ret), 0); });
2671 WARN_ON_ONCE(adev->gfx.gfx_off_req_count)({ static int __warned; int __ret = !!(adev->gfx.gfx_off_req_count
); if (__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "adev->gfx.gfx_off_req_count", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 2671); __warned = 1; } __builtin_expect(!!(__ret), 0); });
2672
2673 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true1))
2674 adev->gfx.gfx_off_state = true1;
2675}
2676
2677/**
2678 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2679 *
2680 * @adev: amdgpu_device pointer
2681 *
2682 * Main suspend function for hardware IPs. The list of all the hardware
2683 * IPs that make up the asic is walked, clockgating is disabled and the
2684 * suspend callbacks are run. suspend puts the hardware and software state
2685 * in each IP into a state suitable for suspend.
2686 * Returns 0 on success, negative error code on failure.
2687 */
2688static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2689{
2690 int i, r;
2691
2692 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2693 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2694
2695 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2696 if (!adev->ip_blocks[i].status.valid)
2697 continue;
2698
2699 /* displays are handled separately */
2700 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2701 continue;
2702
2703 /* XXX handle errors */
2704 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2705 /* XXX handle errors */
2706 if (r) {
2707 DRM_ERROR("suspend of IP block <%s> failed %d\n",__drm_err("suspend of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2708 adev->ip_blocks[i].version->funcs->name, r)__drm_err("suspend of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2709 return r;
2710 }
2711
2712 adev->ip_blocks[i].status.hw = false0;
2713 }
2714
2715 return 0;
2716}
2717
2718/**
2719 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2720 *
2721 * @adev: amdgpu_device pointer
2722 *
2723 * Main suspend function for hardware IPs. The list of all the hardware
2724 * IPs that make up the asic is walked, clockgating is disabled and the
2725 * suspend callbacks are run. suspend puts the hardware and software state
2726 * in each IP into a state suitable for suspend.
2727 * Returns 0 on success, negative error code on failure.
2728 */
2729static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2730{
2731 int i, r;
2732
2733 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2734 if (!adev->ip_blocks[i].status.valid)
2735 continue;
2736 /* displays are handled in phase1 */
2737 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2738 continue;
2739 /* PSP lost connection when err_event_athub occurs */
2740 if (amdgpu_ras_intr_triggered() &&
2741 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2742 adev->ip_blocks[i].status.hw = false0;
2743 continue;
2744 }
2745 /* XXX handle errors */
2746 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2747 /* XXX handle errors */
2748 if (r) {
2749 DRM_ERROR("suspend of IP block <%s> failed %d\n",__drm_err("suspend of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2750 adev->ip_blocks[i].version->funcs->name, r)__drm_err("suspend of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2751 }
2752 adev->ip_blocks[i].status.hw = false0;
2753 /* handle putting the SMC in the appropriate state */
2754 if(!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))){
2755 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2756 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2757 if (r) {
2758 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",__drm_err("SMC failed to set mp1 state %d, %d\n", adev->mp1_state
, r)
2759 adev->mp1_state, r)__drm_err("SMC failed to set mp1 state %d, %d\n", adev->mp1_state
, r);
2760 return r;
2761 }
2762 }
2763 }
2764 adev->ip_blocks[i].status.hw = false0;
2765 }
2766
2767 return 0;
2768}
2769
2770/**
2771 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2772 *
2773 * @adev: amdgpu_device pointer
2774 *
2775 * Main suspend function for hardware IPs. The list of all the hardware
2776 * IPs that make up the asic is walked, clockgating is disabled and the
2777 * suspend callbacks are run. suspend puts the hardware and software state
2778 * in each IP into a state suitable for suspend.
2779 * Returns 0 on success, negative error code on failure.
2780 */
2781int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2782{
2783 int r;
2784
2785 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
2786 amdgpu_virt_request_full_gpu(adev, false0);
2787
2788 r = amdgpu_device_ip_suspend_phase1(adev);
2789 if (r)
2790 return r;
2791 r = amdgpu_device_ip_suspend_phase2(adev);
2792
2793 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
2794 amdgpu_virt_release_full_gpu(adev, false0);
2795
2796 return r;
2797}
2798
2799static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2800{
2801 int i, r;
2802
2803 static enum amd_ip_block_type ip_order[] = {
2804 AMD_IP_BLOCK_TYPE_GMC,
2805 AMD_IP_BLOCK_TYPE_COMMON,
2806 AMD_IP_BLOCK_TYPE_PSP,
2807 AMD_IP_BLOCK_TYPE_IH,
2808 };
2809
2810 for (i = 0; i < adev->num_ip_blocks; i++) {
2811 int j;
2812 struct amdgpu_ip_block *block;
2813
2814 block = &adev->ip_blocks[i];
2815 block->status.hw = false0;
2816
2817 for (j = 0; j < ARRAY_SIZE(ip_order)(sizeof((ip_order)) / sizeof((ip_order)[0])); j++) {
2818
2819 if (block->version->type != ip_order[j] ||
2820 !block->status.valid)
2821 continue;
2822
2823 r = block->version->funcs->hw_init(adev);
2824 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded")printk("\0016" "[" "drm" "] " "RE-INIT-early: %s %s\n", block
->version->funcs->name, r?"failed":"succeeded");
2825 if (r)
2826 return r;
2827 block->status.hw = true1;
2828 }
2829 }
2830
2831 return 0;
2832}
2833
2834static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2835{
2836 int i, r;
2837
2838 static enum amd_ip_block_type ip_order[] = {
2839 AMD_IP_BLOCK_TYPE_SMC,
2840 AMD_IP_BLOCK_TYPE_DCE,
2841 AMD_IP_BLOCK_TYPE_GFX,
2842 AMD_IP_BLOCK_TYPE_SDMA,
2843 AMD_IP_BLOCK_TYPE_UVD,
2844 AMD_IP_BLOCK_TYPE_VCE,
2845 AMD_IP_BLOCK_TYPE_VCN
2846 };
2847
2848 for (i = 0; i < ARRAY_SIZE(ip_order)(sizeof((ip_order)) / sizeof((ip_order)[0])); i++) {
2849 int j;
2850 struct amdgpu_ip_block *block;
2851
2852 for (j = 0; j < adev->num_ip_blocks; j++) {
2853 block = &adev->ip_blocks[j];
2854
2855 if (block->version->type != ip_order[i] ||
2856 !block->status.valid ||
2857 block->status.hw)
2858 continue;
2859
2860 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
2861 r = block->version->funcs->resume(adev);
2862 else
2863 r = block->version->funcs->hw_init(adev);
2864
2865 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded")printk("\0016" "[" "drm" "] " "RE-INIT-late: %s %s\n", block->
version->funcs->name, r?"failed":"succeeded");
2866 if (r)
2867 return r;
2868 block->status.hw = true1;
2869 }
2870 }
2871
2872 return 0;
2873}
2874
2875/**
2876 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2877 *
2878 * @adev: amdgpu_device pointer
2879 *
2880 * First resume function for hardware IPs. The list of all the hardware
2881 * IPs that make up the asic is walked and the resume callbacks are run for
2882 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2883 * after a suspend and updates the software state as necessary. This
2884 * function is also used for restoring the GPU after a GPU reset.
2885 * Returns 0 on success, negative error code on failure.
2886 */
2887static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2888{
2889 int i, r;
2890
2891 for (i = 0; i < adev->num_ip_blocks; i++) {
2892 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2893 continue;
2894 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2895 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2896 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2897
2898 r = adev->ip_blocks[i].version->funcs->resume(adev);
2899 if (r) {
2900 DRM_ERROR("resume of IP block <%s> failed %d\n",__drm_err("resume of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2901 adev->ip_blocks[i].version->funcs->name, r)__drm_err("resume of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2902 return r;
2903 }
2904 adev->ip_blocks[i].status.hw = true1;
2905 }
2906 }
2907
2908 return 0;
2909}
2910
2911/**
2912 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2913 *
2914 * @adev: amdgpu_device pointer
2915 *
2916 * First resume function for hardware IPs. The list of all the hardware
2917 * IPs that make up the asic is walked and the resume callbacks are run for
2918 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2919 * functional state after a suspend and updates the software state as
2920 * necessary. This function is also used for restoring the GPU after a GPU
2921 * reset.
2922 * Returns 0 on success, negative error code on failure.
2923 */
2924static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2925{
2926 int i, r;
2927
2928 for (i = 0; i < adev->num_ip_blocks; i++) {
2929 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2930 continue;
2931 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2932 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2933 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2934 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2935 continue;
2936 r = adev->ip_blocks[i].version->funcs->resume(adev);
2937 if (r) {
2938 DRM_ERROR("resume of IP block <%s> failed %d\n",__drm_err("resume of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r)
2939 adev->ip_blocks[i].version->funcs->name, r)__drm_err("resume of IP block <%s> failed %d\n", adev->
ip_blocks[i].version->funcs->name, r);
2940 return r;
2941 }
2942 adev->ip_blocks[i].status.hw = true1;
2943 }
2944
2945 return 0;
2946}
2947
2948/**
2949 * amdgpu_device_ip_resume - run resume for hardware IPs
2950 *
2951 * @adev: amdgpu_device pointer
2952 *
2953 * Main resume function for hardware IPs. The hardware IPs
2954 * are split into two resume functions because they are
2955 * are also used in in recovering from a GPU reset and some additional
2956 * steps need to be take between them. In this case (S3/S4) they are
2957 * run sequentially.
2958 * Returns 0 on success, negative error code on failure.
2959 */
2960static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2961{
2962 int r;
2963
2964 r = amdgpu_amdkfd_resume_iommu(adev);
2965 if (r)
2966 return r;
2967
2968 r = amdgpu_device_ip_resume_phase1(adev);
2969 if (r)
2970 return r;
2971
2972 r = amdgpu_device_fw_loading(adev);
2973 if (r)
2974 return r;
2975
2976 r = amdgpu_device_ip_resume_phase2(adev);
2977
2978 return r;
2979}
2980
2981/**
2982 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2983 *
2984 * @adev: amdgpu_device pointer
2985 *
2986 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2987 */
2988static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2989{
2990 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
2991 if (adev->is_atom_fw) {
2992 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2993 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS(1 << 0);
2994 } else {
2995 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2996 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS(1 << 0);
2997 }
2998
2999 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS(1 << 0)))
3000 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3001 }
3002}
3003
3004/**
3005 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3006 *
3007 * @asic_type: AMD asic type
3008 *
3009 * Check if there is DC (new modesetting infrastructre) support for an asic.
3010 * returns true if DC has support, false if not.
3011 */
3012bool_Bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3013{
3014 switch (asic_type) {
3015#if defined(CONFIG_DRM_AMD_DC1)
3016#if defined(CONFIG_DRM_AMD_DC_SI)
3017 case CHIP_TAHITI:
3018 case CHIP_PITCAIRN:
3019 case CHIP_VERDE:
3020 case CHIP_OLAND:
3021#endif
3022 case CHIP_BONAIRE:
3023 case CHIP_KAVERI:
3024 case CHIP_KABINI:
3025 case CHIP_MULLINS:
3026 /*
3027 * We have systems in the wild with these ASICs that require
3028 * LVDS and VGA support which is not supported with DC.
3029 *
3030 * Fallback to the non-DC driver here by default so as not to
3031 * cause regressions.
3032 */
3033 return amdgpu_dc > 0;
3034 case CHIP_HAWAII:
3035 case CHIP_CARRIZO:
3036 case CHIP_STONEY:
3037 case CHIP_POLARIS10:
3038 case CHIP_POLARIS11:
3039 case CHIP_POLARIS12:
3040 case CHIP_VEGAM:
3041 case CHIP_TONGA:
3042 case CHIP_FIJI:
3043 case CHIP_VEGA10:
3044 case CHIP_VEGA12:
3045 case CHIP_VEGA20:
3046#if defined(CONFIG_DRM_AMD_DC_DCN1)
3047 case CHIP_RAVEN:
3048 case CHIP_NAVI10:
3049 case CHIP_NAVI14:
3050 case CHIP_NAVI12:
3051 case CHIP_RENOIR:
3052#endif
3053#if defined(CONFIG_DRM_AMD_DC_DCN3_01)
3054 case CHIP_SIENNA_CICHLID:
3055 case CHIP_NAVY_FLOUNDER:
3056#endif
3057 return amdgpu_dc != 0;
3058#endif
3059 default:
3060 if (amdgpu_dc > 0)
3061 DRM_INFO_ONCE("Display Core has been requested via kernel parameter "({ static int __warned; if (!__warned) { printk("\0016" "[" "drm"
"] " "Display Core has been requested via kernel parameter "
"but isn't supported by ASIC, ignoring\n"); __warned = 1; } }
)
3062 "but isn't supported by ASIC, ignoring\n")({ static int __warned; if (!__warned) { printk("\0016" "[" "drm"
"] " "Display Core has been requested via kernel parameter "
"but isn't supported by ASIC, ignoring\n"); __warned = 1; } }
);
3063 return false0;
3064 }
3065}
3066
3067/**
3068 * amdgpu_device_has_dc_support - check if dc is supported
3069 *
3070 * @adev: amdgpu_device pointer
3071 *
3072 * Returns true for supported, false for not supported
3073 */
3074bool_Bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3075{
3076 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) || adev->enable_virtual_display)
3077 return false0;
3078
3079 return amdgpu_device_asic_has_dc_support(adev->asic_type);
3080}
3081
3082
3083static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3084{
3085 struct amdgpu_device *adev =
3086 container_of(__work, struct amdgpu_device, xgmi_reset_work)({ const __typeof( ((struct amdgpu_device *)0)->xgmi_reset_work
) *__mptr = (__work); (struct amdgpu_device *)( (char *)__mptr
- __builtin_offsetof(struct amdgpu_device, xgmi_reset_work) )
;});
3087 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3088
3089 /* It's a bug to not have a hive within this function */
3090 if (WARN_ON(!hive)({ int __ret = !!(!hive); if (__ret) printf("WARNING %s failed at %s:%d\n"
, "!hive", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_device.c"
, 3090); __builtin_expect(!!(__ret), 0); }))
3091 return;
3092
3093 /*
3094 * Use task barrier to synchronize all xgmi reset works across the
3095 * hive. task_barrier_enter and task_barrier_exit will block
3096 * until all the threads running the xgmi reset works reach
3097 * those points. task_barrier_full will do both blocks.
3098 */
3099 if (amdgpu_asic_reset_method(adev)(adev)->asic_funcs->reset_method((adev)) == AMD_RESET_METHOD_BACO) {
3100
3101 task_barrier_enter(&hive->tb);
3102 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3103
3104 if (adev->asic_reset_res)
3105 goto fail;
3106
3107 task_barrier_exit(&hive->tb);
3108 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3109
3110 if (adev->asic_reset_res)
3111 goto fail;
3112
3113 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
3114 adev->mmhub.funcs->reset_ras_error_count(adev);
3115 } else {
3116
3117 task_barrier_full(&hive->tb);
3118 adev->asic_reset_res = amdgpu_asic_reset(adev)(adev)->asic_funcs->reset((adev));
3119 }
3120
3121fail:
3122 if (adev->asic_reset_res)
3123 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",printk("\0014" "[" "drm" "] " "ASIC reset failed with error, %d for drm dev, %s"
, adev->asic_reset_res, adev_to_drm(adev)->unique)
3124 adev->asic_reset_res, adev_to_drm(adev)->unique)printk("\0014" "[" "drm" "] " "ASIC reset failed with error, %d for drm dev, %s"
, adev->asic_reset_res, adev_to_drm(adev)->unique);
3125 amdgpu_put_xgmi_hive(hive);
3126}
3127
3128static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3129{
3130 char *input = amdgpu_lockup_timeout;
3131 char *timeout_setting = NULL((void *)0);
3132 int index = 0;
3133 long timeout;
3134 int ret = 0;
3135
3136 /*
3137 * By default timeout for non compute jobs is 10000.
3138 * And there is no timeout enforced on compute jobs.
3139 * In SR-IOV or passthrough mode, timeout for compute
3140 * jobs are 60000 by default.
3141 */
3142 adev->gfx_timeout = msecs_to_jiffies(10000)(((uint64_t)(10000)) * hz / 1000);
3143 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3144 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) || amdgpu_passthrough(adev)((adev)->virt.caps & (1 << 3)))
3145 adev->compute_timeout = msecs_to_jiffies(60000)(((uint64_t)(60000)) * hz / 1000);
3146 else
3147 adev->compute_timeout = MAX_SCHEDULE_TIMEOUT(0x7fffffff);
3148
3149#ifdef notyet
3150 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH256)) {
3151 while ((timeout_setting = strsep(&input, ",")) &&
3152 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH256)) {
3153 ret = kstrtol(timeout_setting, 0, &timeout);
3154 if (ret)
3155 return ret;
3156
3157 if (timeout == 0) {
3158 index++;
3159 continue;
3160 } else if (timeout < 0) {
3161 timeout = MAX_SCHEDULE_TIMEOUT(0x7fffffff);
3162 } else {
3163 timeout = msecs_to_jiffies(timeout)(((uint64_t)(timeout)) * hz / 1000);
3164 }
3165
3166 switch (index++) {
3167 case 0:
3168 adev->gfx_timeout = timeout;
3169 break;
3170 case 1:
3171 adev->compute_timeout = timeout;
3172 break;
3173 case 2:
3174 adev->sdma_timeout = timeout;
3175 break;
3176 case 3:
3177 adev->video_timeout = timeout;
3178 break;
3179 default:
3180 break;
3181 }
3182 }
3183 /*
3184 * There is only one value specified and
3185 * it should apply to all non-compute jobs.
3186 */
3187 if (index == 1) {
3188 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3189 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) || amdgpu_passthrough(adev)((adev)->virt.caps & (1 << 3)))
3190 adev->compute_timeout = adev->gfx_timeout;
3191 }
3192 }
3193#endif
3194
3195 return ret;
3196}
3197
3198static const struct attribute *amdgpu_dev_attributes[] = {
3199 &dev_attr_product_name.attr,
3200 &dev_attr_product_number.attr,
3201 &dev_attr_serial_number.attr,
3202 &dev_attr_pcie_replay_count.attr,
3203 NULL((void *)0)
3204};
3205
3206
3207/**
3208 * amdgpu_device_init - initialize the driver
3209 *
3210 * @adev: amdgpu_device pointer
3211 * @flags: driver flags
3212 *
3213 * Initializes the driver info and hw (all asics).
3214 * Returns 0 for success or an error on failure.
3215 * Called at driver startup.
3216 */
3217int amdgpu_device_init(struct amdgpu_device *adev,
3218 uint32_t flags)
3219{
3220 struct drm_device *ddev = adev_to_drm(adev);
1
Calling 'adev_to_drm'
→
4
←
Returning from 'adev_to_drm'
→
3221 struct pci_dev *pdev = adev->pdev;
3222 int r, i;
3223 bool_Bool boco = false0;
3224 u32 max_MBps;
3225
3226 adev->shutdown = false0;
3227 adev->flags = flags;
3228
3229 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
5
←
Assuming 'amdgpu_force_asic_type' is < 0
→
3230 adev->asic_type = amdgpu_force_asic_type;
3231 else
3232 adev->asic_type = flags & AMD_ASIC_MASK;
3233
3234 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT100000;
3235 if (amdgpu_emu_mode == 1)
6
←
Assuming 'amdgpu_emu_mode' is not equal to 1
→
7
←
Taking false branch
→
3236 adev->usec_timeout *= 10;
3237 adev->gmc.gart_size = 512 * 1024 * 1024;
3238 adev->accel_working = false0;
3239 adev->num_rings = 0;
3240 adev->mman.buffer_funcs = NULL((void *)0);
3241 adev->mman.buffer_funcs_ring = NULL((void *)0);
3242 adev->vm_manager.vm_pte_funcs = NULL((void *)0);
3243 adev->vm_manager.vm_pte_num_scheds = 0;
3244 adev->gmc.gmc_funcs = NULL((void *)0);
3245 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS28);
3246 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES128);
3247
3248 adev->smc_rreg = &amdgpu_invalid_rreg;
3249 adev->smc_wreg = &amdgpu_invalid_wreg;
3250 adev->pcie_rreg = &amdgpu_invalid_rreg;
3251 adev->pcie_wreg = &amdgpu_invalid_wreg;
3252 adev->pciep_rreg = &amdgpu_invalid_rreg;
3253 adev->pciep_wreg = &amdgpu_invalid_wreg;
3254 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3255 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3256 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3257 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3258 adev->didt_rreg = &amdgpu_invalid_rreg;
3259 adev->didt_wreg = &amdgpu_invalid_wreg;
3260 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3261 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3262 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3263 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3264
3265 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",printk("\0016" "[" "drm" "] " "initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n"
, amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev
->device, pdev->subsystem_vendor, pdev->subsystem_device
, pdev->revision)
3266 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,printk("\0016" "[" "drm" "] " "initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n"
, amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev
->device, pdev->subsystem_vendor, pdev->subsystem_device
, pdev->revision)
3267 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision)printk("\0016" "[" "drm" "] " "initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n"
, amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev
->device, pdev->subsystem_vendor, pdev->subsystem_device
, pdev->revision);
3268
3269 /* mutex initialization are all done here so we
3270 * can recall function without having locking issues */
3271 atomic_set(&adev->irq.ih.lock, 0)({ typeof(*(&adev->irq.ih.lock)) __tmp = ((0)); *(volatile
typeof(*(&adev->irq.ih.lock)) *)&(*(&adev->
irq.ih.lock)) = __tmp; __tmp; });
3272 rw_init(&adev->firmware.mutex, "agfw")_rw_init_flags(&adev->firmware.mutex, "agfw", 0, ((void
*)0));
3273 rw_init(&adev->pm.mutex, "agpm")_rw_init_flags(&adev->pm.mutex, "agpm", 0, ((void *)0)
);
3274 rw_init(&adev->gfx.gpu_clock_mutex, "gfxclk")_rw_init_flags(&adev->gfx.gpu_clock_mutex, "gfxclk", 0
, ((void *)0));
3275 rw_init(&adev->srbm_mutex, "srbm")_rw_init_flags(&adev->srbm_mutex, "srbm", 0, ((void *)
0));
3276 rw_init(&adev->gfx.pipe_reserve_mutex, "pipers")_rw_init_flags(&adev->gfx.pipe_reserve_mutex, "pipers"
, 0, ((void *)0));
3277 rw_init(&adev->gfx.gfx_off_mutex, "gfxoff")_rw_init_flags(&adev->gfx.gfx_off_mutex, "gfxoff", 0, (
(void *)0));
3278 rw_init(&adev->grbm_idx_mutex, "grbmidx")_rw_init_flags(&adev->grbm_idx_mutex, "grbmidx", 0, ((
void *)0));
3279 rw_init(&adev->mn_lock, "agpumn")_rw_init_flags(&adev->mn_lock, "agpumn", 0, ((void *)0
));
3280 rw_init(&adev->virt.vf_errors.lock, "vferr")_rw_init_flags(&adev->virt.vf_errors.lock, "vferr", 0,
((void *)0));
3281 hash_init(adev->mn_hash)__hash_init(adev->mn_hash, (sizeof((adev->mn_hash)) / sizeof
((adev->mn_hash)[0])));
3282 atomic_set(&adev->in_gpu_reset, 0)({ typeof(*(&adev->in_gpu_reset)) __tmp = ((0)); *(volatile
typeof(*(&adev->in_gpu_reset)) *)&(*(&adev->
in_gpu_reset)) = __tmp; __tmp; });
3283 rw_init(&adev->reset_sem, "amrs")_rw_init_flags(&adev->reset_sem, "amrs", 0, ((void *)0
));
3284 rw_init(&adev->psp.mutex, "agpsp")_rw_init_flags(&adev->psp.mutex, "agpsp", 0, ((void *)
0));
3285 rw_init(&adev->notifier_lock, "agnf")_rw_init_flags(&adev->notifier_lock, "agnf", 0, ((void
*)0));
3286
3287 r = amdgpu_device_check_arguments(adev);
8
←
Calling 'amdgpu_device_check_arguments'
→
32
←
Returning from 'amdgpu_device_check_arguments'
→
3288 if (r
32.1
'r' is 0
32.1
'r' is 0
32.1
'r' is 0
)
33
←
Taking false branch
→
3289 return r;
3290
3291 mtx_init(&adev->mmio_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
mmio_idx_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
34
←
'?' condition is false
→
35
←
Loop condition is false. Exiting loop
→
3292 mtx_init(&adev->smc_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
smc_idx_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
36
←
'?' condition is false
→
37
←
Loop condition is false. Exiting loop
→
3293 mtx_init(&adev->pcie_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
pcie_idx_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
38
←
'?' condition is false
→
39
←
Loop condition is false. Exiting loop
→
3294 mtx_init(&adev->uvd_ctx_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
uvd_ctx_idx_lock), ((((0x9)) > 0x0 && ((0x9)) <
0x9) ? 0x9 : ((0x9)))); } while (0);
40
←
'?' condition is false
→
41
←
Loop condition is false. Exiting loop
→
3295 mtx_init(&adev->didt_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
didt_idx_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
42
←
'?' condition is false
→
43
←
Loop condition is false. Exiting loop
→
3296 mtx_init(&adev->gc_cac_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
gc_cac_idx_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
44
←
'?' condition is false
→
45
←
Loop condition is false. Exiting loop
→
3297 mtx_init(&adev->se_cac_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
se_cac_idx_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
46
←
'?' condition is false
→
47
←
Loop condition is false. Exiting loop
→
3298 mtx_init(&adev->audio_endpt_idx_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
audio_endpt_idx_lock), ((((0x9)) > 0x0 && ((0x9)) <
0x9) ? 0x9 : ((0x9)))); } while (0);
48
←
'?' condition is false
→
49
←
Loop condition is false. Exiting loop
→
3299 mtx_init(&adev->mm_stats.lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
mm_stats.lock), ((((0x0)) > 0x0 && ((0x0)) < 0x9
) ? 0x9 : ((0x0)))); } while (0);
50
←
Loop condition is false. Exiting loop
→
3300
3301 INIT_LIST_HEAD(&adev->shadow_list);
3302 rw_init(&adev->shadow_list_lock, "sdwlst")_rw_init_flags(&adev->shadow_list_lock, "sdwlst", 0, (
(void *)0));
3303
3304 INIT_DELAYED_WORK(&adev->delayed_init_work,
3305 amdgpu_device_delayed_init_work_handler);
3306 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3307 amdgpu_device_delay_enable_gfx_off);
3308
3309 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3310
3311 adev->gfx.gfx_off_req_count = 1;
3312 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
51
←
Assuming the condition is false
→
3313
3314 atomic_set(&adev->throttling_logging_enabled, 1)({ typeof(*(&adev->throttling_logging_enabled)) __tmp =
((1)); *(volatile typeof(*(&adev->throttling_logging_enabled
)) *)&(*(&adev->throttling_logging_enabled)) = __tmp
; __tmp; });
3315 /*
3316 * If throttling continues, logging will be performed every minute
3317 * to avoid log flooding. "-1" is subtracted since the thermal
3318 * throttling interrupt comes every second. Thus, the total logging
3319 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3320 * for throttling interrupt) = 60 seconds.
3321 */
3322 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3323 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3324
3325#ifdef __linux__
3326 /* Registers mapping */
3327 /* TODO: block userspace mapping of io register */
3328 if (adev->asic_type >= CHIP_BONAIRE) {
3329 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3330 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3331 } else {
3332 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3333 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3334 }
3335
3336 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3337 if (adev->rmmio == NULL((void *)0)) {
3338 return -ENOMEM12;
3339 }
3340#endif
3341 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base)printk("\0016" "[" "drm" "] " "register mmio base: 0x%08X\n",
(uint32_t)adev->rmmio_base);
3342 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size)printk("\0016" "[" "drm" "] " "register mmio size: %u\n", (unsigned
)adev->rmmio_size);
3343
3344 /* io port mapping */
3345#ifdef __linux__
3346 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
3347 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
3348 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
3349 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
3350 break;
3351 }
3352 }
3353 if (adev->rio_mem == NULL((void *)0))
3354 DRM_INFO("PCI I/O BAR is not found.\n")printk("\0016" "[" "drm" "] " "PCI I/O BAR is not found.\n");
3355#endif
3356
3357 /* enable PCIE atomic ops */
3358#ifdef notyet
3359 r = pci_enable_atomic_ops_to_root(adev->pdev,
3360 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3361 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3362 if (r) {
3363 adev->have_atomics_support = false0;
3364 DRM_INFO("PCIE atomic ops is not supported\n")printk("\0016" "[" "drm" "] " "PCIE atomic ops is not supported\n"
);
3365 } else {
3366 adev->have_atomics_support = true1;
3367 }
3368#else
3369 adev->have_atomics_support = false0;
3370#endif
3371
3372 amdgpu_device_get_pcie_info(adev);
52
←
Calling 'amdgpu_device_get_pcie_info'
→
3373
3374 if (amdgpu_mcbp)
3375 DRM_INFO("MCBP is enabled\n")printk("\0016" "[" "drm" "] " "MCBP is enabled\n");
3376
3377 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3378 adev->enable_mes = true1;
3379
3380 /* detect hw virtualization here */
3381 amdgpu_detect_virtualization(adev);
3382
3383 r = amdgpu_device_get_job_timeout_settings(adev);
3384 if (r) {
3385 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n")printf("drm:pid%d:%s *ERROR* " "invalid lockup_timeout parameter syntax\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3386 return r;
3387 }
3388
3389 /* early init functions */
3390 r = amdgpu_device_ip_early_init(adev);
3391 if (r)
3392 return r;
3393
3394 /* doorbell bar mapping and doorbell index init*/
3395 amdgpu_device_doorbell_init(adev);
3396
3397 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3398 /* this will fail for cards that aren't VGA class devices, just
3399 * ignore it */
3400#ifdef notyet
3401 vga_client_register(adev->pdev, adev, NULL((void *)0), amdgpu_device_vga_set_decode);
3402#endif
3403
3404 if (amdgpu_device_supports_boco(ddev))
3405 boco = true1;
3406 if (amdgpu_has_atpx() &&
3407 (amdgpu_is_atpx_hybrid() ||
3408 amdgpu_has_atpx_dgpu_power_cntl()) &&
3409 !pci_is_thunderbolt_attached(adev->pdev)0)
3410 vga_switcheroo_register_client(adev->pdev,0
3411 &amdgpu_switcheroo_ops, boco)0;
3412 if (boco)
3413 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3414
3415 if (amdgpu_emu_mode == 1) {
3416 /* post the asic on emulation mode */
3417 emu_soc_asic_init(adev);
3418 goto fence_driver_init;
3419 }
3420
3421 /* detect if we are with an SRIOV vbios */
3422 amdgpu_device_detect_sriov_bios(adev);
3423
3424 /* check if we need to reset the asic
3425 * E.g., driver was not cleanly unloaded previously, etc.
3426 */
3427 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) && amdgpu_asic_need_reset_on_init(adev)(adev)->asic_funcs->need_reset_on_init((adev))) {
3428 r = amdgpu_asic_reset(adev)(adev)->asic_funcs->reset((adev));
3429 if (r) {
3430 dev_err(adev->dev, "asic reset on init failed\n")printf("drm:pid%d:%s *ERROR* " "asic reset on init failed\n",
({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3431 goto failed;
3432 }
3433 }
3434
3435 pci_enable_pcie_error_reporting(adev->ddev.pdev);
3436
3437 /* Post card if necessary */
3438 if (amdgpu_device_need_post(adev)) {
3439 if (!adev->bios) {
3440 dev_err(adev->dev, "no vBIOS found\n")printf("drm:pid%d:%s *ERROR* " "no vBIOS found\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__);
3441 r = -EINVAL22;
3442 goto failed;
3443 }
3444 DRM_INFO("GPU posting now...\n")printk("\0016" "[" "drm" "] " "GPU posting now...\n");
3445 r = amdgpu_device_asic_init(adev);
3446 if (r) {
3447 dev_err(adev->dev, "gpu post error!\n")printf("drm:pid%d:%s *ERROR* " "gpu post error!\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__);
3448 goto failed;
3449 }
3450 }
3451
3452 if (adev->is_atom_fw) {
3453 /* Initialize clocks */
3454 r = amdgpu_atomfirmware_get_clock_info(adev);
3455 if (r) {
3456 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_atomfirmware_get_clock_info failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3457 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3458 goto failed;
3459 }
3460 } else {
3461 /* Initialize clocks */
3462 r = amdgpu_atombios_get_clock_info(adev);
3463 if (r) {
3464 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_atombios_get_clock_info failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3465 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3466 goto failed;
3467 }
3468 /* init i2c buses */
3469 if (!amdgpu_device_has_dc_support(adev))
3470 amdgpu_atombios_i2c_init(adev);
3471 }
3472
3473fence_driver_init:
3474 /* Fence driver */
3475 r = amdgpu_fence_driver_init(adev);
3476 if (r) {
3477 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_fence_driver_init failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3478 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3479 goto failed;
3480 }
3481
3482 /* init the mode config */
3483 drm_mode_config_init(adev_to_drm(adev));
3484
3485 r = amdgpu_device_ip_init(adev);
3486 if (r) {
3487 /* failed in exclusive mode due to timeout */
3488 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)) &&
3489 !amdgpu_sriov_runtime(adev)((adev)->virt.caps & (1 << 4)) &&
3490 amdgpu_virt_mmio_blocked(adev) &&
3491 !amdgpu_virt_wait_reset(adev)) {
3492 dev_err(adev->dev, "VF exclusive mode timeout\n")printf("drm:pid%d:%s *ERROR* " "VF exclusive mode timeout\n",
({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3493 /* Don't send request since VF is inactive. */
3494 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME(1 << 4);
3495 adev->virt.ops = NULL((void *)0);
3496 r = -EAGAIN35;
3497 goto failed;
3498 }
3499 dev_err(adev->dev, "amdgpu_device_ip_init failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_device_ip_init failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3500 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3501 goto failed;
3502 }
3503
3504 dev_info(adev->dev,do { } while(0)
3505 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",do { } while(0)
3506 adev->gfx.config.max_shader_engines,do { } while(0)
3507 adev->gfx.config.max_sh_per_se,do { } while(0)
3508 adev->gfx.config.max_cu_per_sh,do { } while(0)
3509 adev->gfx.cu_info.number)do { } while(0);
3510
3511#ifdef __OpenBSD__1
3512{
3513 const char *chip_name;
3514
3515 switch (adev->asic_type) {
3516 case CHIP_RAVEN:
3517 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
3518 chip_name = "RAVEN2";
3519 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
3520 chip_name = "PICASSO";
3521 else
3522 chip_name = "RAVEN";
3523 break;
3524 case CHIP_RENOIR:
3525 if (adev->apu_flags & AMD_APU_IS_RENOIR)
3526 chip_name = "RENOIR";
3527 else
3528 chip_name = "GREEN_SARDINE";
3529 break;
3530 default:
3531 chip_name = amdgpu_asic_name[adev->asic_type];
3532 }
3533 printf("%s: %s %d CU rev 0x%02x\n", adev->self.dv_xname,
3534 chip_name, adev->gfx.cu_info.number, adev->rev_id);
3535}
3536#endif
3537
3538 adev->accel_working = true1;
3539
3540 amdgpu_vm_check_compute_bug(adev);
3541
3542 /* Initialize the buffer migration limit. */
3543 if (amdgpu_moverate >= 0)
3544 max_MBps = amdgpu_moverate;
3545 else
3546 max_MBps = 8; /* Allow 8 MB/s. */
3547 /* Get a log2 for easy divisions. */
3548 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps))((sizeof((((1u)>(max_MBps))?(1u):(max_MBps))) <= 4) ? (
fls((((1u)>(max_MBps))?(1u):(max_MBps))) - 1) : (flsl((((1u
)>(max_MBps))?(1u):(max_MBps))) - 1));
3549
3550 amdgpu_fbdev_init(adev);
3551
3552 r = amdgpu_pm_sysfs_init(adev);
3553 if (r) {
3554 adev->pm_sysfs_en = false0;
3555 DRM_ERROR("registering pm debugfs failed (%d).\n", r)__drm_err("registering pm debugfs failed (%d).\n", r);
3556 } else
3557 adev->pm_sysfs_en = true1;
3558
3559 r = amdgpu_ucode_sysfs_init(adev);
3560 if (r) {
3561 adev->ucode_sysfs_en = false0;
3562 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r)__drm_err("Creating firmware sysfs failed (%d).\n", r);
3563 } else
3564 adev->ucode_sysfs_en = true1;
3565
3566 if ((amdgpu_testing & 1)) {
3567 if (adev->accel_working)
3568 amdgpu_test_moves(adev);
3569 else
3570 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n")printk("\0016" "[" "drm" "] " "amdgpu: acceleration disabled, skipping move tests\n"
);
3571 }
3572 if (amdgpu_benchmarking) {
3573 if (adev->accel_working)
3574 amdgpu_benchmark(adev, amdgpu_benchmarking);
3575 else
3576 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n")printk("\0016" "[" "drm" "] " "amdgpu: acceleration disabled, skipping benchmarks\n"
);
3577 }
3578
3579 /*
3580 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3581 * Otherwise the mgpu fan boost feature will be skipped due to the
3582 * gpu instance is counted less.
3583 */
3584 amdgpu_register_gpu_instance(adev);
3585
3586 /* enable clockgating, etc. after ib tests, etc. since some blocks require
3587 * explicit gating rather than handling it automatically.
3588 */
3589 r = amdgpu_device_ip_late_init(adev);
3590 if (r) {
3591 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_device_ip_late_init failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3592 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3593 goto failed;
3594 }
3595
3596 /* must succeed. */
3597 amdgpu_ras_resume(adev);
3598
3599 queue_delayed_work(system_wq, &adev->delayed_init_work,
3600 msecs_to_jiffies(AMDGPU_RESUME_MS)(((uint64_t)(2000)) * hz / 1000));
3601
3602 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
3603 flush_delayed_work(&adev->delayed_init_work);
3604
3605 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes)0;
3606 if (r)
3607 dev_err(adev->dev, "Could not create amdgpu device attr\n")printf("drm:pid%d:%s *ERROR* " "Could not create amdgpu device attr\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3608
3609 if (IS_ENABLED(CONFIG_PERF_EVENTS)0)
3610 r = amdgpu_pmu_init(adev);
3611 if (r)
3612 dev_err(adev->dev, "amdgpu_pmu_init failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu_pmu_init failed\n", ({
struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3613
3614 /* Have stored pci confspace at hand for restore in sudden PCI error */
3615 if (amdgpu_device_cache_pci_state(adev->pdev))
3616 pci_restore_state(pdev);
3617
3618 return 0;
3619
3620failed:
3621 amdgpu_vf_error_trans_all(adev);
3622 if (boco)
3623 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3624
3625 return r;
3626}
3627
3628/**
3629 * amdgpu_device_fini - tear down the driver
3630 *
3631 * @adev: amdgpu_device pointer
3632 *
3633 * Tear down the driver info (all asics).
3634 * Called at driver shutdown.
3635 */
3636void amdgpu_device_fini(struct amdgpu_device *adev)
3637{
3638 dev_info(adev->dev, "amdgpu: finishing device.\n")do { } while(0);
3639 flush_delayed_work(&adev->delayed_init_work);
3640 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3641 adev->shutdown = true1;
3642
3643 kfree(adev->pci_state);
3644
3645 /* make sure IB test finished before entering exclusive mode
3646 * to avoid preemption on IB test
3647 * */
3648 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
3649 amdgpu_virt_request_full_gpu(adev, false0);
3650 amdgpu_virt_fini_data_exchange(adev);
3651 }
3652
3653 /* disable all interrupts */
3654 amdgpu_irq_disable_all(adev);
3655 if (adev->mode_info.mode_config_initialized){
3656 if (!amdgpu_device_has_dc_support(adev))
3657 drm_helper_force_disable_all(adev_to_drm(adev));
3658 else
3659 drm_atomic_helper_shutdown(adev_to_drm(adev));
3660 }
3661 amdgpu_fence_driver_fini(adev);
3662 if (adev->pm_sysfs_en)
3663 amdgpu_pm_sysfs_fini(adev);
3664 amdgpu_fbdev_fini(adev);
3665 amdgpu_device_ip_fini(adev);
3666 release_firmware(adev->firmware.gpu_info_fw);
3667 adev->firmware.gpu_info_fw = NULL((void *)0);
3668 adev->accel_working = false0;
3669 /* free i2c buses */
3670 if (!amdgpu_device_has_dc_support(adev))
3671 amdgpu_i2c_fini(adev);
3672
3673 if (amdgpu_emu_mode != 1)
3674 amdgpu_atombios_fini(adev);
3675
3676 kfree(adev->bios);
3677 adev->bios = NULL((void *)0);
3678 if (amdgpu_has_atpx() &&
3679 (amdgpu_is_atpx_hybrid() ||
3680 amdgpu_has_atpx_dgpu_power_cntl()) &&
3681 !pci_is_thunderbolt_attached(adev->pdev)0)
3682 vga_switcheroo_unregister_client(adev->pdev);
3683 if (amdgpu_device_supports_boco(adev_to_drm(adev)))
3684 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3685 vga_client_register(adev->pdev, NULL((void *)0), NULL((void *)0), NULL((void *)0));
3686#ifdef __linux__
3687 if (adev->rio_mem)
3688 pci_iounmap(adev->pdev, adev->rio_mem);
3689 adev->rio_mem = NULL((void *)0);
3690 iounmap(adev->rmmio);
3691#else
3692 if (adev->rio_mem_size > 0)
3693 bus_space_unmap(adev->rio_mem_bst, adev->rio_mem_bsh,
3694 adev->rio_mem_size);
3695 adev->rio_mem_size = 0;
3696
3697 if (adev->rmmio_size > 0)
3698 bus_space_unmap(adev->rmmio_bst, adev->rmmio_bsh,
3699 adev->rmmio_size);
3700 adev->rmmio_size = 0;
3701#endif
3702 adev->rmmio = NULL((void *)0);
3703 amdgpu_device_doorbell_fini(adev);
3704
3705 if (adev->ucode_sysfs_en)
3706 amdgpu_ucode_sysfs_fini(adev);
3707
3708 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3709 if (IS_ENABLED(CONFIG_PERF_EVENTS)0)
3710 amdgpu_pmu_fini(adev);
3711 if (adev->mman.discovery_bin)
3712 amdgpu_discovery_fini(adev);
3713}
3714
3715
3716/*
3717 * Suspend & resume.
3718 */
3719/**
3720 * amdgpu_device_suspend - initiate device suspend
3721 *
3722 * @dev: drm dev pointer
3723 * @fbcon : notify the fbdev of suspend
3724 *
3725 * Puts the hw in the suspend state (all asics).
3726 * Returns 0 for success or an error on failure.
3727 * Called at driver suspend.
3728 */
3729int amdgpu_device_suspend(struct drm_device *dev, bool_Bool fbcon)
3730{
3731 struct amdgpu_device *adev;
3732 struct drm_crtc *crtc;
3733 struct drm_connector *connector;
3734 struct drm_connector_list_iter iter;
3735 int r;
3736
3737 adev = drm_to_adev(dev);
3738 if (adev->shutdown)
3739 return 0;
3740
3741#ifdef notyet
3742 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3743 return 0;
3744#endif
3745
3746 adev->in_suspend = true1;
3747 drm_kms_helper_poll_disable(dev);
3748
3749 if (fbcon)
3750 amdgpu_fbdev_set_suspend(adev, 1);
3751
3752 cancel_delayed_work_sync(&adev->delayed_init_work);
3753
3754 if (!amdgpu_device_has_dc_support(adev)) {
3755 /* turn off display hw */
3756 drm_modeset_lock_all(dev);
3757 drm_connector_list_iter_begin(dev, &iter);
3758 drm_for_each_connector_iter(connector, &iter)while ((connector = drm_connector_list_iter_next(&iter)))
3759 drm_helper_connector_dpms(connector,
3760 DRM_MODE_DPMS_OFF3);
3761 drm_connector_list_iter_end(&iter);
3762 drm_modeset_unlock_all(dev);
3763 /* unpin the front buffers and cursors */
3764 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)for (crtc = ({ const __typeof( ((__typeof(*crtc) *)0)->head
) *__mptr = ((&dev->mode_config.crtc_list)->next);
(__typeof(*crtc) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*crtc), head) );}); &crtc->head != (&dev->mode_config
.crtc_list); crtc = ({ const __typeof( ((__typeof(*crtc) *)0)
->head ) *__mptr = (crtc->head.next); (__typeof(*crtc) *
)( (char *)__mptr - __builtin_offsetof(__typeof(*crtc), head)
);})) {
3765 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc)({ const __typeof( ((struct amdgpu_crtc *)0)->base ) *__mptr
= (crtc); (struct amdgpu_crtc *)( (char *)__mptr - __builtin_offsetof
(struct amdgpu_crtc, base) );});
3766 struct drm_framebuffer *fb = crtc->primary->fb;
3767 struct amdgpu_bo *robj;
3768
3769 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3770 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo)({ const __typeof( ((struct amdgpu_bo *)0)->tbo.base ) *__mptr
= ((amdgpu_crtc->cursor_bo)); (struct amdgpu_bo *)( (char
*)__mptr - __builtin_offsetof(struct amdgpu_bo, tbo.base) );
});
3771 r = amdgpu_bo_reserve(aobj, true1);
3772 if (r == 0) {
3773 amdgpu_bo_unpin(aobj);
3774 amdgpu_bo_unreserve(aobj);
3775 }
3776 }
3777
3778 if (fb == NULL((void *)0) || fb->obj[0] == NULL((void *)0)) {
3779 continue;
3780 }
3781 robj = gem_to_amdgpu_bo(fb->obj[0])({ const __typeof( ((struct amdgpu_bo *)0)->tbo.base ) *__mptr
= ((fb->obj[0])); (struct amdgpu_bo *)( (char *)__mptr - __builtin_offsetof
(struct amdgpu_bo, tbo.base) );});
3782 /* don't unpin kernel fb objects */
3783 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
3784 r = amdgpu_bo_reserve(robj, true1);
3785 if (r == 0) {
3786 amdgpu_bo_unpin(robj);
3787 amdgpu_bo_unreserve(robj);
3788 }
3789 }
3790 }
3791 }
3792
3793 amdgpu_ras_suspend(adev);
3794
3795 r = amdgpu_device_ip_suspend_phase1(adev);
3796
3797 amdgpu_amdkfd_suspend(adev, !fbcon);
3798
3799 /* evict vram memory */
3800 amdgpu_bo_evict_vram(adev);
3801
3802 amdgpu_fence_driver_suspend(adev);
3803
3804 r = amdgpu_device_ip_suspend_phase2(adev);
3805
3806 /* evict remaining vram memory
3807 * This second call to evict vram is to evict the gart page table
3808 * using the CPU.
3809 */
3810 amdgpu_bo_evict_vram(adev);
3811
3812 return 0;
3813}
3814
3815/**
3816 * amdgpu_device_resume - initiate device resume
3817 *
3818 * @dev: drm dev pointer
3819 * @fbcon : notify the fbdev of resume
3820 *
3821 * Bring the hw back to operating state (all asics).
3822 * Returns 0 for success or an error on failure.
3823 * Called at driver resume.
3824 */
3825int amdgpu_device_resume(struct drm_device *dev, bool_Bool fbcon)
3826{
3827 struct drm_connector *connector;
3828 struct drm_connector_list_iter iter;
3829 struct amdgpu_device *adev = drm_to_adev(dev);
3830 struct drm_crtc *crtc;
3831 int r = 0;
3832
3833#ifdef notyet
3834 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3835 return 0;
3836#endif
3837
3838 /* post card */
3839 if (amdgpu_device_need_post(adev)) {
3840 r = amdgpu_device_asic_init(adev);
3841 if (r)
3842 dev_err(adev->dev, "amdgpu asic init failed\n")printf("drm:pid%d:%s *ERROR* " "amdgpu asic init failed\n", (
{struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
3843 }
3844
3845 r = amdgpu_device_ip_resume(adev);
3846 if (r) {
3847 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r)printf("drm:pid%d:%s *ERROR* " "amdgpu_device_ip_resume failed (%d).\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
3848 return r;
3849 }
3850 amdgpu_fence_driver_resume(adev);
3851
3852
3853 r = amdgpu_device_ip_late_init(adev);
3854 if (r)
3855 return r;
3856
3857 queue_delayed_work(system_wq, &adev->delayed_init_work,
3858 msecs_to_jiffies(AMDGPU_RESUME_MS)(((uint64_t)(2000)) * hz / 1000));
3859
3860 if (!amdgpu_device_has_dc_support(adev)) {
3861 /* pin cursors */
3862 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)for (crtc = ({ const __typeof( ((__typeof(*crtc) *)0)->head
) *__mptr = ((&dev->mode_config.crtc_list)->next);
(__typeof(*crtc) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*crtc), head) );}); &crtc->head != (&dev->mode_config
.crtc_list); crtc = ({ const __typeof( ((__typeof(*crtc) *)0)
->head ) *__mptr = (crtc->head.next); (__typeof(*crtc) *
)( (char *)__mptr - __builtin_offsetof(__typeof(*crtc), head)
);})) {
3863 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc)({ const __typeof( ((struct amdgpu_crtc *)0)->base ) *__mptr
= (crtc); (struct amdgpu_crtc *)( (char *)__mptr - __builtin_offsetof
(struct amdgpu_crtc, base) );});
3864
3865 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3866 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo)({ const __typeof( ((struct amdgpu_bo *)0)->tbo.base ) *__mptr
= ((amdgpu_crtc->cursor_bo)); (struct amdgpu_bo *)( (char
*)__mptr - __builtin_offsetof(struct amdgpu_bo, tbo.base) );
});
3867 r = amdgpu_bo_reserve(aobj, true1);
3868 if (r == 0) {
3869 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM0x4);
3870 if (r != 0)
3871 dev_err(adev->dev, "Failed to pin cursor BO (%d)\n", r)printf("drm:pid%d:%s *ERROR* " "Failed to pin cursor BO (%d)\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
3872 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
3873 amdgpu_bo_unreserve(aobj);
3874 }
3875 }
3876 }
3877 }
3878 r = amdgpu_amdkfd_resume(adev, !fbcon);
3879 if (r)
3880 return r;
3881
3882 /* Make sure IB tests flushed */
3883 flush_delayed_work(&adev->delayed_init_work);
3884
3885 /* blat the mode back in */
3886 if (fbcon) {
3887 if (!amdgpu_device_has_dc_support(adev)) {
3888 /* pre DCE11 */
3889 drm_helper_resume_force_mode(dev);
3890
3891 /* turn on display hw */
3892 drm_modeset_lock_all(dev);
3893
3894 drm_connector_list_iter_begin(dev, &iter);
3895 drm_for_each_connector_iter(connector, &iter)while ((connector = drm_connector_list_iter_next(&iter)))
3896 drm_helper_connector_dpms(connector,
3897 DRM_MODE_DPMS_ON0);
3898 drm_connector_list_iter_end(&iter);
3899
3900 drm_modeset_unlock_all(dev);
3901 }
3902 amdgpu_fbdev_set_suspend(adev, 0);
3903 }
3904
3905 drm_kms_helper_poll_enable(dev);
3906
3907 amdgpu_ras_resume(adev);
3908
3909 /*
3910 * Most of the connector probing functions try to acquire runtime pm
3911 * refs to ensure that the GPU is powered on when connector polling is
3912 * performed. Since we're calling this from a runtime PM callback,
3913 * trying to acquire rpm refs will cause us to deadlock.
3914 *
3915 * Since we're guaranteed to be holding the rpm lock, it's safe to
3916 * temporarily disable the rpm helpers so this doesn't deadlock us.
3917 */
3918#if defined(CONFIG_PM) && defined(__linux__)
3919 dev->dev->power.disable_depth++;
3920#endif
3921 if (!amdgpu_device_has_dc_support(adev))
3922 drm_helper_hpd_irq_event(dev);
3923 else
3924 drm_kms_helper_hotplug_event(dev);
3925#if defined(CONFIG_PM) && defined(__linux__)
3926 dev->dev->power.disable_depth--;
3927#endif
3928 adev->in_suspend = false0;
3929
3930 return 0;
3931}
3932
3933/**
3934 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3935 *
3936 * @adev: amdgpu_device pointer
3937 *
3938 * The list of all the hardware IPs that make up the asic is walked and
3939 * the check_soft_reset callbacks are run. check_soft_reset determines
3940 * if the asic is still hung or not.
3941 * Returns true if any of the IPs are still in a hung state, false if not.
3942 */
3943static bool_Bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3944{
3945 int i;
3946 bool_Bool asic_hang = false0;
3947
3948 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
3949 return true1;
3950
3951 if (amdgpu_asic_need_full_reset(adev)(adev)->asic_funcs->need_full_reset((adev)))
3952 return true1;
3953
3954 for (i = 0; i < adev->num_ip_blocks; i++) {
3955 if (!adev->ip_blocks[i].status.valid)
3956 continue;
3957 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3958 adev->ip_blocks[i].status.hang =
3959 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3960 if (adev->ip_blocks[i].status.hang) {
3961 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name)do { } while(0);
3962 asic_hang = true1;
3963 }
3964 }
3965 return asic_hang;
3966}
3967
3968/**
3969 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3970 *
3971 * @adev: amdgpu_device pointer
3972 *
3973 * The list of all the hardware IPs that make up the asic is walked and the
3974 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
3975 * handles any IP specific hardware or software state changes that are
3976 * necessary for a soft reset to succeed.
3977 * Returns 0 on success, negative error code on failure.
3978 */
3979static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3980{
3981 int i, r = 0;
3982
3983 for (i = 0; i < adev->num_ip_blocks; i++) {
3984 if (!adev->ip_blocks[i].status.valid)
3985 continue;
3986 if (adev->ip_blocks[i].status.hang &&
3987 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3988 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3989 if (r)
3990 return r;
3991 }
3992 }
3993
3994 return 0;
3995}
3996
3997/**
3998 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3999 *
4000 * @adev: amdgpu_device pointer
4001 *
4002 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
4003 * reset is necessary to recover.
4004 * Returns true if a full asic reset is required, false if not.
4005 */
4006static bool_Bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4007{
4008 int i;
4009
4010 if (amdgpu_asic_need_full_reset(adev)(adev)->asic_funcs->need_full_reset((adev)))
4011 return true1;
4012
4013 for (i = 0; i < adev->num_ip_blocks; i++) {
4014 if (!adev->ip_blocks[i].status.valid)
4015 continue;
4016 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4017 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4018 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4019 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4020 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4021 if (adev->ip_blocks[i].status.hang) {
4022 dev_info(adev->dev, "Some block need full reset!\n")do { } while(0);
4023 return true1;
4024 }
4025 }
4026 }
4027 return false0;
4028}
4029
4030/**
4031 * amdgpu_device_ip_soft_reset - do a soft reset
4032 *
4033 * @adev: amdgpu_device pointer
4034 *
4035 * The list of all the hardware IPs that make up the asic is walked and the
4036 * soft_reset callbacks are run if the block is hung. soft_reset handles any
4037 * IP specific hardware or software state changes that are necessary to soft
4038 * reset the IP.
4039 * Returns 0 on success, negative error code on failure.
4040 */
4041static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4042{
4043 int i, r = 0;
4044
4045 for (i = 0; i < adev->num_ip_blocks; i++) {
4046 if (!adev->ip_blocks[i].status.valid)
4047 continue;
4048 if (adev->ip_blocks[i].status.hang &&
4049 adev->ip_blocks[i].version->funcs->soft_reset) {
4050 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4051 if (r)
4052 return r;
4053 }
4054 }
4055
4056 return 0;
4057}
4058
4059/**
4060 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4061 *
4062 * @adev: amdgpu_device pointer
4063 *
4064 * The list of all the hardware IPs that make up the asic is walked and the
4065 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
4066 * handles any IP specific hardware or software state changes that are
4067 * necessary after the IP has been soft reset.
4068 * Returns 0 on success, negative error code on failure.
4069 */
4070static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4071{
4072 int i, r = 0;
4073
4074 for (i = 0; i < adev->num_ip_blocks; i++) {
4075 if (!adev->ip_blocks[i].status.valid)
4076 continue;
4077 if (adev->ip_blocks[i].status.hang &&
4078 adev->ip_blocks[i].version->funcs->post_soft_reset)
4079 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4080 if (r)
4081 return r;
4082 }
4083
4084 return 0;
4085}
4086
4087/**
4088 * amdgpu_device_recover_vram - Recover some VRAM contents
4089 *
4090 * @adev: amdgpu_device pointer
4091 *
4092 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
4093 * restore things like GPUVM page tables after a GPU reset where
4094 * the contents of VRAM might be lost.
4095 *
4096 * Returns:
4097 * 0 on success, negative error code on failure.
4098 */
4099static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4100{
4101 struct dma_fence *fence = NULL((void *)0), *next = NULL((void *)0);
4102 struct amdgpu_bo *shadow;
4103 long r = 1, tmo;
4104
4105 if (amdgpu_sriov_runtime(adev)((adev)->virt.caps & (1 << 4)))
4106 tmo = msecs_to_jiffies(8000)(((uint64_t)(8000)) * hz / 1000);
4107 else
4108 tmo = msecs_to_jiffies(100)(((uint64_t)(100)) * hz / 1000);
4109
4110 dev_info(adev->dev, "recover vram bo from shadow start\n")do { } while(0);
4111 mutex_lock(&adev->shadow_list_lock)rw_enter_write(&adev->shadow_list_lock);
4112 list_for_each_entry(shadow, &adev->shadow_list, shadow_list)for (shadow = ({ const __typeof( ((__typeof(*shadow) *)0)->
shadow_list ) *__mptr = ((&adev->shadow_list)->next
); (__typeof(*shadow) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*shadow), shadow_list) );}); &shadow->shadow_list
!= (&adev->shadow_list); shadow = ({ const __typeof( (
(__typeof(*shadow) *)0)->shadow_list ) *__mptr = (shadow->
shadow_list.next); (__typeof(*shadow) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*shadow), shadow_list) );})) {
4113
4114 /* No need to recover an evicted BO */
4115 if (shadow->tbo.mem.mem_type != TTM_PL_TT1 ||
4116 shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET0x7fffffffffffffffL ||
4117 shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM2)
4118 continue;
4119
4120 r = amdgpu_bo_restore_shadow(shadow, &next);
4121 if (r)
4122 break;
4123
4124 if (fence) {
4125 tmo = dma_fence_wait_timeout(fence, false0, tmo);
4126 dma_fence_put(fence);
4127 fence = next;
4128 if (tmo == 0) {
4129 r = -ETIMEDOUT60;
4130 break;
4131 } else if (tmo < 0) {
4132 r = tmo;
4133 break;
4134 }
4135 } else {
4136 fence = next;
4137 }
4138 }
4139 mutex_unlock(&adev->shadow_list_lock)rw_exit_write(&adev->shadow_list_lock);
4140
4141 if (fence)
4142 tmo = dma_fence_wait_timeout(fence, false0, tmo);
4143 dma_fence_put(fence);
4144
4145 if (r < 0 || tmo <= 0) {
4146 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo)printf("drm:pid%d:%s *ERROR* " "recover vram bo from shadow failed, r is %ld, tmo is %ld\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r, tmo
);
4147 return -EIO5;
4148 }
4149
4150 dev_info(adev->dev, "recover vram bo from shadow done\n")do { } while(0);
4151 return 0;
4152}
4153
4154
4155/**
4156 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4157 *
4158 * @adev: amdgpu_device pointer
4159 * @from_hypervisor: request from hypervisor
4160 *
4161 * do VF FLR and reinitialize Asic
4162 * return 0 means succeeded otherwise failed
4163 */
4164static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4165 bool_Bool from_hypervisor)
4166{
4167 int r;
4168
4169 if (from_hypervisor)
4170 r = amdgpu_virt_request_full_gpu(adev, true1);
4171 else
4172 r = amdgpu_virt_reset_gpu(adev);
4173 if (r)
4174 return r;
4175
4176 amdgpu_amdkfd_pre_reset(adev);
4177
4178 /* Resume IP prior to SMC */
4179 r = amdgpu_device_ip_reinit_early_sriov(adev);
4180 if (r)
4181 goto error;
4182
4183 amdgpu_virt_init_data_exchange(adev);
4184 /* we need recover gart prior to run SMC/CP/SDMA resume */
4185 amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT1));
4186
4187 r = amdgpu_device_fw_loading(adev);
4188 if (r)
4189 return r;
4190
4191 /* now we are okay to resume SMC/CP/SDMA */
4192 r = amdgpu_device_ip_reinit_late_sriov(adev);
4193 if (r)
4194 goto error;
4195
4196 amdgpu_irq_gpu_reset_resume_helper(adev);
4197 r = amdgpu_ib_ring_tests(adev);
4198 amdgpu_amdkfd_post_reset(adev);
4199
4200error:
4201 amdgpu_virt_release_full_gpu(adev, true1);
4202 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4203 amdgpu_inc_vram_lost(adev)__sync_fetch_and_add(&((adev)->vram_lost_counter), 1);;
4204 r = amdgpu_device_recover_vram(adev);
4205 }
4206
4207 return r;
4208}
4209
4210/**
4211 * amdgpu_device_has_job_running - check if there is any job in mirror list
4212 *
4213 * @adev: amdgpu_device pointer
4214 *
4215 * check if there is any job in mirror list
4216 */
4217bool_Bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4218{
4219 int i;
4220 struct drm_sched_job *job;
4221
4222 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
4223 struct amdgpu_ring *ring = adev->rings[i];
4224
4225 if (!ring || !ring->sched.thread)
4226 continue;
4227
4228 spin_lock(&ring->sched.job_list_lock)mtx_enter(&ring->sched.job_list_lock);
4229 job = list_first_entry_or_null(&ring->sched.ring_mirror_list,(list_empty(&ring->sched.ring_mirror_list) ? ((void *)
0) : ({ const __typeof( ((struct drm_sched_job *)0)->node )
*__mptr = ((&ring->sched.ring_mirror_list)->next);
(struct drm_sched_job *)( (char *)__mptr - __builtin_offsetof
(struct drm_sched_job, node) );}))
4230 struct drm_sched_job, node)(list_empty(&ring->sched.ring_mirror_list) ? ((void *)
0) : ({ const __typeof( ((struct drm_sched_job *)0)->node )
*__mptr = ((&ring->sched.ring_mirror_list)->next);
(struct drm_sched_job *)( (char *)__mptr - __builtin_offsetof
(struct drm_sched_job, node) );}));
4231 spin_unlock(&ring->sched.job_list_lock)mtx_leave(&ring->sched.job_list_lock);
4232 if (job)
4233 return true1;
4234 }
4235 return false0;
4236}
4237
4238/**
4239 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4240 *
4241 * @adev: amdgpu_device pointer
4242 *
4243 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4244 * a hung GPU.
4245 */
4246bool_Bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4247{
4248 if (!amdgpu_device_ip_check_soft_reset(adev)) {
4249 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n")do { } while(0);
4250 return false0;
4251 }
4252
4253 if (amdgpu_gpu_recovery == 0)
4254 goto disabled;
4255
4256 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
4257 return true1;
4258
4259 if (amdgpu_gpu_recovery == -1) {
4260 switch (adev->asic_type) {
4261 case CHIP_BONAIRE:
4262 case CHIP_HAWAII:
4263 case CHIP_TOPAZ:
4264 case CHIP_TONGA:
4265 case CHIP_FIJI:
4266 case CHIP_POLARIS10:
4267 case CHIP_POLARIS11:
4268 case CHIP_POLARIS12:
4269 case CHIP_VEGAM:
4270 case CHIP_VEGA20:
4271 case CHIP_VEGA10:
4272 case CHIP_VEGA12:
4273 case CHIP_RAVEN:
4274 case CHIP_ARCTURUS:
4275 case CHIP_RENOIR:
4276 case CHIP_NAVI10:
4277 case CHIP_NAVI14:
4278 case CHIP_NAVI12:
4279 case CHIP_SIENNA_CICHLID:
4280 break;
4281 default:
4282 goto disabled;
4283 }
4284 }
4285
4286 return true1;
4287
4288disabled:
4289 dev_info(adev->dev, "GPU recovery disabled.\n")do { } while(0);
4290 return false0;
4291}
4292
4293
4294static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4295 struct amdgpu_job *job,
4296 bool_Bool *need_full_reset_arg)
4297{
4298 int i, r = 0;
4299 bool_Bool need_full_reset = *need_full_reset_arg;
4300
4301 amdgpu_debugfs_wait_dump(adev);
4302
4303 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
4304 /* stop the data exchange thread */
4305 amdgpu_virt_fini_data_exchange(adev);
4306 }
4307
4308 /* block all schedulers and reset given job's ring */
4309 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
4310 struct amdgpu_ring *ring = adev->rings[i];
4311
4312 if (!ring || !ring->sched.thread)
4313 continue;
4314
4315 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4316 amdgpu_fence_driver_force_completion(ring);
4317 }
4318
4319 if(job)
4320 drm_sched_increase_karma(&job->base);
4321
4322 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4323 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
4324
4325 if (!need_full_reset)
4326 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4327
4328 if (!need_full_reset) {
4329 amdgpu_device_ip_pre_soft_reset(adev);
4330 r = amdgpu_device_ip_soft_reset(adev);
4331 amdgpu_device_ip_post_soft_reset(adev);
4332 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4333 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n")do { } while(0);
4334 need_full_reset = true1;
4335 }
4336 }
4337
4338 if (need_full_reset)
4339 r = amdgpu_device_ip_suspend(adev);
4340
4341 *need_full_reset_arg = need_full_reset;
4342 }
4343
4344 return r;
4345}
4346
4347static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
4348 struct list_head *device_list_handle,
4349 bool_Bool *need_full_reset_arg,
4350 bool_Bool skip_hw_reset)
4351{
4352 struct amdgpu_device *tmp_adev = NULL((void *)0);
4353 bool_Bool need_full_reset = *need_full_reset_arg, vram_lost = false0;
4354 int r = 0;
4355
4356 /*
4357 * ASIC reset has to be done on all HGMI hive nodes ASAP
4358 * to allow proper links negotiation in FW (within 1 sec)
4359 */
4360 if (!skip_hw_reset && need_full_reset) {
4361 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4362 /* For XGMI run all resets in parallel to speed up the process */
4363 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4364 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4365 r = -EALREADY37;
4366 } else
4367 r = amdgpu_asic_reset(tmp_adev)(tmp_adev)->asic_funcs->reset((tmp_adev));
4368
4369 if (r) {
4370 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",printf("drm:pid%d:%s *ERROR* " "ASIC reset failed with error, %d for drm dev, %s"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r, adev_to_drm
(tmp_adev)->unique)
4371 r, adev_to_drm(tmp_adev)->unique)printf("drm:pid%d:%s *ERROR* " "ASIC reset failed with error, %d for drm dev, %s"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r, adev_to_drm
(tmp_adev)->unique);
4372 break;
4373 }
4374 }
4375
4376 /* For XGMI wait for all resets to complete before proceed */
4377 if (!r) {
4378 list_for_each_entry(tmp_adev, device_list_handle,for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
))
4379 gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4380 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4381 flush_work(&tmp_adev->xgmi_reset_work);
4382 r = tmp_adev->asic_reset_res;
4383 if (r)
4384 break;
4385 }
4386 }
4387 }
4388 }
4389
4390 if (!r && amdgpu_ras_intr_triggered()) {
4391 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4392 if (tmp_adev->mmhub.funcs &&
4393 tmp_adev->mmhub.funcs->reset_ras_error_count)
4394 tmp_adev->mmhub.funcs->reset_ras_error_count(tmp_adev);
4395 }
4396
4397 amdgpu_ras_intr_cleared();
4398 }
4399
4400 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4401 if (need_full_reset) {
4402 /* post card */
4403 if (amdgpu_device_asic_init(tmp_adev))
4404 dev_warn(tmp_adev->dev, "asic atom init failed!")printf("drm:pid%d:%s *WARNING* " "asic atom init failed!", ({
struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
4405
4406 if (!r) {
4407 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n")do { } while(0);
4408 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4409 if (r)
4410 goto out;
4411
4412 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4413 if (r)
4414 goto out;
4415
4416 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4417 if (vram_lost) {
4418 DRM_INFO("VRAM is lost due to GPU reset!\n")printk("\0016" "[" "drm" "] " "VRAM is lost due to GPU reset!\n"
);
4419 amdgpu_inc_vram_lost(tmp_adev)__sync_fetch_and_add(&((tmp_adev)->vram_lost_counter),
1);;
4420 }
4421
4422 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT1));
4423 if (r)
4424 goto out;
4425
4426 r = amdgpu_device_fw_loading(tmp_adev);
4427 if (r)
4428 return r;
4429
4430 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4431 if (r)
4432 goto out;
4433
4434 if (vram_lost)
4435 amdgpu_device_fill_reset_magic(tmp_adev);
4436
4437 /*
4438 * Add this ASIC as tracked as reset was already
4439 * complete successfully.
4440 */
4441 amdgpu_register_gpu_instance(tmp_adev);
4442
4443 r = amdgpu_device_ip_late_init(tmp_adev);
4444 if (r)
4445 goto out;
4446
4447 amdgpu_fbdev_set_suspend(tmp_adev, 0);
4448
4449 /*
4450 * The GPU enters bad state once faulty pages
4451 * by ECC has reached the threshold, and ras
4452 * recovery is scheduled next. So add one check
4453 * here to break recovery if it indeed exceeds
4454 * bad page threshold, and remind user to
4455 * retire this GPU or setting one bigger
4456 * bad_page_threshold value to fix this once
4457 * probing driver again.
4458 */
4459 if (!amdgpu_ras_check_err_threshold(tmp_adev)) {
4460 /* must succeed. */
4461 amdgpu_ras_resume(tmp_adev);
4462 } else {
4463 r = -EINVAL22;
4464 goto out;
4465 }
4466
4467 /* Update PSP FW topology after reset */
4468 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4469 r = amdgpu_xgmi_update_topology(hive, tmp_adev);
4470 }
4471 }
4472
4473out:
4474 if (!r) {
4475 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4476 r = amdgpu_ib_ring_tests(tmp_adev);
4477 if (r) {
4478 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r)printf("drm:pid%d:%s *ERROR* " "ib ring test failed (%d).\n",
({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
4479 need_full_reset = true1;
4480 r = -EAGAIN35;
4481 goto end;
4482 }
4483 }
4484
4485 if (!r)
4486 r = amdgpu_device_recover_vram(tmp_adev);
4487 else
4488 tmp_adev->asic_reset_res = r;
4489 }
4490
4491end:
4492 *need_full_reset_arg = need_full_reset;
4493 return r;
4494}
4495
4496static bool_Bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4497 struct amdgpu_hive_info *hive)
4498{
4499 if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1)__sync_val_compare_and_swap(&adev->in_gpu_reset, 0, 1) != 0)
4500 return false0;
4501
4502 if (hive) {
4503 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock)rw_enter_write(&adev->reset_sem);
4504 } else {
4505 down_write(&adev->reset_sem)rw_enter_write(&adev->reset_sem);
4506 }
4507
4508 atomic_inc(&adev->gpu_reset_counter)__sync_fetch_and_add(&adev->gpu_reset_counter, 1);
4509 switch (amdgpu_asic_reset_method(adev)(adev)->asic_funcs->reset_method((adev))) {
4510 case AMD_RESET_METHOD_MODE1:
4511 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4512 break;
4513 case AMD_RESET_METHOD_MODE2:
4514 adev->mp1_state = PP_MP1_STATE_RESET;
4515 break;
4516 default:
4517 adev->mp1_state = PP_MP1_STATE_NONE;
4518 break;
4519 }
4520
4521 return true1;
4522}
4523
4524static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4525{
4526 amdgpu_vf_error_trans_all(adev);
4527 adev->mp1_state = PP_MP1_STATE_NONE;
4528 atomic_set(&adev->in_gpu_reset, 0)({ typeof(*(&adev->in_gpu_reset)) __tmp = ((0)); *(volatile
typeof(*(&adev->in_gpu_reset)) *)&(*(&adev->
in_gpu_reset)) = __tmp; __tmp; });
4529 up_write(&adev->reset_sem)rw_exit_write(&adev->reset_sem);
4530}
4531
4532static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4533{
4534 STUB()do { printf("%s: stub\n", __func__); } while(0);
4535#ifdef notyet
4536 struct pci_dev *p = NULL((void *)0);
4537
4538 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4539 adev->pdev->bus->number, 1);
4540 if (p) {
4541 pm_runtime_enable(&(p->dev));
4542 pm_runtime_resume(&(p->dev));
4543 }
4544#endif
4545}
4546
4547static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4548{
4549 enum amd_reset_method reset_method;
4550 struct pci_dev *p = NULL((void *)0);
4551 u64 expires;
4552
4553 /*
4554 * For now, only BACO and mode1 reset are confirmed
4555 * to suffer the audio issue without proper suspended.
4556 */
4557 reset_method = amdgpu_asic_reset_method(adev)(adev)->asic_funcs->reset_method((adev));
4558 if ((reset_method != AMD_RESET_METHOD_BACO) &&
4559 (reset_method != AMD_RESET_METHOD_MODE1))
4560 return -EINVAL22;
4561
4562 STUB()do { printf("%s: stub\n", __func__); } while(0);
4563 return -ENOSYS78;
4564#ifdef notyet
4565
4566 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4567 adev->pdev->bus->number, 1);
4568 if (!p)
4569 return -ENODEV19;
4570
4571 expires = pm_runtime_autosuspend_expiration(&(p->dev));
4572 if (!expires)
4573 /*
4574 * If we cannot get the audio device autosuspend delay,
4575 * a fixed 4S interval will be used. Considering 3S is
4576 * the audio controller default autosuspend delay setting.
4577 * 4S used here is guaranteed to cover that.
4578 */
4579 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC1000000000L * 4ULL;
4580
4581 while (!pm_runtime_status_suspended(&(p->dev))) {
4582 if (!pm_runtime_suspend(&(p->dev)))
4583 break;
4584
4585 if (expires < ktime_get_mono_fast_ns()) {
4586 dev_warn(adev->dev, "failed to suspend display audio\n")printf("drm:pid%d:%s *WARNING* " "failed to suspend display audio\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
4587 /* TODO: abort the succeeding gpu reset? */
4588 return -ETIMEDOUT60;
4589 }
4590 }
4591
4592 pm_runtime_disable(&(p->dev));
4593
4594 return 0;
4595#endif
4596}
4597
4598/**
4599 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4600 *
4601 * @adev: amdgpu_device pointer
4602 * @job: which job trigger hang
4603 *
4604 * Attempt to reset the GPU if it has hung (all asics).
4605 * Attempt to do soft-reset or full-reset and reinitialize Asic
4606 * Returns 0 for success or an error on failure.
4607 */
4608
4609int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4610 struct amdgpu_job *job)
4611{
4612 struct list_head device_list, *device_list_handle = NULL((void *)0);
4613 bool_Bool need_full_reset = false0;
4614 bool_Bool job_signaled = false0;
4615 struct amdgpu_hive_info *hive = NULL((void *)0);
4616 struct amdgpu_device *tmp_adev = NULL((void *)0);
4617 int i, r = 0;
4618 bool_Bool need_emergency_restart = false0;
4619 bool_Bool audio_suspended = false0;
4620
4621 /*
4622 * Special case: RAS triggered and full reset isn't supported
4623 */
4624 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4625
4626 /*
4627 * Flush RAM to disk so that after reboot
4628 * the user can read log and see why the system rebooted.
4629 */
4630 if (need_emergency_restart && amdgpu_ras_get_context(adev)((adev)->psp.ras.ras)->reboot) {
4631 DRM_WARN("Emergency reboot.")printk("\0014" "[" "drm" "] " "Emergency reboot.");
4632
4633#ifdef notyet
4634 ksys_sync_helper();
4635 emergency_restart();
4636#else
4637 panic("emergency_restart");
4638#endif
4639 }
4640
4641 dev_info(adev->dev, "GPU %s begin!\n",do { } while(0)
4642 need_emergency_restart ? "jobs stop":"reset")do { } while(0);
4643
4644 /*
4645 * Here we trylock to avoid chain of resets executing from
4646 * either trigger by jobs on different adevs in XGMI hive or jobs on
4647 * different schedulers for same device while this TO handler is running.
4648 * We always reset all schedulers for device and all devices for XGMI
4649 * hive so that should take care of them too.
4650 */
4651 hive = amdgpu_get_xgmi_hive(adev);
4652 if (hive) {
4653 if (atomic_cmpxchg(&hive->in_reset, 0, 1)__sync_val_compare_and_swap(&hive->in_reset, 0, 1) != 0) {
4654 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",printk("\0016" "[" "drm" "] " "Bailing on TDR for s_job:%llx, hive: %llx as another already in progress"
, job ? job->base.id : -1, hive->hive_id)
4655 job ? job->base.id : -1, hive->hive_id)printk("\0016" "[" "drm" "] " "Bailing on TDR for s_job:%llx, hive: %llx as another already in progress"
, job ? job->base.id : -1, hive->hive_id);
4656 amdgpu_put_xgmi_hive(hive);
4657 return 0;
4658 }
4659 mutex_lock(&hive->hive_lock)rw_enter_write(&hive->hive_lock);
4660 }
4661
4662 /*
4663 * Build list of devices to reset.
4664 * In case we are in XGMI hive mode, resort the device list
4665 * to put adev in the 1st position.
4666 */
4667 INIT_LIST_HEAD(&device_list);
4668 if (adev->gmc.xgmi.num_physical_nodes > 1) {
4669 if (!hive)
4670 return -ENODEV19;
4671 if (!list_is_first(&adev->gmc.xgmi.head, &hive->device_list))
4672 list_rotate_to_front(&adev->gmc.xgmi.head, &hive->device_list);
4673 device_list_handle = &hive->device_list;
4674 } else {
4675 list_add_tail(&adev->gmc.xgmi.head, &device_list);
4676 device_list_handle = &device_list;
4677 }
4678
4679 /* block all schedulers and reset given job's ring */
4680 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4681 if (!amdgpu_device_lock_adev(tmp_adev, hive)) {
4682 dev_info(tmp_adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",do { } while(0)
4683 job ? job->base.id : -1)do { } while(0);
4684 r = 0;
4685 goto skip_recovery;
4686 }
4687
4688 /*
4689 * Try to put the audio codec into suspend state
4690 * before gpu reset started.
4691 *
4692 * Due to the power domain of the graphics device
4693 * is shared with AZ power domain. Without this,
4694 * we may change the audio hardware from behind
4695 * the audio driver's back. That will trigger
4696 * some audio codec errors.
4697 */
4698 if (!amdgpu_device_suspend_display_audio(tmp_adev))
4699 audio_suspended = true1;
4700
4701 amdgpu_ras_set_error_query_ready(tmp_adev, false0);
4702
4703 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
4704
4705 if (!amdgpu_sriov_vf(tmp_adev)((tmp_adev)->virt.caps & (1 << 2)))
4706 amdgpu_amdkfd_pre_reset(tmp_adev);
4707
4708 /*
4709 * Mark these ASICs to be reseted as untracked first
4710 * And add them back after reset completed
4711 */
4712 amdgpu_unregister_gpu_instance(tmp_adev);
4713
4714 amdgpu_fbdev_set_suspend(tmp_adev, 1);
4715
4716 /* disable ras on ALL IPs */
4717 if (!need_emergency_restart &&
4718 amdgpu_device_ip_need_full_reset(tmp_adev))
4719 amdgpu_ras_suspend(tmp_adev);
4720
4721 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
4722 struct amdgpu_ring *ring = tmp_adev->rings[i];
4723
4724 if (!ring || !ring->sched.thread)
4725 continue;
4726
4727 drm_sched_stop(&ring->sched, job ? &job->base : NULL((void *)0));
4728
4729 if (need_emergency_restart)
4730 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
4731 }
4732 }
4733
4734 if (need_emergency_restart)
4735 goto skip_sched_resume;
4736
4737 /*
4738 * Must check guilty signal here since after this point all old
4739 * HW fences are force signaled.
4740 *
4741 * job->base holds a reference to parent fence
4742 */
4743 if (job && job->base.s_fence->parent &&
4744 dma_fence_is_signaled(job->base.s_fence->parent)) {
4745 job_signaled = true1;
4746 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset")do { } while(0);
4747 goto skip_hw_reset;
4748 }
4749
4750retry: /* Rest of adevs pre asic reset from XGMI hive. */
4751 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4752 r = amdgpu_device_pre_asic_reset(tmp_adev,
4753 (tmp_adev == adev) ? job : NULL((void *)0),
4754 &need_full_reset);
4755 /*TODO Should we stop ?*/
4756 if (r) {
4757 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",printf("drm:pid%d:%s *ERROR* " "GPU pre asic reset failed with err, %d for drm dev, %s "
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r, adev_to_drm
(tmp_adev)->unique)
4758 r, adev_to_drm(tmp_adev)->unique)printf("drm:pid%d:%s *ERROR* " "GPU pre asic reset failed with err, %d for drm dev, %s "
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r, adev_to_drm
(tmp_adev)->unique);
4759 tmp_adev->asic_reset_res = r;
4760 }
4761 }
4762
4763 /* Actual ASIC resets if needed.*/
4764 /* TODO Implement XGMI hive reset logic for SRIOV */
4765 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
4766 r = amdgpu_device_reset_sriov(adev, job ? false0 : true1);
4767 if (r)
4768 adev->asic_reset_res = r;
4769 } else {
4770 r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset, false0);
4771 if (r && r == -EAGAIN35)
4772 goto retry;
4773 }
4774
4775skip_hw_reset:
4776
4777 /* Post ASIC reset for all devs .*/
4778 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4779
4780 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
4781 struct amdgpu_ring *ring = tmp_adev->rings[i];
4782
4783 if (!ring || !ring->sched.thread)
4784 continue;
4785
4786 /* No point to resubmit jobs if we didn't HW reset*/
4787 if (!tmp_adev->asic_reset_res && !job_signaled)
4788 drm_sched_resubmit_jobs(&ring->sched);
4789
4790 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
4791 }
4792
4793 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
4794 drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
4795 }
4796
4797 tmp_adev->asic_reset_res = 0;
4798
4799 if (r) {
4800 /* bad news, how to tell it to userspace ? */
4801 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter))do { } while(0);
4802 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
4803 } else {
4804 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter))do { } while(0);
4805 }
4806 }
4807
4808skip_sched_resume:
4809 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)for (tmp_adev = ({ const __typeof( ((__typeof(*tmp_adev) *)0)
->gmc.xgmi.head ) *__mptr = ((device_list_handle)->next
); (__typeof(*tmp_adev) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*tmp_adev), gmc.xgmi.head) );}); &tmp_adev->
gmc.xgmi.head != (device_list_handle); tmp_adev = ({ const __typeof
( ((__typeof(*tmp_adev) *)0)->gmc.xgmi.head ) *__mptr = (tmp_adev
->gmc.xgmi.head.next); (__typeof(*tmp_adev) *)( (char *)__mptr
- __builtin_offsetof(__typeof(*tmp_adev), gmc.xgmi.head) );}
)) {
4810 /*unlock kfd: SRIOV would do it separately */
4811 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)((tmp_adev)->virt.caps & (1 << 2)))
4812 amdgpu_amdkfd_post_reset(tmp_adev);
4813 if (audio_suspended)
4814 amdgpu_device_resume_display_audio(tmp_adev);
4815 amdgpu_device_unlock_adev(tmp_adev);
4816 }
4817
4818skip_recovery:
4819 if (hive) {
4820 atomic_set(&hive->in_reset, 0)({ typeof(*(&hive->in_reset)) __tmp = ((0)); *(volatile
typeof(*(&hive->in_reset)) *)&(*(&hive->in_reset
)) = __tmp; __tmp; });
4821 mutex_unlock(&hive->hive_lock)rw_exit_write(&hive->hive_lock);
4822 amdgpu_put_xgmi_hive(hive);
4823 }
4824
4825 if (r)
4826 dev_info(adev->dev, "GPU reset end with ret = %d\n", r)do { } while(0);
4827 return r;
4828}
4829
4830/**
4831 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
4832 *
4833 * @adev: amdgpu_device pointer
4834 *
4835 * Fetchs and stores in the driver the PCIE capabilities (gen speed
4836 * and lanes) of the slot the device is in. Handles APUs and
4837 * virtualized environments where PCIE config space may not be available.
4838 */
4839static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
4840{
4841 struct pci_dev *pdev;
4842 enum pci_bus_speed speed_cap, platform_speed_cap;
4843 enum pcie_link_width platform_link_width;
53
←
'platform_link_width' declared without an initial value
→
4844
4845 if (amdgpu_pcie_gen_cap)
54
←
Assuming 'amdgpu_pcie_gen_cap' is 0
→
55
←
Taking false branch
→
4846 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
4847
4848 if (amdgpu_pcie_lane_cap)
56
←
Assuming 'amdgpu_pcie_lane_cap' is 0
→
57
←
Taking false branch
→
4849 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
4850
4851 /* covers APUs as well */
4852 if (pci_is_root_bus(adev->pdev->bus)) {
58
←
Calling 'pci_is_root_bus'
→
61
←
Returning from 'pci_is_root_bus'
→
62
←
Taking false branch
→
4853 if (adev->pm.pcie_gen_mask == 0)
4854 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK(0x00010000 | 0x00020000 | 0x00000001 | 0x00000002 | 0x00000004
);
4855 if (adev->pm.pcie_mlw_mask == 0)
4856 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK(0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00200000
);
4857 return;
4858 }
4859
4860 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
63
←
Assuming field 'pcie_gen_mask' is not equal to 0
→
64
←
Assuming field 'pcie_mlw_mask' is 0
→
65
←
Taking false branch
→
4861 return;
4862
4863 pcie_bandwidth_available(adev->pdev, NULL((void *)0),
66
←
Calling 'pcie_bandwidth_available'
→
70
←
Returning from 'pcie_bandwidth_available'
→
4864 &platform_speed_cap, &platform_link_width);
4865
4866 if (adev->pm.pcie_gen_mask
70.1
Field 'pcie_gen_mask' is not equal to 0
70.1
Field 'pcie_gen_mask' is not equal to 0
70.1
Field 'pcie_gen_mask' is not equal to 0
 == 0) {
71
←
Taking false branch
→
4867 /* asic caps */
4868 pdev = adev->pdev;
4869 speed_cap = pcie_get_speed_cap(pdev);
4870 if (speed_cap == PCI_SPEED_UNKNOWN) {
4871 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN10x00000001 |
4872 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN20x00000002 |
4873 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN30x00000004);
4874 } else {
4875 if (speed_cap == PCIE_SPEED_16_0GT)
4876 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN10x00000001 |
4877 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN20x00000002 |
4878 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN30x00000004 |
4879 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN40x00000008);
4880 else if (speed_cap == PCIE_SPEED_8_0GT)
4881 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN10x00000001 |
4882 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN20x00000002 |
4883 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN30x00000004);
4884 else if (speed_cap == PCIE_SPEED_5_0GT)
4885 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN10x00000001 |
4886 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN20x00000002);
4887 else
4888 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN10x00000001;
4889 }
4890 /* platform caps */
4891 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
4892 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN10x00010000 |
4893 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN20x00020000);
4894 } else {
4895 if (platform_speed_cap == PCIE_SPEED_16_0GT)
4896 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN10x00010000 |
4897 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN20x00020000 |
4898 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN30x00040000 |
4899 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN40x00080000);
4900 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
4901 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN10x00010000 |
4902 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN20x00020000 |
4903 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN30x00040000);
4904 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
4905 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN10x00010000 |
4906 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN20x00020000);
4907 else
4908 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN10x00010000;
4909
4910 }
4911 }
4912 if (adev->pm.pcie_mlw_mask
71.1
Field 'pcie_mlw_mask' is equal to 0
71.1
Field 'pcie_mlw_mask' is equal to 0
71.1
Field 'pcie_mlw_mask' is equal to 0
 == 0) {
72
←
Taking true branch
→
4913 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
73
←
The left operand of '==' is a garbage value
4914 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK(0x00010000 | 0x00020000 | 0x00040000 | 0x00080000 | 0x00200000
);
4915 } else {
4916 switch (platform_link_width) {
4917 case PCIE_LNK_X32:
4918 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X320x00400000 |
4919 CAIL_PCIE_LINK_WIDTH_SUPPORT_X160x00200000 |
4920 CAIL_PCIE_LINK_WIDTH_SUPPORT_X120x00100000 |
4921 CAIL_PCIE_LINK_WIDTH_SUPPORT_X80x00080000 |
4922 CAIL_PCIE_LINK_WIDTH_SUPPORT_X40x00040000 |
4923 CAIL_PCIE_LINK_WIDTH_SUPPORT_X20x00020000 |
4924 CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000);
4925 break;
4926 case PCIE_LNK_X16:
4927 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X160x00200000 |
4928 CAIL_PCIE_LINK_WIDTH_SUPPORT_X120x00100000 |
4929 CAIL_PCIE_LINK_WIDTH_SUPPORT_X80x00080000 |
4930 CAIL_PCIE_LINK_WIDTH_SUPPORT_X40x00040000 |
4931 CAIL_PCIE_LINK_WIDTH_SUPPORT_X20x00020000 |
4932 CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000);
4933 break;
4934 case PCIE_LNK_X12:
4935 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X120x00100000 |
4936 CAIL_PCIE_LINK_WIDTH_SUPPORT_X80x00080000 |
4937 CAIL_PCIE_LINK_WIDTH_SUPPORT_X40x00040000 |
4938 CAIL_PCIE_LINK_WIDTH_SUPPORT_X20x00020000 |
4939 CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000);
4940 break;
4941 case PCIE_LNK_X8:
4942 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X80x00080000 |
4943 CAIL_PCIE_LINK_WIDTH_SUPPORT_X40x00040000 |
4944 CAIL_PCIE_LINK_WIDTH_SUPPORT_X20x00020000 |
4945 CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000);
4946 break;
4947 case PCIE_LNK_X4:
4948 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X40x00040000 |
4949 CAIL_PCIE_LINK_WIDTH_SUPPORT_X20x00020000 |
4950 CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000);
4951 break;
4952 case PCIE_LNK_X2:
4953 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X20x00020000 |
4954 CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000);
4955 break;
4956 case PCIE_LNK_X1:
4957 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X10x00010000;
4958 break;
4959 default:
4960 break;
4961 }
4962 }
4963 }
4964}
4965
4966int amdgpu_device_baco_enter(struct drm_device *dev)
4967{
4968 struct amdgpu_device *adev = drm_to_adev(dev);
4969 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev)((adev)->psp.ras.ras);
4970
4971 if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
4972 return -ENOTSUPP91;
4973
4974 if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
4975 adev->nbio.funcs->enable_doorbell_interrupt(adev, false0);
4976
4977 return amdgpu_dpm_baco_enter(adev);
4978}
4979
4980int amdgpu_device_baco_exit(struct drm_device *dev)
4981{
4982 struct amdgpu_device *adev = drm_to_adev(dev);
4983 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev)((adev)->psp.ras.ras);
4984 int ret = 0;
4985
4986 if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
4987 return -ENOTSUPP91;
4988
4989 ret = amdgpu_dpm_baco_exit(adev);
4990 if (ret)
4991 return ret;
4992
4993 if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
4994 adev->nbio.funcs->enable_doorbell_interrupt(adev, true1);
4995
4996 return 0;
4997}
4998
4999static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5000{
5001 int i;
5002
5003 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
5004 struct amdgpu_ring *ring = adev->rings[i];
5005
5006 if (!ring || !ring->sched.thread)
5007 continue;
5008
5009 cancel_delayed_work_sync(&ring->sched.work_tdr);
5010 }
5011}
5012
5013/**
5014 * amdgpu_pci_error_detected - Called when a PCI error is detected.
5015 * @pdev: PCI device struct
5016 * @state: PCI channel state
5017 *
5018 * Description: Called when a PCI error is detected.
5019 *
5020 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5021 */
5022pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5023{
5024 STUB()do { printf("%s: stub\n", __func__); } while(0);
5025 return 0;
5026#ifdef notyet
5027 struct drm_device *dev = pci_get_drvdata(pdev);
5028 struct amdgpu_device *adev = drm_to_adev(dev);
5029 int i;
5030
5031 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state)printk("\0016" "[" "drm" "] " "PCI error: detected callback, state(%d)!!\n"
, state);
5032
5033 if (adev->gmc.xgmi.num_physical_nodes > 1) {
5034 DRM_WARN("No support for XGMI hive yet...")printk("\0014" "[" "drm" "] " "No support for XGMI hive yet..."
);
5035 return PCI_ERS_RESULT_DISCONNECT0;
5036 }
5037
5038 switch (state) {
5039 case pci_channel_io_normal:
5040 return PCI_ERS_RESULT_CAN_RECOVER;
5041 /* Fatal error, prepare for slot reset */
5042 case pci_channel_io_frozen:
5043 /*
5044 * Cancel and wait for all TDRs in progress if failing to
5045 * set adev->in_gpu_reset in amdgpu_device_lock_adev
5046 *
5047 * Locking adev->reset_sem will prevent any external access
5048 * to GPU during PCI error recovery
5049 */
5050 while (!amdgpu_device_lock_adev(adev, NULL((void *)0)))
5051 amdgpu_cancel_all_tdr(adev);
5052
5053 /*
5054 * Block any work scheduling as we do for regular GPU reset
5055 * for the duration of the recovery
5056 */
5057 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
5058 struct amdgpu_ring *ring = adev->rings[i];
5059
5060 if (!ring || !ring->sched.thread)
5061 continue;
5062
5063 drm_sched_stop(&ring->sched, NULL((void *)0));
5064 }
5065 return PCI_ERS_RESULT_NEED_RESET;
5066 case pci_channel_io_perm_failure:
5067 /* Permanent error, prepare for device removal */
5068 return PCI_ERS_RESULT_DISCONNECT0;
5069 }
5070
5071 return PCI_ERS_RESULT_NEED_RESET;
5072#endif
5073}
5074
5075/**
5076 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5077 * @pdev: pointer to PCI device
5078 */
5079pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5080{
5081
5082 DRM_INFO("PCI error: mmio enabled callback!!\n")printk("\0016" "[" "drm" "] " "PCI error: mmio enabled callback!!\n"
);
5083
5084 /* TODO - dump whatever for debugging purposes */
5085
5086 /* This called only if amdgpu_pci_error_detected returns
5087 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5088 * works, no need to reset slot.
5089 */
5090
5091 return PCI_ERS_RESULT_RECOVERED1;
5092}
5093
5094/**
5095 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5096 * @pdev: PCI device struct
5097 *
5098 * Description: This routine is called by the pci error recovery
5099 * code after the PCI slot has been reset, just before we
5100 * should resume normal operations.
5101 */
5102pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5103{
5104 STUB()do { printf("%s: stub\n", __func__); } while(0);
5105 return PCI_ERS_RESULT_RECOVERED1;
5106#ifdef notyet
5107 struct drm_device *dev = pci_get_drvdata(pdev);
5108 struct amdgpu_device *adev = drm_to_adev(dev);
5109 int r, i;
5110 bool_Bool need_full_reset = true1;
5111 u32 memsize;
5112 struct list_head device_list;
5113
5114 DRM_INFO("PCI error: slot reset callback!!\n")printk("\0016" "[" "drm" "] " "PCI error: slot reset callback!!\n"
);
5115
5116 INIT_LIST_HEAD(&device_list);
5117 list_add_tail(&adev->gmc.xgmi.head, &device_list);
5118
5119 /* wait for asic to come out of reset */
5120 drm_msleep(500)mdelay(500);
5121
5122 /* Restore PCI confspace */
5123 amdgpu_device_load_pci_state(pdev);
5124
5125 /* confirm ASIC came out of reset */
5126 for (i = 0; i < adev->usec_timeout; i++) {
5127 memsize = amdgpu_asic_get_config_memsize(adev)(adev)->asic_funcs->get_config_memsize((adev));
5128
5129 if (memsize != 0xffffffff)
5130 break;
5131 udelay(1);
5132 }
5133 if (memsize == 0xffffffff) {
5134 r = -ETIME60;
5135 goto out;
5136 }
5137
5138 adev->in_pci_err_recovery = true1;
5139 r = amdgpu_device_pre_asic_reset(adev, NULL((void *)0), &need_full_reset);
5140 adev->in_pci_err_recovery = false0;
5141 if (r)
5142 goto out;
5143
5144 r = amdgpu_do_asic_reset(NULL((void *)0), &device_list, &need_full_reset, true1);
5145
5146out:
5147 if (!r) {
5148 if (amdgpu_device_cache_pci_state(adev->pdev))
5149 pci_restore_state(adev->pdev);
5150
5151 DRM_INFO("PCIe error recovery succeeded\n")printk("\0016" "[" "drm" "] " "PCIe error recovery succeeded\n"
);
5152 } else {
5153 DRM_ERROR("PCIe error recovery failed, err:%d", r)__drm_err("PCIe error recovery failed, err:%d", r);
5154 amdgpu_device_unlock_adev(adev);
5155 }
5156
5157 return r ? PCI_ERS_RESULT_DISCONNECT0 : PCI_ERS_RESULT_RECOVERED1;
5158#endif
5159}
5160
5161/**
5162 * amdgpu_pci_resume() - resume normal ops after PCI reset
5163 * @pdev: pointer to PCI device
5164 *
5165 * Called when the error recovery driver tells us that its
5166 * OK to resume normal operation. Use completion to allow
5167 * halted scsi ops to resume.
5168 */
5169void amdgpu_pci_resume(struct pci_dev *pdev)
5170{
5171 STUB()do { printf("%s: stub\n", __func__); } while(0);
5172#ifdef notyet
5173 struct drm_device *dev = pci_get_drvdata(pdev);
5174 struct amdgpu_device *adev = drm_to_adev(dev);
5175 int i;
5176
5177
5178 DRM_INFO("PCI error: resume callback!!\n")printk("\0016" "[" "drm" "] " "PCI error: resume callback!!\n"
);
5179
5180 for (i = 0; i < AMDGPU_MAX_RINGS28; ++i) {
5181 struct amdgpu_ring *ring = adev->rings[i];
5182
5183 if (!ring || !ring->sched.thread)
5184 continue;
5185
5186
5187 drm_sched_resubmit_jobs(&ring->sched);
5188 drm_sched_start(&ring->sched, true1);
5189 }
5190
5191 amdgpu_device_unlock_adev(adev);
5192#endif
5193}
5194
5195bool_Bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5196{
5197 return false0;
5198#ifdef notyet
5199 struct drm_device *dev = pci_get_drvdata(pdev);
5200 struct amdgpu_device *adev = drm_to_adev(dev);
5201 int r;
5202
5203 r = pci_save_state(pdev);
5204 if (!r) {
5205 kfree(adev->pci_state);
5206
5207 adev->pci_state = pci_store_saved_state(pdev);
5208
5209 if (!adev->pci_state) {
5210 DRM_ERROR("Failed to store PCI saved state")__drm_err("Failed to store PCI saved state");
5211 return false0;
5212 }
5213 } else {
5214 DRM_WARN("Failed to save PCI state, err:%d\n", r)printk("\0014" "[" "drm" "] " "Failed to save PCI state, err:%d\n"
, r);
5215 return false0;
5216 }
5217
5218 return true1;
5219#endif
5220}
5221
5222bool_Bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5223{
5224 STUB()do { printf("%s: stub\n", __func__); } while(0);
5225 return false0;
5226#ifdef notyet
5227 struct drm_device *dev = pci_get_drvdata(pdev);
5228 struct amdgpu_device *adev = drm_to_adev(dev);
5229 int r;
5230
5231 if (!adev->pci_state)
5232 return false0;
5233
5234 r = pci_load_saved_state(pdev, adev->pci_state);
5235
5236 if (!r) {
5237 pci_restore_state(pdev);
5238 } else {
5239 DRM_WARN("Failed to load PCI state, err:%d\n", r)printk("\0014" "[" "drm" "] " "Failed to load PCI state, err:%d\n"
, r);
5240 return false0;
5241 }
5242
5243 return true1;
5244#endif
5245}
5246
5247
←

/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu.h

→
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#ifndef __AMDGPU_H__
29#define __AMDGPU_H__
30
31#ifdef pr_fmt
32#undef pr_fmt
33#endif
34
35#define pr_fmt(fmt)"amdgpu: " fmt "amdgpu: " fmt
36
37#ifdef dev_fmt
38#undef dev_fmt
39#endif
40
41#define dev_fmt(fmt)"amdgpu: " fmt "amdgpu: " fmt
42
43#include "amdgpu_ctx.h"
44
45#include <linux/atomic.h>
46#include <linux/wait.h>
47#include <linux/list.h>
48#include <linux/kref.h>
49#include <linux/rbtree.h>
50#include <linux/hashtable.h>
51#include <linux/dma-fence.h>
52#include <linux/pci.h>
53#include <linux/aer.h>
54
55#include <drm/ttm/ttm_bo_api.h>
56#include <drm/ttm/ttm_bo_driver.h>
57#include <drm/ttm/ttm_placement.h>
58#include <drm/ttm/ttm_module.h>
59#include <drm/ttm/ttm_execbuf_util.h>
60
61#include <drm/amdgpu_drm.h>
62#include <drm/drm_gem.h>
63#include <drm/drm_ioctl.h>
64#include <drm/gpu_scheduler.h>
65
66#include <dev/wscons/wsconsio.h>
67#include <dev/wscons/wsdisplayvar.h>
68#include <dev/rasops/rasops.h>
69
70#include <kgd_kfd_interface.h>
71#include "dm_pp_interface.h"
72#include "kgd_pp_interface.h"
73
74#include "amd_shared.h"
75#include "amdgpu_mode.h"
76#include "amdgpu_ih.h"
77#include "amdgpu_irq.h"
78#include "amdgpu_ucode.h"
79#include "amdgpu_ttm.h"
80#include "amdgpu_psp.h"
81#include "amdgpu_gds.h"
82#include "amdgpu_sync.h"
83#include "amdgpu_ring.h"
84#include "amdgpu_vm.h"
85#include "amdgpu_dpm.h"
86#include "amdgpu_acp.h"
87#include "amdgpu_uvd.h"
88#include "amdgpu_vce.h"
89#include "amdgpu_vcn.h"
90#include "amdgpu_jpeg.h"
91#include "amdgpu_mn.h"
92#include "amdgpu_gmc.h"
93#include "amdgpu_gfx.h"
94#include "amdgpu_sdma.h"
95#include "amdgpu_nbio.h"
96#include "amdgpu_dm.h"
97#include "amdgpu_virt.h"
98#include "amdgpu_csa.h"
99#include "amdgpu_gart.h"
100#include "amdgpu_debugfs.h"
101#include "amdgpu_job.h"
102#include "amdgpu_bo_list.h"
103#include "amdgpu_gem.h"
104#include "amdgpu_doorbell.h"
105#include "amdgpu_amdkfd.h"
106#include "amdgpu_smu.h"
107#include "amdgpu_discovery.h"
108#include "amdgpu_mes.h"
109#include "amdgpu_umc.h"
110#include "amdgpu_mmhub.h"
111#include "amdgpu_gfxhub.h"
112#include "amdgpu_df.h"
113
114#define MAX_GPU_INSTANCE16 16
115
116struct amdgpu_gpu_instance
117{
118 struct amdgpu_device *adev;
119 int mgpu_fan_enabled;
120};
121
122struct amdgpu_mgpu_info
123{
124 struct amdgpu_gpu_instance gpu_ins[MAX_GPU_INSTANCE16];
125 struct rwlock mutex;
126 uint32_t num_gpu;
127 uint32_t num_dgpu;
128 uint32_t num_apu;
129};
130
131#define AMDGPU_MAX_TIMEOUT_PARAM_LENGTH256 256
132
133/*
134 * Modules parameters.
135 */
136extern int amdgpu_modeset;
137extern int amdgpu_vram_limit;
138extern int amdgpu_vis_vram_limit;
139extern int amdgpu_gart_size;
140extern int amdgpu_gtt_size;
141extern int amdgpu_moverate;
142extern int amdgpu_benchmarking;
143extern int amdgpu_testing;
144extern int amdgpu_audio;
145extern int amdgpu_disp_priority;
146extern int amdgpu_hw_i2c;
147extern int amdgpu_pcie_gen2;
148extern int amdgpu_msi;
149extern char amdgpu_lockup_timeout[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH256];
150extern int amdgpu_dpm;
151extern int amdgpu_fw_load_type;
152extern int amdgpu_aspm;
153extern int amdgpu_runtime_pm;
154extern uint amdgpu_ip_block_mask;
155extern int amdgpu_bapm;
156extern int amdgpu_deep_color;
157extern int amdgpu_vm_size;
158extern int amdgpu_vm_block_size;
159extern int amdgpu_vm_fragment_size;
160extern int amdgpu_vm_fault_stop;
161extern int amdgpu_vm_debug;
162extern int amdgpu_vm_update_mode;
163extern int amdgpu_exp_hw_support;
164extern int amdgpu_dc;
165extern int amdgpu_sched_jobs;
166extern int amdgpu_sched_hw_submission;
167extern uint amdgpu_pcie_gen_cap;
168extern uint amdgpu_pcie_lane_cap;
169extern uint amdgpu_cg_mask;
170extern uint amdgpu_pg_mask;
171extern uint amdgpu_sdma_phase_quantum;
172extern char *amdgpu_disable_cu;
173extern char *amdgpu_virtual_display;
174extern uint amdgpu_pp_feature_mask;
175extern uint amdgpu_force_long_training;
176extern int amdgpu_job_hang_limit;
177extern int amdgpu_lbpw;
178extern int amdgpu_compute_multipipe;
179extern int amdgpu_gpu_recovery;
180extern int amdgpu_emu_mode;
181extern uint amdgpu_smu_memory_pool_size;
182extern uint amdgpu_dc_feature_mask;
183extern uint amdgpu_dc_debug_mask;
184extern uint amdgpu_dm_abm_level;
185extern int amdgpu_backlight;
186extern struct amdgpu_mgpu_info mgpu_info;
187extern int amdgpu_ras_enable;
188extern uint amdgpu_ras_mask;
189extern int amdgpu_bad_page_threshold;
190extern int amdgpu_async_gfx_ring;
191extern int amdgpu_mcbp;
192extern int amdgpu_discovery;
193extern int amdgpu_mes;
194extern int amdgpu_noretry;
195extern int amdgpu_force_asic_type;
196#ifdef CONFIG_HSA_AMD
197extern int sched_policy;
198extern bool_Bool debug_evictions;
199extern bool_Bool no_system_mem_limit;
200#else
201static const int sched_policy = KFD_SCHED_POLICY_HWS;
202static const bool_Bool debug_evictions; /* = false */
203static const bool_Bool no_system_mem_limit;
204#endif
205
206extern int amdgpu_tmz;
207extern int amdgpu_reset_method;
208
209#ifdef CONFIG_DRM_AMDGPU_SI
210extern int amdgpu_si_support;
211#endif
212#ifdef CONFIG_DRM_AMDGPU_CIK
213extern int amdgpu_cik_support;
214#endif
215extern int amdgpu_num_kcq;
216
217#define AMDGPU_VM_MAX_NUM_CTX4096 4096
218#define AMDGPU_SG_THRESHOLD(256*1024*1024) (256*1024*1024)
219#define AMDGPU_DEFAULT_GTT_SIZE_MB3072ULL 3072ULL /* 3GB by default */
220#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS3000 3000
221#define AMDGPU_MAX_USEC_TIMEOUT100000 100000 /* 100 ms */
222#define AMDGPU_FENCE_JIFFIES_TIMEOUT(hz / 2) (HZhz / 2)
223#define AMDGPU_DEBUGFS_MAX_COMPONENTS32 32
224#define AMDGPUFB_CONN_LIMIT4 4
225#define AMDGPU_BIOS_NUM_SCRATCH16 16
226
227#define AMDGPU_VBIOS_VGA_ALLOCATION(9 * 1024 * 1024) (9 * 1024 * 1024) /* reserve 8MB for vga emulator and 1 MB for FB */
228
229/* hard reset data */
230#define AMDGPU_ASIC_RESET_DATA0x39d5e86b 0x39d5e86b
231
232/* reset flags */
233#define AMDGPU_RESET_GFX(1 << 0) (1 << 0)
234#define AMDGPU_RESET_COMPUTE(1 << 1) (1 << 1)
235#define AMDGPU_RESET_DMA(1 << 2) (1 << 2)
236#define AMDGPU_RESET_CP(1 << 3) (1 << 3)
237#define AMDGPU_RESET_GRBM(1 << 4) (1 << 4)
238#define AMDGPU_RESET_DMA1(1 << 5) (1 << 5)
239#define AMDGPU_RESET_RLC(1 << 6) (1 << 6)
240#define AMDGPU_RESET_SEM(1 << 7) (1 << 7)
241#define AMDGPU_RESET_IH(1 << 8) (1 << 8)
242#define AMDGPU_RESET_VMC(1 << 9) (1 << 9)
243#define AMDGPU_RESET_MC(1 << 10) (1 << 10)
244#define AMDGPU_RESET_DISPLAY(1 << 11) (1 << 11)
245#define AMDGPU_RESET_UVD(1 << 12) (1 << 12)
246#define AMDGPU_RESET_VCE(1 << 13) (1 << 13)
247#define AMDGPU_RESET_VCE1(1 << 14) (1 << 14)
248
249/* max cursor sizes (in pixels) */
250#define CIK_CURSOR_WIDTH128 128
251#define CIK_CURSOR_HEIGHT128 128
252
253struct amdgpu_device;
254struct amdgpu_ib;
255struct amdgpu_cs_parser;
256struct amdgpu_job;
257struct amdgpu_irq_src;
258struct amdgpu_fpriv;
259struct amdgpu_bo_va_mapping;
260struct amdgpu_atif;
261struct kfd_vm_fault_info;
262struct amdgpu_hive_info;
263
264enum amdgpu_cp_irq {
265 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
266 AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP,
267 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP,
268 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP,
269 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP,
270 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP,
271 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP,
272 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP,
273 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP,
274 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP,
275
276 AMDGPU_CP_IRQ_LAST
277};
278
279enum amdgpu_thermal_irq {
280 AMDGPU_THERMAL_IRQ_LOW_TO_HIGH = 0,
281 AMDGPU_THERMAL_IRQ_HIGH_TO_LOW,
282
283 AMDGPU_THERMAL_IRQ_LAST
284};
285
286enum amdgpu_kiq_irq {
287 AMDGPU_CP_KIQ_IRQ_DRIVER0 = 0,
288 AMDGPU_CP_KIQ_IRQ_LAST
289};
290
291#define MAX_KIQ_REG_WAIT5000 5000 /* in usecs, 5ms */
292#define MAX_KIQ_REG_BAILOUT_INTERVAL5 5 /* in msecs, 5ms */
293#define MAX_KIQ_REG_TRY80 80 /* 20 -> 80 */
294
295int amdgpu_device_ip_set_clockgating_state(void *dev,
296 enum amd_ip_block_type block_type,
297 enum amd_clockgating_state state);
298int amdgpu_device_ip_set_powergating_state(void *dev,
299 enum amd_ip_block_type block_type,
300 enum amd_powergating_state state);
301void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
302 u32 *flags);
303int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
304 enum amd_ip_block_type block_type);
305bool_Bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
306 enum amd_ip_block_type block_type);
307
308#define AMDGPU_MAX_IP_NUM16 16
309
310struct amdgpu_ip_block_status {
311 bool_Bool valid;
312 bool_Bool sw;
313 bool_Bool hw;
314 bool_Bool late_initialized;
315 bool_Bool hang;
316};
317
318struct amdgpu_ip_block_version {
319 const enum amd_ip_block_type type;
320 const u32 major;
321 const u32 minor;
322 const u32 rev;
323 const struct amd_ip_funcs *funcs;
324};
325
326#define HW_REV(_Major, _Minor, _Rev)((((uint32_t) (_Major)) << 16) | ((uint32_t) (_Minor) <<
8) | ((uint32_t) (_Rev))) \
327 ((((uint32_t) (_Major)) << 16) | ((uint32_t) (_Minor) << 8) | ((uint32_t) (_Rev)))
328
329struct amdgpu_ip_block {
330 struct amdgpu_ip_block_status status;
331 const struct amdgpu_ip_block_version *version;
332};
333
334int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
335 enum amd_ip_block_type type,
336 u32 major, u32 minor);
337
338struct amdgpu_ip_block *
339amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
340 enum amd_ip_block_type type);
341
342int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
343 const struct amdgpu_ip_block_version *ip_block_version);
344
345/*
346 * BIOS.
347 */
348bool_Bool amdgpu_get_bios(struct amdgpu_device *adev);
349bool_Bool amdgpu_read_bios(struct amdgpu_device *adev);
350
351/*
352 * Clocks
353 */
354
355#define AMDGPU_MAX_PPLL3 3
356
357struct amdgpu_clock {
358 struct amdgpu_pll ppll[AMDGPU_MAX_PPLL3];
359 struct amdgpu_pll spll;
360 struct amdgpu_pll mpll;
361 /* 10 Khz units */
362 uint32_t default_mclk;
363 uint32_t default_sclk;
364 uint32_t default_dispclk;
365 uint32_t current_dispclk;
366 uint32_t dp_extclk;
367 uint32_t max_pixel_clock;
368};
369
370/* sub-allocation manager, it has to be protected by another lock.
371 * By conception this is an helper for other part of the driver
372 * like the indirect buffer or semaphore, which both have their
373 * locking.
374 *
375 * Principe is simple, we keep a list of sub allocation in offset
376 * order (first entry has offset == 0, last entry has the highest
377 * offset).
378 *
379 * When allocating new object we first check if there is room at
380 * the end total_size - (last_object_offset + last_object_size) >=
381 * alloc_size. If so we allocate new object there.
382 *
383 * When there is not enough room at the end, we start waiting for
384 * each sub object until we reach object_offset+object_size >=
385 * alloc_size, this object then become the sub object we return.
386 *
387 * Alignment can't be bigger than page size.
388 *
389 * Hole are not considered for allocation to keep things simple.
390 * Assumption is that there won't be hole (all object on same
391 * alignment).
392 */
393
394#define AMDGPU_SA_NUM_FENCE_LISTS32 32
395
396struct amdgpu_sa_manager {
397 wait_queue_head_t wq;
398 struct amdgpu_bo *bo;
399 struct list_head *hole;
400 struct list_head flist[AMDGPU_SA_NUM_FENCE_LISTS32];
401 struct list_head olist;
402 unsigned size;
403 uint64_t gpu_addr;
404 void *cpu_ptr;
405 uint32_t domain;
406 uint32_t align;
407};
408
409/* sub-allocation buffer */
410struct amdgpu_sa_bo {
411 struct list_head olist;
412 struct list_head flist;
413 struct amdgpu_sa_manager *manager;
414 unsigned soffset;
415 unsigned eoffset;
416 struct dma_fence *fence;
417};
418
419int amdgpu_fence_slab_init(void);
420void amdgpu_fence_slab_fini(void);
421
422/*
423 * IRQS.
424 */
425
426struct amdgpu_flip_work {
427 struct delayed_work flip_work;
428 struct work_struct unpin_work;
429 struct amdgpu_device *adev;
430 int crtc_id;
431 u32 target_vblank;
432 uint64_t base;
433 struct drm_pending_vblank_event *event;
434 struct amdgpu_bo *old_abo;
435 struct dma_fence *excl;
436 unsigned shared_count;
437 struct dma_fence **shared;
438 struct dma_fence_cb cb;
439 bool_Bool async;
440};
441
442
443/*
444 * CP & rings.
445 */
446
447struct amdgpu_ib {
448 struct amdgpu_sa_bo *sa_bo;
449 uint32_t length_dw;
450 uint64_t gpu_addr;
451 uint32_t *ptr;
452 uint32_t flags;
453};
454
455extern const struct drm_sched_backend_ops amdgpu_sched_ops;
456
457/*
458 * file private structure
459 */
460
461struct amdgpu_fpriv {
462 struct amdgpu_vm vm;
463 struct amdgpu_bo_va *prt_va;
464 struct amdgpu_bo_va *csa_va;
465 struct rwlock bo_list_lock;
466 struct idr bo_list_handles;
467 struct amdgpu_ctx_mgr ctx_mgr;
468};
469
470int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
471
472int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
473 unsigned size,
474 enum amdgpu_ib_pool_type pool,
475 struct amdgpu_ib *ib);
476void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
477 struct dma_fence *f);
478int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
479 struct amdgpu_ib *ibs, struct amdgpu_job *job,
480 struct dma_fence **f);
481int amdgpu_ib_pool_init(struct amdgpu_device *adev);
482void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
483int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
484
485/*
486 * CS.
487 */
488struct amdgpu_cs_chunk {
489 uint32_t chunk_id;
490 uint32_t length_dw;
491 void *kdata;
492};
493
494struct amdgpu_cs_post_dep {
495 struct drm_syncobj *syncobj;
496 struct dma_fence_chain *chain;
497 u64 point;
498};
499
500struct amdgpu_cs_parser {
501 struct amdgpu_device *adev;
502 struct drm_file *filp;
503 struct amdgpu_ctx *ctx;
504
505 /* chunks */
506 unsigned nchunks;
507 struct amdgpu_cs_chunk *chunks;
508
509 /* scheduler job object */
510 struct amdgpu_job *job;
511 struct drm_sched_entity *entity;
512
513 /* buffer objects */
514 struct ww_acquire_ctx ticket;
515 struct amdgpu_bo_list *bo_list;
516 struct amdgpu_mn *mn;
517 struct amdgpu_bo_list_entry vm_pd;
518 struct list_head validated;
519 struct dma_fence *fence;
520 uint64_t bytes_moved_threshold;
521 uint64_t bytes_moved_vis_threshold;
522 uint64_t bytes_moved;
523 uint64_t bytes_moved_vis;
524
525 /* user fence */
526 struct amdgpu_bo_list_entry uf_entry;
527
528 unsigned num_post_deps;
529 struct amdgpu_cs_post_dep *post_deps;
530};
531
532static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p,
533 uint32_t ib_idx, int idx)
534{
535 return p->job->ibs[ib_idx].ptr[idx];
536}
537
538static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p,
539 uint32_t ib_idx, int idx,
540 uint32_t value)
541{
542 p->job->ibs[ib_idx].ptr[idx] = value;
543}
544
545/*
546 * Writeback
547 */
548#define AMDGPU_MAX_WB256 256 /* Reserve at most 256 WB slots for amdgpu-owned rings. */
549
550struct amdgpu_wb {
551 struct amdgpu_bo *wb_obj;
552 volatile uint32_t *wb;
553 uint64_t gpu_addr;
554 u32 num_wb; /* Number of wb slots actually reserved for amdgpu. */
555 unsigned long used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)(((256) + ((64) - 1)) / (64))];
556};
557
558int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb);
559void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb);
560
561/*
562 * Benchmarking
563 */
564void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
565
566
567/*
568 * Testing
569 */
570void amdgpu_test_moves(struct amdgpu_device *adev);
571
572/*
573 * ASIC specific register table accessible by UMD
574 */
575struct amdgpu_allowed_register_entry {
576 uint32_t reg_offset;
577 bool_Bool grbm_indexed;
578};
579
580enum amd_reset_method {
581 AMD_RESET_METHOD_LEGACY = 0,
582 AMD_RESET_METHOD_MODE0,
583 AMD_RESET_METHOD_MODE1,
584 AMD_RESET_METHOD_MODE2,
585 AMD_RESET_METHOD_BACO
586};
587
588/*
589 * ASIC specific functions.
590 */
591struct amdgpu_asic_funcs {
592 bool_Bool (*read_disabled_bios)(struct amdgpu_device *adev);
593 bool_Bool (*read_bios_from_rom)(struct amdgpu_device *adev,
594 u8 *bios, u32 length_bytes);
595 int (*read_register)(struct amdgpu_device *adev, u32 se_num,
596 u32 sh_num, u32 reg_offset, u32 *value);
597 void (*set_vga_state)(struct amdgpu_device *adev, bool_Bool state);
598 int (*reset)(struct amdgpu_device *adev);
599 enum amd_reset_method (*reset_method)(struct amdgpu_device *adev);
600 /* get the reference clock */
601 u32 (*get_xclk)(struct amdgpu_device *adev);
602 /* MM block clocks */
603 int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
604 int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
605 /* static power management */
606 int (*get_pcie_lanes)(struct amdgpu_device *adev);
607 void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes);
608 /* get config memsize register */
609 u32 (*get_config_memsize)(struct amdgpu_device *adev);
610 /* flush hdp write queue */
611 void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring);
612 /* invalidate hdp read cache */
613 void (*invalidate_hdp)(struct amdgpu_device *adev,
614 struct amdgpu_ring *ring);
615 void (*reset_hdp_ras_error_count)(struct amdgpu_device *adev);
616 /* check if the asic needs a full reset of if soft reset will work */
617 bool_Bool (*need_full_reset)(struct amdgpu_device *adev);
618 /* initialize doorbell layout for specific asic*/
619 void (*init_doorbell_index)(struct amdgpu_device *adev);
620 /* PCIe bandwidth usage */
621 void (*get_pcie_usage)(struct amdgpu_device *adev, uint64_t *count0,
622 uint64_t *count1);
623 /* do we need to reset the asic at init time (e.g., kexec) */
624 bool_Bool (*need_reset_on_init)(struct amdgpu_device *adev);
625 /* PCIe replay counter */
626 uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev);
627 /* device supports BACO */
628 bool_Bool (*supports_baco)(struct amdgpu_device *adev);
629 /* pre asic_init quirks */
630 void (*pre_asic_init)(struct amdgpu_device *adev);
631};
632
633/*
634 * IOCTL.
635 */
636int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
637 struct drm_file *filp);
638
639int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
640int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
641 struct drm_file *filp);
642int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
643int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
644 struct drm_file *filp);
645
646/* VRAM scratch page for HDP bug, default vram page */
647struct amdgpu_vram_scratch {
648 struct amdgpu_bo *robj;
649 volatile uint32_t *ptr;
650 u64 gpu_addr;
651};
652
653/*
654 * ACPI
655 */
656struct amdgpu_atcs_functions {
657 bool_Bool get_ext_state;
658 bool_Bool pcie_perf_req;
659 bool_Bool pcie_dev_rdy;
660 bool_Bool pcie_bus_width;
661};
662
663struct amdgpu_atcs {
664 struct amdgpu_atcs_functions functions;
665};
666
667/*
668 * CGS
669 */
670struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
671void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device);
672
673/*
674 * Core structure, functions and helpers.
675 */
676typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device*, uint32_t);
677typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
678
679typedef uint64_t (*amdgpu_rreg64_t)(struct amdgpu_device*, uint32_t);
680typedef void (*amdgpu_wreg64_t)(struct amdgpu_device*, uint32_t, uint64_t);
681
682typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
683typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
684
685struct amdgpu_mmio_remap {
686 u32 reg_offset;
687 resource_size_t bus_addr;
688};
689
690/* Define the HW IP blocks will be used in driver , add more if necessary */
691enum amd_hw_ip_block_type {
692 GC_HWIP = 1,
693 HDP_HWIP,
694 SDMA0_HWIP,
695 SDMA1_HWIP,
696 SDMA2_HWIP,
697 SDMA3_HWIP,
698 SDMA4_HWIP,
699 SDMA5_HWIP,
700 SDMA6_HWIP,
701 SDMA7_HWIP,
702 MMHUB_HWIP,
703 ATHUB_HWIP,
704 NBIO_HWIP,
705 MP0_HWIP,
706 MP1_HWIP,
707 UVD_HWIP,
708 VCN_HWIP = UVD_HWIP,
709 JPEG_HWIP = VCN_HWIP,
710 VCE_HWIP,
711 DF_HWIP,
712 DCE_HWIP,
713 OSSSYS_HWIP,
714 SMUIO_HWIP,
715 PWR_HWIP,
716 NBIF_HWIP,
717 THM_HWIP,
718 CLK_HWIP,
719 UMC_HWIP,
720 RSMU_HWIP,
721 MAX_HWIP
722};
723
724#define HWIP_MAX_INSTANCE10 10
725
726struct amd_powerplay {
727 void *pp_handle;
728 const struct amd_pm_funcs *pp_funcs;
729};
730
731#define AMDGPU_RESET_MAGIC_NUM64 64
732#define AMDGPU_MAX_DF_PERFMONS4 4
733struct amdgpu_device {
734 struct device self;
735 struct device *dev;
736 struct pci_dev *pdev;
737 struct drm_device ddev;
738
739 pci_chipset_tag_t pc;
740 pcitag_t pa_tag;
741 pci_intr_handle_t intrh;
742 bus_space_tag_t iot;
743 bus_space_tag_t memt;
744 bus_dma_tag_t dmat;
745 void *irqh;
746
747 void (*switchcb)(void *, int, int);
748 void *switchcbarg;
749 void *switchcookie;
750 struct task switchtask;
751 struct rasops_info ro;
752 int console;
753 int primary;
754
755 struct task burner_task;
756 int burner_fblank;
757
758 unsigned long fb_aper_offset;
759 unsigned long fb_aper_size;
760
761#ifdef CONFIG_DRM_AMD_ACP
762 struct amdgpu_acp acp;
763#endif
764 struct amdgpu_hive_info *hive;
765 /* ASIC */
766 enum amd_asic_type asic_type;
767 uint32_t family;
768 uint32_t rev_id;
769 uint32_t external_rev_id;
770 unsigned long flags;
771 unsigned long apu_flags;
772 int usec_timeout;
773 const struct amdgpu_asic_funcs *asic_funcs;
774 bool_Bool shutdown;
775 bool_Bool need_swiotlb;
776 bool_Bool accel_working;
777 struct notifier_block acpi_nb;
778 struct amdgpu_i2c_chan *i2c_bus[AMDGPU_MAX_I2C_BUS16];
779 struct amdgpu_debugfs debugfs[AMDGPU_DEBUGFS_MAX_COMPONENTS32];
780 unsigned debugfs_count;
781#if defined(CONFIG_DEBUG_FS)
782 struct dentry *debugfs_preempt;
783 struct dentry *debugfs_regs[AMDGPU_DEBUGFS_MAX_COMPONENTS32];
784#endif
785 struct amdgpu_atif *atif;
786 struct amdgpu_atcs atcs;
787 struct rwlock srbm_mutex;
788 /* GRBM index mutex. Protects concurrent access to GRBM index */
789 struct rwlock grbm_idx_mutex;
790 struct dev_pm_domain vga_pm_domain;
791 bool_Bool have_disp_power_ref;
792 bool_Bool have_atomics_support;
793
794 /* BIOS */
795 bool_Bool is_atom_fw;
796 uint8_t *bios;
797 uint32_t bios_size;
798 uint32_t bios_scratch_reg_offset;
799 uint32_t bios_scratch[AMDGPU_BIOS_NUM_SCRATCH16];
800
801 /* Register/doorbell mmio */
802 resource_size_t rmmio_base;
803 resource_size_t rmmio_size;
804 void __iomem *rmmio;
805 bus_space_tag_t rmmio_bst;
806 bus_space_handle_t rmmio_bsh;
807 /* protects concurrent MM_INDEX/DATA based register access */
808 spinlock_t mmio_idx_lock;
809 struct amdgpu_mmio_remap rmmio_remap;
810 /* protects concurrent SMC based register access */
811 spinlock_t smc_idx_lock;
812 amdgpu_rreg_t smc_rreg;
813 amdgpu_wreg_t smc_wreg;
814 /* protects concurrent PCIE register access */
815 spinlock_t pcie_idx_lock;
816 amdgpu_rreg_t pcie_rreg;
817 amdgpu_wreg_t pcie_wreg;
818 amdgpu_rreg_t pciep_rreg;
819 amdgpu_wreg_t pciep_wreg;
820 amdgpu_rreg64_t pcie_rreg64;
821 amdgpu_wreg64_t pcie_wreg64;
822 /* protects concurrent UVD register access */
823 spinlock_t uvd_ctx_idx_lock;
824 amdgpu_rreg_t uvd_ctx_rreg;
825 amdgpu_wreg_t uvd_ctx_wreg;
826 /* protects concurrent DIDT register access */
827 spinlock_t didt_idx_lock;
828 amdgpu_rreg_t didt_rreg;
829 amdgpu_wreg_t didt_wreg;
830 /* protects concurrent gc_cac register access */
831 spinlock_t gc_cac_idx_lock;
832 amdgpu_rreg_t gc_cac_rreg;
833 amdgpu_wreg_t gc_cac_wreg;
834 /* protects concurrent se_cac register access */
835 spinlock_t se_cac_idx_lock;
836 amdgpu_rreg_t se_cac_rreg;
837 amdgpu_wreg_t se_cac_wreg;
838 /* protects concurrent ENDPOINT (audio) register access */
839 spinlock_t audio_endpt_idx_lock;
840 amdgpu_block_rreg_t audio_endpt_rreg;
841 amdgpu_block_wreg_t audio_endpt_wreg;
842#ifdef notyet
843 void __iomem *rio_mem;
844#endif
845 bus_space_tag_t rio_mem_bst;
846 bus_space_handle_t rio_mem_bsh;
847 resource_size_t rio_mem_size;
848 struct amdgpu_doorbell doorbell;
849
850 /* clock/pll info */
851 struct amdgpu_clock clock;
852
853 /* MC */
854 struct amdgpu_gmc gmc;
855 struct amdgpu_gart gart;
856 dma_addr_t dummy_page_addr;
857 struct amdgpu_vm_manager vm_manager;
858 struct amdgpu_vmhub vmhub[AMDGPU_MAX_VMHUBS3];
859 unsigned num_vmhubs;
860
861 /* memory management */
862 struct amdgpu_mman mman;
863 struct amdgpu_vram_scratch vram_scratch;
864 struct amdgpu_wb wb;
865 atomic64_t num_bytes_moved;
866 atomic64_t num_evictions;
867 atomic64_t num_vram_cpu_page_faults;
868 atomic_t gpu_reset_counter;
869 atomic_t vram_lost_counter;
870
871 /* data for buffer migration throttling */
872 struct {
873 spinlock_t lock;
874 s64 last_update_us;
875 s64 accum_us; /* accumulated microseconds */
876 s64 accum_us_vis; /* for visible VRAM */
877 u32 log2_max_MBps;
878 } mm_stats;
879
880 /* display */
881 bool_Bool enable_virtual_display;
882 struct amdgpu_mode_info mode_info;
883 /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */
884 struct work_struct hotplug_work;
885 struct amdgpu_irq_src crtc_irq;
886 struct amdgpu_irq_src vupdate_irq;
887 struct amdgpu_irq_src pageflip_irq;
888 struct amdgpu_irq_src hpd_irq;
889
890 /* rings */
891 u64 fence_context;
892 unsigned num_rings;
893 struct amdgpu_ring *rings[AMDGPU_MAX_RINGS28];
894 bool_Bool ib_pool_ready;
895 struct amdgpu_sa_manager ib_pools[AMDGPU_IB_POOL_MAX];
896 struct amdgpu_sched gpu_sched[AMDGPU_HW_IP_NUM9][AMDGPU_RING_PRIO_MAXAMDGPU_GFX_PIPE_PRIO_MAX];
897
898 /* interrupts */
899 struct amdgpu_irq irq;
900
901 /* powerplay */
902 struct amd_powerplay powerplay;
903 bool_Bool pp_force_state_enabled;
904
905 /* smu */
906 struct smu_context smu;
907
908 /* dpm */
909 struct amdgpu_pm pm;
910 u32 cg_flags;
911 u32 pg_flags;
912
913 /* nbio */
914 struct amdgpu_nbio nbio;
915
916 /* mmhub */
917 struct amdgpu_mmhub mmhub;
918
919 /* gfxhub */
920 struct amdgpu_gfxhub gfxhub;
921
922 /* gfx */
923 struct amdgpu_gfx gfx;
924
925 /* sdma */
926 struct amdgpu_sdma sdma;
927
928 /* uvd */
929 struct amdgpu_uvd uvd;
930
931 /* vce */
932 struct amdgpu_vce vce;
933
934 /* vcn */
935 struct amdgpu_vcn vcn;
936
937 /* jpeg */
938 struct amdgpu_jpeg jpeg;
939
940 /* firmwares */
941 struct amdgpu_firmware firmware;
942
943 /* PSP */
944 struct psp_context psp;
945
946 /* GDS */
947 struct amdgpu_gds gds;
948
949 /* KFD */
950 struct amdgpu_kfd_dev kfd;
951
952 /* UMC */
953 struct amdgpu_umc umc;
954
955 /* display related functionality */
956 struct amdgpu_display_manager dm;
957
958 /* mes */
959 bool_Bool enable_mes;
960 struct amdgpu_mes mes;
961
962 /* df */
963 struct amdgpu_df df;
964
965 struct amdgpu_ip_block ip_blocks[AMDGPU_MAX_IP_NUM16];
966 int num_ip_blocks;
967 struct rwlock mn_lock;
968 DECLARE_HASHTABLE(mn_hash, 7)struct hlist_head mn_hash[1 << (7)];
969
970 /* tracking pinned memory */
971 atomic64_t vram_pin_size;
972 atomic64_t visible_pin_size;
973 atomic64_t gart_pin_size;
974
975 /* soc15 register offset based on ip, instance and segment */
976 uint32_t *reg_offset[MAX_HWIP][HWIP_MAX_INSTANCE10];
977
978 /* delayed work_func for deferring clockgating during resume */
979 struct delayed_work delayed_init_work;
980
981 struct amdgpu_virt virt;
982
983 /* link all shadow bo */
984 struct list_head shadow_list;
985 struct rwlock shadow_list_lock;
986
987 /* record hw reset is performed */
988 bool_Bool has_hw_reset;
989 u8 reset_magic[AMDGPU_RESET_MAGIC_NUM64];
990
991 /* s3/s4 mask */
992 bool_Bool in_suspend;
993 bool_Bool in_hibernate;
994
995 atomic_t in_gpu_reset;
996 enum pp_mp1_state mp1_state;
997 struct rwlock reset_sem;
998 struct amdgpu_doorbell_index doorbell_index;
999
1000 struct rwlock notifier_lock;
1001
1002 int asic_reset_res;
1003 struct work_struct xgmi_reset_work;
1004
1005 long gfx_timeout;
1006 long sdma_timeout;
1007 long video_timeout;
1008 long compute_timeout;
1009
1010 uint64_t unique_id;
1011 uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS4];
1012
1013 /* enable runtime pm on the device */
1014 bool_Bool runpm;
1015 bool_Bool in_runpm;
1016
1017 bool_Bool pm_sysfs_en;
1018 bool_Bool ucode_sysfs_en;
1019
1020 /* Chip product information */
1021 char product_number[16];
1022 char product_name[32];
1023 char serial[20];
1024
1025 struct amdgpu_autodump autodump;
1026
1027 atomic_t throttling_logging_enabled;
1028#ifdef notyet
1029 struct ratelimit_state throttling_logging_rs;
1030#endif
1031 uint32_t ras_features;
1032
1033 bool_Bool in_pci_err_recovery;
1034 struct pci_saved_state *pci_state;
1035};
1036
1037static inline struct amdgpu_device *drm_to_adev(struct drm_device *ddev)
1038{
1039 return container_of(ddev, struct amdgpu_device, ddev)({ const __typeof( ((struct amdgpu_device *)0)->ddev ) *__mptr
= (ddev); (struct amdgpu_device *)( (char *)__mptr - __builtin_offsetof
(struct amdgpu_device, ddev) );});
1040}
1041
1042static inline struct drm_device *adev_to_drm(struct amdgpu_device *adev)
1043{
1044 return &adev->ddev;
2
←
Returning without writing to 'adev->pm.pcie_gen_mask', which participates in a condition later
→
3
←
Returning without writing to 'adev->pm.pcie_mlw_mask', which participates in a condition later
→
1045}
1046
1047static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
1048{
1049 return container_of(bdev, struct amdgpu_device, mman.bdev)({ const __typeof( ((struct amdgpu_device *)0)->mman.bdev )
*__mptr = (bdev); (struct amdgpu_device *)( (char *)__mptr -
__builtin_offsetof(struct amdgpu_device, mman.bdev) );});
1050}
1051
1052int amdgpu_device_init(struct amdgpu_device *adev,
1053 uint32_t flags);
1054void amdgpu_device_fini(struct amdgpu_device *adev);
1055int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev);
1056
1057void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
1058 uint32_t *buf, size_t size, bool_Bool write);
1059uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
1060 uint32_t reg, uint32_t acc_flags);
1061void amdgpu_device_wreg(struct amdgpu_device *adev,
1062 uint32_t reg, uint32_t v,
1063 uint32_t acc_flags);
1064void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
1065 uint32_t reg, uint32_t v);
1066void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value);
1067uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset);
1068
1069u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg);
1070void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v);
1071
1072u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
1073 u32 pcie_index, u32 pcie_data,
1074 u32 reg_addr);
1075u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
1076 u32 pcie_index, u32 pcie_data,
1077 u32 reg_addr);
1078void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
1079 u32 pcie_index, u32 pcie_data,
1080 u32 reg_addr, u32 reg_data);
1081void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
1082 u32 pcie_index, u32 pcie_data,
1083 u32 reg_addr, u64 reg_data);
1084
1085bool_Bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);
1086bool_Bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);
1087
1088int emu_soc_asic_init(struct amdgpu_device *adev);
1089
1090/*
1091 * Registers read & write functions.
1092 */
1093#define AMDGPU_REGS_NO_KIQ(1<<1) (1<<1)
1094
1095#define RREG32_NO_KIQ(reg)amdgpu_device_rreg(adev, (reg), (1<<1)) amdgpu_device_rreg(adev, (reg), AMDGPU_REGS_NO_KIQ(1<<1))
1096#define WREG32_NO_KIQ(reg, v)amdgpu_device_wreg(adev, (reg), (v), (1<<1)) amdgpu_device_wreg(adev, (reg), (v), AMDGPU_REGS_NO_KIQ(1<<1))
1097
1098#define RREG32_KIQ(reg)amdgpu_kiq_rreg(adev, (reg)) amdgpu_kiq_rreg(adev, (reg))
1099#define WREG32_KIQ(reg, v)amdgpu_kiq_wreg(adev, (reg), (v)) amdgpu_kiq_wreg(adev, (reg), (v))
1100
1101#define RREG8(reg)amdgpu_mm_rreg8(adev, (reg)) amdgpu_mm_rreg8(adev, (reg))
1102#define WREG8(reg, v)amdgpu_mm_wreg8(adev, (reg), (v)) amdgpu_mm_wreg8(adev, (reg), (v))
1103
1104#define RREG32(reg)amdgpu_device_rreg(adev, (reg), 0) amdgpu_device_rreg(adev, (reg), 0)
1105#define DREG32(reg)printk("\0016" "REGISTER: " "reg" " : 0x%08X\n", amdgpu_device_rreg
(adev, (reg), 0)) printk(KERN_INFO"\0016" "REGISTER: " #reg " : 0x%08X\n", amdgpu_device_rreg(adev, (reg), 0))
1106#define WREG32(reg, v)amdgpu_device_wreg(adev, (reg), (v), 0) amdgpu_device_wreg(adev, (reg), (v), 0)
1107#define REG_SET(FIELD, v)(((v) << FIELD_SHIFT) & FIELD_MASK) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1108#define REG_GET(FIELD, v)(((v) << FIELD_SHIFT) & FIELD_MASK) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1109#define RREG32_PCIE(reg)adev->pcie_rreg(adev, (reg)) adev->pcie_rreg(adev, (reg))
1110#define WREG32_PCIE(reg, v)adev->pcie_wreg(adev, (reg), (v)) adev->pcie_wreg(adev, (reg), (v))
1111#define RREG32_PCIE_PORT(reg)adev->pciep_rreg(adev, (reg)) adev->pciep_rreg(adev, (reg))
1112#define WREG32_PCIE_PORT(reg, v)adev->pciep_wreg(adev, (reg), (v)) adev->pciep_wreg(adev, (reg), (v))
1113#define RREG64_PCIE(reg)adev->pcie_rreg64(adev, (reg)) adev->pcie_rreg64(adev, (reg))
1114#define WREG64_PCIE(reg, v)adev->pcie_wreg64(adev, (reg), (v)) adev->pcie_wreg64(adev, (reg), (v))
1115#define RREG32_SMC(reg)adev->smc_rreg(adev, (reg)) adev->smc_rreg(adev, (reg))
1116#define WREG32_SMC(reg, v)adev->smc_wreg(adev, (reg), (v)) adev->smc_wreg(adev, (reg), (v))
1117#define RREG32_UVD_CTX(reg)adev->uvd_ctx_rreg(adev, (reg)) adev->uvd_ctx_rreg(adev, (reg))
1118#define WREG32_UVD_CTX(reg, v)adev->uvd_ctx_wreg(adev, (reg), (v)) adev->uvd_ctx_wreg(adev, (reg), (v))
1119#define RREG32_DIDT(reg)adev->didt_rreg(adev, (reg)) adev->didt_rreg(adev, (reg))
1120#define WREG32_DIDT(reg, v)adev->didt_wreg(adev, (reg), (v)) adev->didt_wreg(adev, (reg), (v))
1121#define RREG32_GC_CAC(reg)adev->gc_cac_rreg(adev, (reg)) adev->gc_cac_rreg(adev, (reg))
1122#define WREG32_GC_CAC(reg, v)adev->gc_cac_wreg(adev, (reg), (v)) adev->gc_cac_wreg(adev, (reg), (v))
1123#define RREG32_SE_CAC(reg)adev->se_cac_rreg(adev, (reg)) adev->se_cac_rreg(adev, (reg))
1124#define WREG32_SE_CAC(reg, v)adev->se_cac_wreg(adev, (reg), (v)) adev->se_cac_wreg(adev, (reg), (v))
1125#define RREG32_AUDIO_ENDPT(block, reg)adev->audio_endpt_rreg(adev, (block), (reg)) adev->audio_endpt_rreg(adev, (block), (reg))
1126#define WREG32_AUDIO_ENDPT(block, reg, v)adev->audio_endpt_wreg(adev, (block), (reg), (v)) adev->audio_endpt_wreg(adev, (block), (reg), (v))
1127#define WREG32_P(reg, val, mask)do { uint32_t tmp_ = amdgpu_device_rreg(adev, (reg), 0); tmp_
&= (mask); tmp_ |= ((val) & ~(mask)); amdgpu_device_wreg
(adev, (reg), (tmp_), 0); } while (0) \
1128 do { \
1129 uint32_t tmp_ = RREG32(reg)amdgpu_device_rreg(adev, (reg), 0); \
1130 tmp_ &= (mask); \
1131 tmp_ |= ((val) & ~(mask)); \
1132 WREG32(reg, tmp_)amdgpu_device_wreg(adev, (reg), (tmp_), 0); \
1133 } while (0)
1134#define WREG32_AND(reg, and)do { uint32_t tmp_ = amdgpu_device_rreg(adev, (reg), 0); tmp_
&= (and); tmp_ |= ((0) & ~(and)); amdgpu_device_wreg
(adev, (reg), (tmp_), 0); } while (0) WREG32_P(reg, 0, and)do { uint32_t tmp_ = amdgpu_device_rreg(adev, (reg), 0); tmp_
&= (and); tmp_ |= ((0) & ~(and)); amdgpu_device_wreg
(adev, (reg), (tmp_), 0); } while (0)
1135#define WREG32_OR(reg, or)do { uint32_t tmp_ = amdgpu_device_rreg(adev, (reg), 0); tmp_
&= (~(or)); tmp_ |= ((or) & ~(~(or))); amdgpu_device_wreg
(adev, (reg), (tmp_), 0); } while (0) WREG32_P(reg, or, ~(or))do { uint32_t tmp_ = amdgpu_device_rreg(adev, (reg), 0); tmp_
&= (~(or)); tmp_ |= ((or) & ~(~(or))); amdgpu_device_wreg
(adev, (reg), (tmp_), 0); } while (0)
1136#define WREG32_PLL_P(reg, val, mask)do { uint32_t tmp_ = RREG32_PLL(reg); tmp_ &= (mask); tmp_
|= ((val) & ~(mask)); WREG32_PLL(reg, tmp_); } while (0) \
1137 do { \
1138 uint32_t tmp_ = RREG32_PLL(reg); \
1139 tmp_ &= (mask); \
1140 tmp_ |= ((val) & ~(mask)); \
1141 WREG32_PLL(reg, tmp_); \
1142 } while (0)
1143
1144#define WREG32_SMC_P(_Reg, _Val, _Mask)do { u32 tmp = adev->smc_rreg(adev, (_Reg)); tmp &= (_Mask
); tmp |= ((_Val) & ~(_Mask)); adev->smc_wreg(adev, (_Reg
), (tmp)); } while (0) \
1145 do { \
1146 u32 tmp = RREG32_SMC(_Reg)adev->smc_rreg(adev, (_Reg)); \
1147 tmp &= (_Mask); \
1148 tmp |= ((_Val) & ~(_Mask)); \
1149 WREG32_SMC(_Reg, tmp)adev->smc_wreg(adev, (_Reg), (tmp)); \
1150 } while (0)
1151
1152#define DREG32_SYS(sqf, adev, reg)seq_printf((sqf), "reg" " : 0x%08X\n", amdgpu_device_rreg((adev
), (reg), 0)) seq_printf((sqf), #reg " : 0x%08X\n", amdgpu_device_rreg((adev), (reg), false0))
1153#define RREG32_IO(reg)amdgpu_io_rreg(adev, (reg)) amdgpu_io_rreg(adev, (reg))
1154#define WREG32_IO(reg, v)amdgpu_io_wreg(adev, (reg), (v)) amdgpu_io_wreg(adev, (reg), (v))
1155
1156#define REG_FIELD_SHIFT(reg, field)reg__field__SHIFT reg##__##field##__SHIFT
1157#define REG_FIELD_MASK(reg, field)reg__field_MASK reg##__##field##_MASK
1158
1159#define REG_SET_FIELD(orig_val, reg, field, field_val)(((orig_val) & ~reg__field_MASK) | (reg__field_MASK &
((field_val) << reg__field__SHIFT))) \
1160 (((orig_val) & ~REG_FIELD_MASK(reg, field)reg__field_MASK) | \
1161 (REG_FIELD_MASK(reg, field)reg__field_MASK & ((field_val) << REG_FIELD_SHIFT(reg, field)reg__field__SHIFT)))
1162
1163#define REG_GET_FIELD(value, reg, field)(((value) & reg__field_MASK) >> reg__field__SHIFT) \
1164 (((value) & REG_FIELD_MASK(reg, field)reg__field_MASK) >> REG_FIELD_SHIFT(reg, field)reg__field__SHIFT)
1165
1166#define WREG32_FIELD(reg, field, val)amdgpu_device_wreg(adev, (mmreg), ((amdgpu_device_rreg(adev, (
mmreg), 0) & ~reg__field_MASK) | (val) << reg__field__SHIFT
), 0) \
1167 WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))amdgpu_device_wreg(adev, (mm##reg), ((amdgpu_device_rreg(adev
, (mm##reg), 0) & ~reg__field_MASK) | (val) << reg__field__SHIFT
), 0)
1168
1169#define WREG32_FIELD_OFFSET(reg, offset, field, val)amdgpu_device_wreg(adev, (mmreg + offset), ((amdgpu_device_rreg
(adev, (mmreg + offset), 0) & ~reg__field_MASK) | (val) <<
reg__field__SHIFT), 0) \
1170 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))amdgpu_device_wreg(adev, (mm##reg + offset), ((amdgpu_device_rreg
(adev, (mm##reg + offset), 0) & ~reg__field_MASK) | (val)
<< reg__field__SHIFT), 0)
1171
1172/*
1173 * BIOS helpers.
1174 */
1175#define RBIOS8(i)(adev->bios[i]) (adev->bios[i])
1176#define RBIOS16(i)((adev->bios[i]) | ((adev->bios[(i)+1]) << 8)) (RBIOS8(i)(adev->bios[i]) | (RBIOS8((i)+1)(adev->bios[(i)+1]) << 8))
1177#define RBIOS32(i)((((adev->bios[i]) | ((adev->bios[(i)+1]) << 8)))
| (((adev->bios[(i)+2]) | ((adev->bios[((i)+2)+1]) <<
8)) << 16)) ((RBIOS16(i)((adev->bios[i]) | ((adev->bios[(i)+1]) << 8))) | (RBIOS16((i)+2)((adev->bios[(i)+2]) | ((adev->bios[((i)+2)+1]) <<
8)) << 16))
1178
1179/*
1180 * ASICs macro.
1181 */
1182#define amdgpu_asic_set_vga_state(adev, state)(adev)->asic_funcs->set_vga_state((adev), (state)) (adev)->asic_funcs->set_vga_state((adev), (state))
1183#define amdgpu_asic_reset(adev)(adev)->asic_funcs->reset((adev)) (adev)->asic_funcs->reset((adev))
1184#define amdgpu_asic_reset_method(adev)(adev)->asic_funcs->reset_method((adev)) (adev)->asic_funcs->reset_method((adev))
1185#define amdgpu_asic_get_xclk(adev)(adev)->asic_funcs->get_xclk((adev)) (adev)->asic_funcs->get_xclk((adev))
1186#define amdgpu_asic_set_uvd_clocks(adev, v, d)(adev)->asic_funcs->set_uvd_clocks((adev), (v), (d)) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
1187#define amdgpu_asic_set_vce_clocks(adev, ev, ec)(adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec)) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
1188#define amdgpu_get_pcie_lanes(adev)(adev)->asic_funcs->get_pcie_lanes((adev)) (adev)->asic_funcs->get_pcie_lanes((adev))
1189#define amdgpu_set_pcie_lanes(adev, l)(adev)->asic_funcs->set_pcie_lanes((adev), (l)) (adev)->asic_funcs->set_pcie_lanes((adev), (l))
1190#define amdgpu_asic_get_gpu_clock_counter(adev)(adev)->asic_funcs->get_gpu_clock_counter((adev)) (adev)->asic_funcs->get_gpu_clock_counter((adev))
1191#define amdgpu_asic_read_disabled_bios(adev)(adev)->asic_funcs->read_disabled_bios((adev)) (adev)->asic_funcs->read_disabled_bios((adev))
1192#define amdgpu_asic_read_bios_from_rom(adev, b, l)(adev)->asic_funcs->read_bios_from_rom((adev), (b), (l)
) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
1193#define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (
offset), (v)))((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
1194#define amdgpu_asic_get_config_memsize(adev)(adev)->asic_funcs->get_config_memsize((adev)) (adev)->asic_funcs->get_config_memsize((adev))
1195#define amdgpu_asic_flush_hdp(adev, r)(adev)->asic_funcs->flush_hdp((adev), (r)) (adev)->asic_funcs->flush_hdp((adev), (r))
1196#define amdgpu_asic_invalidate_hdp(adev, r)(adev)->asic_funcs->invalidate_hdp((adev), (r)) (adev)->asic_funcs->invalidate_hdp((adev), (r))
1197#define amdgpu_asic_need_full_reset(adev)(adev)->asic_funcs->need_full_reset((adev)) (adev)->asic_funcs->need_full_reset((adev))
1198#define amdgpu_asic_init_doorbell_index(adev)(adev)->asic_funcs->init_doorbell_index((adev)) (adev)->asic_funcs->init_doorbell_index((adev))
1199#define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1)((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1
))) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
1200#define amdgpu_asic_need_reset_on_init(adev)(adev)->asic_funcs->need_reset_on_init((adev)) (adev)->asic_funcs->need_reset_on_init((adev))
1201#define amdgpu_asic_get_pcie_replay_count(adev)((adev)->asic_funcs->get_pcie_replay_count((adev))) ((adev)->asic_funcs->get_pcie_replay_count((adev)))
1202#define amdgpu_asic_supports_baco(adev)(adev)->asic_funcs->supports_baco((adev)) (adev)->asic_funcs->supports_baco((adev))
1203#define amdgpu_asic_pre_asic_init(adev)(adev)->asic_funcs->pre_asic_init((adev)) (adev)->asic_funcs->pre_asic_init((adev))
1204
1205#define amdgpu_inc_vram_lost(adev)__sync_fetch_and_add(&((adev)->vram_lost_counter), 1); atomic_inc(&((adev)->vram_lost_counter))__sync_fetch_and_add(&((adev)->vram_lost_counter), 1);
1206
1207/* Common functions */
1208bool_Bool amdgpu_device_has_job_running(struct amdgpu_device *adev);
1209bool_Bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev);
1210int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
1211 struct amdgpu_job* job);
1212void amdgpu_device_pci_config_reset(struct amdgpu_device *adev);
1213bool_Bool amdgpu_device_need_post(struct amdgpu_device *adev);
1214
1215void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
1216 u64 num_vis_bytes);
1217int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev);
1218void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1219 const u32 *registers,
1220 const u32 array_size);
1221
1222bool_Bool amdgpu_device_supports_boco(struct drm_device *dev);
1223bool_Bool amdgpu_device_supports_baco(struct drm_device *dev);
1224bool_Bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
1225 struct amdgpu_device *peer_adev);
1226int amdgpu_device_baco_enter(struct drm_device *dev);
1227int amdgpu_device_baco_exit(struct drm_device *dev);
1228
1229/* atpx handler */
1230#if defined(CONFIG_VGA_SWITCHEROO)
1231void amdgpu_register_atpx_handler(void);
1232void amdgpu_unregister_atpx_handler(void);
1233bool_Bool amdgpu_has_atpx_dgpu_power_cntl(void);
1234bool_Bool amdgpu_is_atpx_hybrid(void);
1235bool_Bool amdgpu_atpx_dgpu_req_power_for_displays(void);
1236bool_Bool amdgpu_has_atpx(void);
1237#else
1238static inline void amdgpu_register_atpx_handler(void) {}
1239static inline void amdgpu_unregister_atpx_handler(void) {}
1240static inline bool_Bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false0; }
1241static inline bool_Bool amdgpu_is_atpx_hybrid(void) { return false0; }
1242static inline bool_Bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false0; }
1243static inline bool_Bool amdgpu_has_atpx(void) { return false0; }
1244#endif
1245
1246#if defined(CONFIG_VGA_SWITCHEROO) && defined(CONFIG_ACPI1)
1247void *amdgpu_atpx_get_dhandle(void);
1248#else
1249static inline void *amdgpu_atpx_get_dhandle(void) { return NULL((void *)0); }
1250#endif
1251
1252/*
1253 * KMS
1254 */
1255extern const struct drm_ioctl_desc amdgpu_ioctls_kms[];
1256extern const int amdgpu_max_kms_ioctl;
1257
1258int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags);
1259void amdgpu_driver_unload_kms(struct drm_device *dev);
1260void amdgpu_driver_lastclose_kms(struct drm_device *dev);
1261int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv);
1262void amdgpu_driver_postclose_kms(struct drm_device *dev,
1263 struct drm_file *file_priv);
1264int amdgpu_device_ip_suspend(struct amdgpu_device *adev);
1265int amdgpu_device_suspend(struct drm_device *dev, bool_Bool fbcon);
1266int amdgpu_device_resume(struct drm_device *dev, bool_Bool fbcon);
1267u32 amdgpu_get_vblank_counter_kms(struct drm_crtc *crtc);
1268int amdgpu_enable_vblank_kms(struct drm_crtc *crtc);
1269void amdgpu_disable_vblank_kms(struct drm_crtc *crtc);
1270long amdgpu_kms_compat_ioctl(struct file *filp, unsigned int cmd,
1271 unsigned long arg);
1272
1273/*
1274 * functions used by amdgpu_encoder.c
1275 */
1276struct amdgpu_afmt_acr {
1277 u32 clock;
1278
1279 int n_32khz;
1280 int cts_32khz;
1281
1282 int n_44_1khz;
1283 int cts_44_1khz;
1284
1285 int n_48khz;
1286 int cts_48khz;
1287
1288};
1289
1290struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock);
1291
1292/* amdgpu_acpi.c */
1293#if defined(CONFIG_ACPI1)
1294int amdgpu_acpi_init(struct amdgpu_device *adev);
1295void amdgpu_acpi_fini(struct amdgpu_device *adev);
1296bool_Bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev);
1297int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
1298 u8 perf_req, bool_Bool advertise);
1299int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
1300
1301void amdgpu_acpi_get_backlight_caps(struct amdgpu_device *adev,
1302 struct amdgpu_dm_backlight_caps *caps);
1303#else
1304static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
1305static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
1306#endif
1307
1308int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1309 uint64_t addr, struct amdgpu_bo **bo,
1310 struct amdgpu_bo_va_mapping **mapping);
1311
1312#if defined(CONFIG_DRM_AMD_DC1)
1313int amdgpu_dm_display_resume(struct amdgpu_device *adev );
1314#else
1315static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return 0; }
1316#endif
1317
1318
1319void amdgpu_register_gpu_instance(struct amdgpu_device *adev);
1320void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev);
1321
1322pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev,
1323 pci_channel_state_t state);
1324pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev);
1325pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev);
1326void amdgpu_pci_resume(struct pci_dev *pdev);
1327
1328bool_Bool amdgpu_device_cache_pci_state(struct pci_dev *pdev);
1329bool_Bool amdgpu_device_load_pci_state(struct pci_dev *pdev);
1330
1331#include "amdgpu_object.h"
1332
1333/* used by df_v3_6.c and amdgpu_pmu.c */
1334#define AMDGPU_PMU_ATTR(_name, _object)static ssize_t _name_show(struct device *dev, struct device_attribute
*attr, char *page) { extern char _ctassert[(!(sizeof(_object
) >= (1 << 12) - 1)) ? 1 : -1 ] __attribute__((__unused__
)); return sprintf(page, _object "\n"); } static struct device_attribute
pmu_attr__name = __ATTR_RO(_name) \
1335static ssize_t \
1336_name##_show(struct device *dev, \
1337 struct device_attribute *attr, \
1338 char *page) \
1339{ \
1340 BUILD_BUG_ON(sizeof(_object) >= PAGE_SIZE - 1)extern char _ctassert[(!(sizeof(_object) >= (1 << 12
) - 1)) ? 1 : -1 ] __attribute__((__unused__)); \
1341 return sprintf(page, _object "\n"); \
1342} \
1343 \
1344static struct device_attribute pmu_attr_##_name = __ATTR_RO(_name)
1345
1346static inline bool_Bool amdgpu_is_tmz(struct amdgpu_device *adev)
1347{
1348 return adev->gmc.tmz_enabled;
1349}
1350
1351static inline int amdgpu_in_reset(struct amdgpu_device *adev)
1352{
1353 return atomic_read(&adev->in_gpu_reset)({ typeof(*(&adev->in_gpu_reset)) __tmp = *(volatile typeof
(*(&adev->in_gpu_reset)) *)&(*(&adev->in_gpu_reset
)); membar_datadep_consumer(); __tmp; });
1354}
1355#endif
←

/usr/src/sys/dev/pci/drm/include/linux/pci.h

1/* $OpenBSD: pci.h,v 1.7 2021/07/07 02:38:36 jsg Exp $ */
2/*
3 * Copyright (c) 2015 Mark Kettenis
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18#ifndef _LINUX_PCI_H_
19#define _LINUX_PCI_H_
20
21#include <sys/types.h>
22/* sparc64 cpu.h needs time.h and siginfo.h (indirect via param.h) */
23#include <sys/param.h>
24#include <machine/cpu.h>
25
26#include <dev/pci/pcireg.h>
27#include <dev/pci/pcivar.h>
28#include <dev/pci/pcidevs.h>
29#include <uvm/uvm_extern.h>
30
31#include <linux/io.h>
32#include <linux/ioport.h>
33#include <linux/kobject.h>
34#include <linux/dma-mapping.h> /* pci-dma-compat.h -> dma-mapping.h */
35#include <linux/mod_devicetable.h>
36
37struct pci_dev;
38
39struct pci_bus {
40 pci_chipset_tag_t pc;
41 unsigned char number;
42 int domain_nr;
43 pcitag_t *bridgetag;
44 struct pci_dev *self;
45};
46
47struct pci_acpi {
48 struct aml_node *node;
49};
50
51struct pci_dev {
52 struct pci_bus _bus;
53 struct pci_bus *bus;
54
55 unsigned int devfn;
56 uint16_t vendor;
57 uint16_t device;
58 uint16_t subsystem_vendor;
59 uint16_t subsystem_device;
60 uint8_t revision;
61
62 pci_chipset_tag_t pc;
63 pcitag_t tag;
64 struct pci_softc *pci;
65
66 int irq;
67 int msi_enabled;
68 uint8_t no_64bit_msi;
69
70 struct pci_acpi dev;
71};
72#define PCI_ANY_ID(uint16_t) (~0U) (uint16_t) (~0U)
73
74#ifndef PCI_MEM_START0x100000
75#define PCI_MEM_START0x100000 0
76#endif
77
78#ifndef PCI_MEM_END0xffffffff
79#define PCI_MEM_END0xffffffff 0xffffffff
80#endif
81
82#ifndef PCI_MEM64_END0xffffffffffffffff
83#define PCI_MEM64_END0xffffffffffffffff 0xffffffffffffffff
84#endif
85
86#define PCI_VENDOR_ID_APPLE0x106b PCI_VENDOR_APPLE0x106b
87#define PCI_VENDOR_ID_ASUSTEK0x1043 PCI_VENDOR_ASUSTEK0x1043
88#define PCI_VENDOR_ID_ATI0x1002 PCI_VENDOR_ATI0x1002
89#define PCI_VENDOR_ID_DELL0x1028 PCI_VENDOR_DELL0x1028
90#define PCI_VENDOR_ID_HP0x103c PCI_VENDOR_HP0x103c
91#define PCI_VENDOR_ID_IBM0x1014 PCI_VENDOR_IBM0x1014
92#define PCI_VENDOR_ID_INTEL0x8086 PCI_VENDOR_INTEL0x8086
93#define PCI_VENDOR_ID_SONY0x104d PCI_VENDOR_SONY0x104d
94#define PCI_VENDOR_ID_VIA0x1106 PCI_VENDOR_VIATECH0x1106
95
96#define PCI_DEVICE_ID_ATI_RADEON_QY0x5159 PCI_PRODUCT_ATI_RADEON_QY0x5159
97
98#define PCI_SUBVENDOR_ID_REDHAT_QUMRANET0x1af4 0x1af4
99#define PCI_SUBDEVICE_ID_QEMU0x1100 0x1100
100
101#define PCI_DEVFN(slot, func)((slot) << 3 | (func)) ((slot) << 3 | (func))
102#define PCI_SLOT(devfn)((devfn) >> 3) ((devfn) >> 3)
103#define PCI_FUNC(devfn)((devfn) & 0x7) ((devfn) & 0x7)
104#define PCI_BUS_NUM(devfn)(((devfn) >> 8) & 0xff) (((devfn) >> 8) & 0xff)
105
106#define pci_dev_put(x)
107
108#define PCI_EXP_DEVSTA0x0a 0x0a
109#define PCI_EXP_DEVSTA_TRPND0x0020 0x0020
110#define PCI_EXP_LNKCAP0x0c 0x0c
111#define PCI_EXP_LNKCAP_CLKPM0x00040000 0x00040000
112#define PCI_EXP_LNKCTL0x10 0x10
113#define PCI_EXP_LNKCTL_HAWD0x0200 0x0200
114#define PCI_EXP_LNKCTL20x30 0x30
115#define PCI_EXP_LNKCTL2_ENTER_COMP0x0010 0x0010
116#define PCI_EXP_LNKCTL2_TX_MARGIN0x0380 0x0380
117#define PCI_EXP_LNKCTL2_TLS0x0000000f PCI_PCIE_LCSR2_TLS0x0000000f
118#define PCI_EXP_LNKCTL2_TLS_2_5GT0x00000001 PCI_PCIE_LCSR2_TLS_2_50x00000001
119#define PCI_EXP_LNKCTL2_TLS_5_0GT0x00000002 PCI_PCIE_LCSR2_TLS_50x00000002
120#define PCI_EXP_LNKCTL2_TLS_8_0GT0x00000003 PCI_PCIE_LCSR2_TLS_80x00000003
121
122#define PCI_COMMAND0x04 PCI_COMMAND_STATUS_REG0x04
123#define PCI_COMMAND_MEMORY0x00000002 PCI_COMMAND_MEM_ENABLE0x00000002
124
125static inline int
126pci_read_config_dword(struct pci_dev *pdev, int reg, u32 *val)
127{
128 *val = pci_conf_read(pdev->pc, pdev->tag, reg);
129 return 0;
130}
131
132static inline int
133pci_read_config_word(struct pci_dev *pdev, int reg, u16 *val)
134{
135 uint32_t v;
136
137 v = pci_conf_read(pdev->pc, pdev->tag, (reg & ~0x2));
138 *val = (v >> ((reg & 0x2) * 8));
139 return 0;
140}
141
142static inline int
143pci_read_config_byte(struct pci_dev *pdev, int reg, u8 *val)
144{
145 uint32_t v;
146
147 v = pci_conf_read(pdev->pc, pdev->tag, (reg & ~0x3));
148 *val = (v >> ((reg & 0x3) * 8));
149 return 0;
150}
151
152static inline int
153pci_write_config_dword(struct pci_dev *pdev, int reg, u32 val)
154{
155 pci_conf_write(pdev->pc, pdev->tag, reg, val);
156 return 0;
157}
158
159static inline int
160pci_write_config_word(struct pci_dev *pdev, int reg, u16 val)
161{
162 uint32_t v;
163
164 v = pci_conf_read(pdev->pc, pdev->tag, (reg & ~0x2));
165 v &= ~(0xffff << ((reg & 0x2) * 8));
166 v |= (val << ((reg & 0x2) * 8));
167 pci_conf_write(pdev->pc, pdev->tag, (reg & ~0x2), v);
168 return 0;
169}
170
171static inline int
172pci_write_config_byte(struct pci_dev *pdev, int reg, u8 val)
173{
174 uint32_t v;
175
176 v = pci_conf_read(pdev->pc, pdev->tag, (reg & ~0x3));
177 v &= ~(0xff << ((reg & 0x3) * 8));
178 v |= (val << ((reg & 0x3) * 8));
179 pci_conf_write(pdev->pc, pdev->tag, (reg & ~0x3), v);
180 return 0;
181}
182
183static inline int
184pci_bus_read_config_word(struct pci_bus *bus, unsigned int devfn,
185 int reg, u16 *val)
186{
187 pcitag_t tag = pci_make_tag(bus->pc, bus->number,
188 PCI_SLOT(devfn)((devfn) >> 3), PCI_FUNC(devfn)((devfn) & 0x7));
189 uint32_t v;
190
191 v = pci_conf_read(bus->pc, tag, (reg & ~0x2));
192 *val = (v >> ((reg & 0x2) * 8));
193 return 0;
194}
195
196static inline int
197pci_bus_read_config_byte(struct pci_bus *bus, unsigned int devfn,
198 int reg, u8 *val)
199{
200 pcitag_t tag = pci_make_tag(bus->pc, bus->number,
201 PCI_SLOT(devfn)((devfn) >> 3), PCI_FUNC(devfn)((devfn) & 0x7));
202 uint32_t v;
203
204 v = pci_conf_read(bus->pc, tag, (reg & ~0x3));
205 *val = (v >> ((reg & 0x3) * 8));
206 return 0;
207}
208
209static inline int
210pci_bus_write_config_byte(struct pci_bus *bus, unsigned int devfn,
211 int reg, u8 val)
212{
213 pcitag_t tag = pci_make_tag(bus->pc, bus->number,
214 PCI_SLOT(devfn)((devfn) >> 3), PCI_FUNC(devfn)((devfn) & 0x7));
215 uint32_t v;
216
217 v = pci_conf_read(bus->pc, tag, (reg & ~0x3));
218 v &= ~(0xff << ((reg & 0x3) * 8));
219 v |= (val << ((reg & 0x3) * 8));
220 pci_conf_write(bus->pc, tag, (reg & ~0x3), v);
221 return 0;
222}
223
224static inline int
225pci_pcie_cap(struct pci_dev *pdev)
226{
227 int pos;
228 if (!pci_get_capability(pdev->pc, pdev->tag, PCI_CAP_PCIEXPRESS0x10,
229 &pos, NULL((void *)0)))
230 return -EINVAL22;
231 return pos;
232}
233
234static inline bool_Bool
235pci_is_pcie(struct pci_dev *pdev)
236{
237 return (pci_pcie_cap(pdev) > 0);
238}
239
240static inline bool_Bool
241pci_is_root_bus(struct pci_bus *pbus)
242{
243 return (pbus->bridgetag == NULL((void *)0));
59
←
Assuming field 'bridgetag' is not equal to NULL
→
60
←
Returning zero, which participates in a condition later
→
244}
245
246static inline int
247pcie_capability_read_dword(struct pci_dev *pdev, int off, u32 *val)
248{
249 int pos;
250 if (!pci_get_capability(pdev->pc, pdev->tag, PCI_CAP_PCIEXPRESS0x10,
251 &pos, NULL((void *)0))) {
252 *val = 0;
253 return -EINVAL22;
254 }
255 *val = pci_conf_read(pdev->pc, pdev->tag, pos + off);
256 return 0;
257}
258
259static inline int
260pcie_capability_read_word(struct pci_dev *pdev, int off, u16 *val)
261{
262 int pos;
263 if (!pci_get_capability(pdev->pc, pdev->tag, PCI_CAP_PCIEXPRESS0x10,
264 &pos, NULL((void *)0))) {
265 *val = 0;
266 return -EINVAL22;
267 }
268 pci_read_config_word(pdev, pos + off, val);
269 return 0;
270}
271
272static inline int
273pcie_capability_write_word(struct pci_dev *pdev, int off, u16 val)
274{
275 int pos;
276 if (!pci_get_capability(pdev->pc, pdev->tag, PCI_CAP_PCIEXPRESS0x10,
277 &pos, NULL((void *)0)))
278 return -EINVAL22;
279 pci_write_config_word(pdev, pos + off, val);
280 return 0;
281}
282
283static inline int
284pcie_get_readrq(struct pci_dev *pdev)
285{
286 uint16_t val;
287
288 pcie_capability_read_word(pdev, PCI_PCIE_DCSR0x08, &val);
289
290 return 128 << ((val & PCI_PCIE_DCSR_MPS0x00007000) >> 12);
291}
292
293static inline int
294pcie_set_readrq(struct pci_dev *pdev, int rrq)
295{
296 uint16_t val;
297
298 pcie_capability_read_word(pdev, PCI_PCIE_DCSR0x08, &val);
299 val &= ~PCI_PCIE_DCSR_MPS0x00007000;
300 val |= (ffs(rrq) - 8) << 12;
301 return pcie_capability_write_word(pdev, PCI_PCIE_DCSR0x08, val);
302}
303
304#define pci_set_master(x)
305#define pci_clear_master(x)
306
307#define pci_save_state(x)
308#define pci_restore_state(x)
309
310#define pci_enable_msi(x)0 0
311#define pci_disable_msi(x)
312
313typedef enum {
314 PCI_D0,
315 PCI_D1,
316 PCI_D2,
317 PCI_D3hot,
318 PCI_D3cold
319} pci_power_t;
320
321enum pci_bus_speed {
322 PCIE_SPEED_2_5GT,
323 PCIE_SPEED_5_0GT,
324 PCIE_SPEED_8_0GT,
325 PCIE_SPEED_16_0GT,
326 PCI_SPEED_UNKNOWN
327};
328
329enum pcie_link_width {
330 PCIE_LNK_X1 = 1,
331 PCIE_LNK_X2 = 2,
332 PCIE_LNK_X4 = 4,
333 PCIE_LNK_X8 = 8,
334 PCIE_LNK_X12 = 12,
335 PCIE_LNK_X16 = 16,
336 PCIE_LNK_X32 = 32,
337 PCIE_LNK_WIDTH_UNKNOWN = 0xff
338};
339
340typedef unsigned int pci_ers_result_t;
341typedef unsigned int pci_channel_state_t;
342
343#define PCI_ERS_RESULT_DISCONNECT0 0
344#define PCI_ERS_RESULT_RECOVERED1 1
345
346enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *);
347enum pcie_link_width pcie_get_width_cap(struct pci_dev *);
348int pci_resize_resource(struct pci_dev *, int, int);
349
350static inline void
351pcie_bandwidth_available(struct pci_dev *pdev, struct pci_dev **ldev,
352 enum pci_bus_speed *speed, enum pcie_link_width *width)
353{
354 struct pci_dev *bdev = pdev->bus->self;
355 if (bdev == NULL((void *)0))
67
←
Assuming 'bdev' is equal to NULL
→
68
←
Taking true branch
→
356 return;
69
←
Returning without writing to '*width'
→
357
358 if (speed)
359 *speed = pcie_get_speed_cap(bdev);
360 if (width)
361 *width = pcie_get_width_cap(bdev);
362}
363
364#define pci_save_state(x)
365#define pci_enable_device(x)0 0
366#define pci_disable_device(x)
367#define pci_is_thunderbolt_attached(x)0 false0
368#define pci_set_drvdata(x, y)
369
370static inline int
371pci_domain_nr(struct pci_bus *pbus)
372{
373 return pbus->domain_nr;
374}
375
376static inline int
377pci_irq_vector(struct pci_dev *pdev, unsigned int num)
378{
379 return pdev->irq;
380}
381
382static inline void
383pci_free_irq_vectors(struct pci_dev *pdev)
384{
385}
386
387static inline int
388pci_set_power_state(struct pci_dev *dev, int state)
389{
390 return 0;
391}
392
393static inline struct pci_dev *
394pci_get_class(pcireg_t class, struct pci_dev *pdev)
395{
396 return NULL((void *)0);
397}
398
399#define PCI_CLASS_DISPLAY_VGA(0x03 | 0x00) \
400 (PCI_CLASS_DISPLAY0x03 | PCI_SUBCLASS_DISPLAY_VGA0x00)
401#define PCI_CLASS_DISPLAY_OTHER(0x03 | 0x80) \
402 (PCI_CLASS_DISPLAY0x03 | PCI_SUBCLASS_DISPLAY_MISC0x80)
403
404#if defined(__amd64__1) || defined(__arm64__) || \
405 defined(__i386__) || defined(__riscv64__)
406
407#define PCI_DMA_BIDIRECTIONAL0 0
408
409static inline dma_addr_t
410pci_map_page(struct pci_dev *pdev, struct vm_page *page, unsigned long offset, size_t size, int direction)
411{
412 return VM_PAGE_TO_PHYS(page)((page)->phys_addr);
413}
414
415static inline void
416pci_unmap_page(struct pci_dev *pdev, dma_addr_t dma_address, size_t size, int direction)
417{
418}
419
420static inline int
421pci_dma_mapping_error(struct pci_dev *pdev, dma_addr_t dma_addr)
422{
423 return 0;
424}
425
426#define pci_set_dma_mask(x, y)0 0
427#define pci_set_consistent_dma_mask(x, y)0 0
428
429#endif
430
431#endif /* _LINUX_PCI_H_ */