Bug Summary

File:dev/pci/drm/amd/amdgpu/gmc_v11_0.c
Warning:line 760, column 2
Value stored to 'r' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name gmc_v11_0.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/amd/amdgpu/gmc_v11_0.c
1/*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include <linux/firmware.h>
24#include <linux/pci.h>
25
26#include <drm/drm_cache.h>
27
28#include "amdgpu.h"
29#include "amdgpu_atomfirmware.h"
30#include "gmc_v11_0.h"
31#include "umc_v8_10.h"
32#include "athub/athub_3_0_0_sh_mask.h"
33#include "athub/athub_3_0_0_offset.h"
34#include "dcn/dcn_3_2_0_offset.h"
35#include "dcn/dcn_3_2_0_sh_mask.h"
36#include "oss/osssys_6_0_0_offset.h"
37#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
38#include "navi10_enum.h"
39#include "soc15.h"
40#include "soc15d.h"
41#include "soc15_common.h"
42#include "nbio_v4_3.h"
43#include "gfxhub_v3_0.h"
44#include "gfxhub_v3_0_3.h"
45#include "mmhub_v3_0.h"
46#include "mmhub_v3_0_1.h"
47#include "mmhub_v3_0_2.h"
48#include "athub_v3_0.h"
49
50
51static int gmc_v11_0_ecc_interrupt_state(struct amdgpu_device *adev,
52 struct amdgpu_irq_src *src,
53 unsigned type,
54 enum amdgpu_interrupt_state state)
55{
56 return 0;
57}
58
59static int
60gmc_v11_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
61 struct amdgpu_irq_src *src, unsigned type,
62 enum amdgpu_interrupt_state state)
63{
64 switch (state) {
65 case AMDGPU_IRQ_STATE_DISABLE:
66 /* MM HUB */
67 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_01, false0);
68 /* GFX HUB */
69 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_00, false0);
70 break;
71 case AMDGPU_IRQ_STATE_ENABLE:
72 /* MM HUB */
73 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB_01, true1);
74 /* GFX HUB */
75 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB_00, true1);
76 break;
77 default:
78 break;
79 }
80
81 return 0;
82}
83
84static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev,
85 struct amdgpu_irq_src *source,
86 struct amdgpu_iv_entry *entry)
87{
88 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
89 uint32_t status = 0;
90 u64 addr;
91
92 addr = (u64)entry->src_data[0] << 12;
93 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
94
95 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
96 /*
97 * Issue a dummy read to wait for the status register to
98 * be updated to avoid reading an incorrect value due to
99 * the new fast GRBM interface.
100 */
101 if (entry->vmid_src == AMDGPU_GFXHUB_00)
102 RREG32(hub->vm_l2_pro_fault_status)amdgpu_device_rreg(adev, (hub->vm_l2_pro_fault_status), 0);
103
104 status = RREG32(hub->vm_l2_pro_fault_status)amdgpu_device_rreg(adev, (hub->vm_l2_pro_fault_status), 0);
105 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1)do { uint32_t tmp_ = amdgpu_device_rreg(adev, (hub->vm_l2_pro_fault_cntl
), 0); tmp_ &= (~1); tmp_ |= ((1) & ~(~1)); amdgpu_device_wreg
(adev, (hub->vm_l2_pro_fault_cntl), (tmp_), 0); } while (0
);
106 }
107
108 if (printk_ratelimit()1) {
109 struct amdgpu_task_info task_info;
110
111 memset(&task_info, 0, sizeof(struct amdgpu_task_info))__builtin_memset((&task_info), (0), (sizeof(struct amdgpu_task_info
)));
112 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
113
114 dev_err(adev->dev,printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
115 "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
116 "for process %s pid %d thread %s pid %d)\n",printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
117 entry->vmid_src ? "mmhub" : "gfxhub",printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
118 entry->src_id, entry->ring_id, entry->vmid,printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
119 entry->pasid, task_info.process_name, task_info.tgid,printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid)
120 task_info.task_name, task_info.pid)printf("drm:pid%d:%s *ERROR* " "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, "
"for process %s pid %d thread %s pid %d)\n", ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->
ps_pid, __func__ , entry->vmid_src ? "mmhub" : "gfxhub", entry
->src_id, entry->ring_id, entry->vmid, entry->pasid
, task_info.process_name, task_info.tgid, task_info.task_name
, task_info.pid);
121 dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n",printf("drm:pid%d:%s *ERROR* " " in page starting at address 0x%016llx from client %d\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , addr, entry
->client_id)
122 addr, entry->client_id)printf("drm:pid%d:%s *ERROR* " " in page starting at address 0x%016llx from client %d\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , addr, entry
->client_id);
123 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
124 hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
125 }
126
127 return 0;
128}
129
130static const struct amdgpu_irq_src_funcs gmc_v11_0_irq_funcs = {
131 .set = gmc_v11_0_vm_fault_interrupt_state,
132 .process = gmc_v11_0_process_interrupt,
133};
134
135static const struct amdgpu_irq_src_funcs gmc_v11_0_ecc_funcs = {
136 .set = gmc_v11_0_ecc_interrupt_state,
137 .process = amdgpu_umc_process_ecc_irq,
138};
139
140static void gmc_v11_0_set_irq_funcs(struct amdgpu_device *adev)
141{
142 adev->gmc.vm_fault.num_types = 1;
143 adev->gmc.vm_fault.funcs = &gmc_v11_0_irq_funcs;
144
145 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
146 adev->gmc.ecc_irq.num_types = 1;
147 adev->gmc.ecc_irq.funcs = &gmc_v11_0_ecc_funcs;
148 }
149}
150
151/**
152 * gmc_v11_0_use_invalidate_semaphore - judge whether to use semaphore
153 *
154 * @adev: amdgpu_device pointer
155 * @vmhub: vmhub type
156 *
157 */
158static bool_Bool gmc_v11_0_use_invalidate_semaphore(struct amdgpu_device *adev,
159 uint32_t vmhub)
160{
161 return ((vmhub == AMDGPU_MMHUB_01) &&
162 (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))));
163}
164
165static bool_Bool gmc_v11_0_get_vmid_pasid_mapping_info(
166 struct amdgpu_device *adev,
167 uint8_t vmid, uint16_t *p_pasid)
168{
169 *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid)amdgpu_device_rreg(adev, ((adev->reg_offset[OSSSYS_HWIP][0
][0] + 0x0000) + vmid), 0) & 0xffff;
170
171 return !!(*p_pasid);
172}
173
174/*
175 * GART
176 * VMID 0 is the physical GPU addresses as used by the kernel.
177 * VMIDs 1-15 are used for userspace clients and are handled
178 * by the amdgpu vm/hsa code.
179 */
180
181static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
182 unsigned int vmhub, uint32_t flush_type)
183{
184 bool_Bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(adev, vmhub);
185 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
186 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
187 u32 tmp;
188 /* Use register 17 for GART */
189 const unsigned eng = 17;
190 unsigned int i;
191 unsigned char hub_ip = 0;
192
193 hub_ip = (vmhub == AMDGPU_GFXHUB_00) ?
194 GC_HWIP : MMHUB_HWIP;
195
196 spin_lock(&adev->gmc.invalidate_lock)mtx_enter(&adev->gmc.invalidate_lock);
197 /*
198 * It may lose gpuvm invalidate acknowldege state across power-gating
199 * off cycle, add semaphore acquire before invalidation and semaphore
200 * release after invalidation to avoid entering power gated state
201 * to WA the Issue
202 */
203
204 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
205 if (use_semaphore) {
206 for (i = 0; i < adev->usec_timeout; i++) {
207 /* a read return value of 1 means semaphore acuqire */
208 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_sem + hub->eng_distance * eng)
, 0))
209 hub->eng_distance * eng, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_sem + hub->eng_distance * eng)
, 0));
210 if (tmp & 0x1)
211 break;
212 udelay(1);
213 }
214
215 if (i >= adev->usec_timeout)
216 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n")__drm_err("Timeout waiting for sem acquire in VM flush!\n");
217 }
218
219 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_req + hub->
eng_distance * eng, inv_req, (1<<1) | (1<<2), hub_ip
) : amdgpu_device_wreg(adev, (hub->vm_inv_eng0_req + hub->
eng_distance * eng), (inv_req), 0));
220
221 /* Wait for ACK with a delay.*/
222 for (i = 0; i < adev->usec_timeout; i++) {
223 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_ack + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_ack + hub->eng_distance * eng)
, 0))
224 hub->eng_distance * eng, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, hub->vm_inv_eng0_ack + hub->
eng_distance * eng, (1<<1) | (1<<2), hub_ip) : amdgpu_device_rreg
(adev, (hub->vm_inv_eng0_ack + hub->eng_distance * eng)
, 0));
225 tmp &= 1 << vmid;
226 if (tmp)
227 break;
228
229 udelay(1);
230 }
231
232 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
233 if (use_semaphore)
234 /*
235 * add semaphore release after invalidation,
236 * write with 0 means semaphore release
237 */
238 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, 0, (1<<1) | (1<<2), hub_ip) :
amdgpu_device_wreg(adev, (hub->vm_inv_eng0_sem + hub->
eng_distance * eng), (0), 0))
239 hub->eng_distance * eng, 0, hub_ip)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, hub->vm_inv_eng0_sem + hub->
eng_distance * eng, 0, (1<<1) | (1<<2), hub_ip) :
amdgpu_device_wreg(adev, (hub->vm_inv_eng0_sem + hub->
eng_distance * eng), (0), 0));
240
241 /* Issue additional private vm invalidation to MMHUB */
242 if ((vmhub != AMDGPU_GFXHUB_00) &&
243 (hub->vm_l2_bank_select_reserved_cid2) &&
244 !amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
245 inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2)amdgpu_device_rreg(adev, (hub->vm_l2_bank_select_reserved_cid2
), (1<<1));
246 /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
247 inv_req |= (1 << 25);
248 /* Issue private invalidation */
249 WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req)amdgpu_device_wreg(adev, (hub->vm_l2_bank_select_reserved_cid2
), (inv_req), (1<<1));
250 /* Read back to ensure invalidation is done*/
251 RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2)amdgpu_device_rreg(adev, (hub->vm_l2_bank_select_reserved_cid2
), (1<<1));
252 }
253
254 spin_unlock(&adev->gmc.invalidate_lock)mtx_leave(&adev->gmc.invalidate_lock);
255
256 if (i < adev->usec_timeout)
257 return;
258
259 DRM_ERROR("Timeout waiting for VM flush ACK!\n")__drm_err("Timeout waiting for VM flush ACK!\n");
260}
261
262/**
263 * gmc_v11_0_flush_gpu_tlb - gart tlb flush callback
264 *
265 * @adev: amdgpu_device pointer
266 * @vmid: vm instance to flush
267 *
268 * Flush the TLB for the requested page table.
269 */
270static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
271 uint32_t vmhub, uint32_t flush_type)
272{
273 if ((vmhub == AMDGPU_GFXHUB_00) && !adev->gfx.is_poweron)
274 return;
275
276 /* flush hdp cache */
277 adev->hdp.funcs->flush_hdp(adev, NULL((void *)0));
278
279 /* For SRIOV run time, driver shouldn't access the register through MMIO
280 * Directly use kiq to do the vm invalidation instead
281 */
282 if ((adev->gfx.kiq.ring.sched.ready || adev->mes.ring.sched.ready) &&
283 (amdgpu_sriov_runtime(adev)((adev)->virt.caps & (1 << 4)) || !amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))) {
284 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
285 const unsigned eng = 17;
286 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
287 u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
288 u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
289
290 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
291 1 << vmid);
292 return;
293 }
294
295 mutex_lock(&adev->mman.gtt_window_lock)rw_enter_write(&adev->mman.gtt_window_lock);
296 gmc_v11_0_flush_vm_hub(adev, vmid, vmhub, 0);
297 mutex_unlock(&adev->mman.gtt_window_lock)rw_exit_write(&adev->mman.gtt_window_lock);
298 return;
299}
300
301/**
302 * gmc_v11_0_flush_gpu_tlb_pasid - tlb flush via pasid
303 *
304 * @adev: amdgpu_device pointer
305 * @pasid: pasid to be flush
306 *
307 * Flush the TLB for the requested pasid.
308 */
309static int gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
310 uint16_t pasid, uint32_t flush_type,
311 bool_Bool all_hub)
312{
313 int vmid, i;
314 signed long r;
315 uint32_t seq;
316 uint16_t queried_pasid;
317 bool_Bool ret;
318 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
319 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
320
321 if (amdgpu_emu_mode == 0 && ring->sched.ready) {
322 spin_lock(&adev->gfx.kiq.ring_lock)mtx_enter(&adev->gfx.kiq.ring_lock);
323 /* 2 dwords flush + 8 dwords fence */
324 amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size + 8);
325 kiq->pmf->kiq_invalidate_tlbs(ring,
326 pasid, flush_type, all_hub);
327 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT5000);
328 if (r) {
329 amdgpu_ring_undo(ring);
330 spin_unlock(&adev->gfx.kiq.ring_lock)mtx_leave(&adev->gfx.kiq.ring_lock);
331 return -ETIME60;
332 }
333
334 amdgpu_ring_commit(ring);
335 spin_unlock(&adev->gfx.kiq.ring_lock)mtx_leave(&adev->gfx.kiq.ring_lock);
336 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
337 if (r < 1) {
338 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r)printf("drm:pid%d:%s *ERROR* " "wait for kiq fence error: %ld.\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
339 return -ETIME60;
340 }
341
342 return 0;
343 }
344
345 for (vmid = 1; vmid < 16; vmid++) {
346
347 ret = gmc_v11_0_get_vmid_pasid_mapping_info(adev, vmid,
348 &queried_pasid);
349 if (ret && queried_pasid == pasid) {
350 if (all_hub) {
351 for (i = 0; i < adev->num_vmhubs; i++)
352 gmc_v11_0_flush_gpu_tlb(adev, vmid,
353 i, flush_type);
354 } else {
355 gmc_v11_0_flush_gpu_tlb(adev, vmid,
356 AMDGPU_GFXHUB_00, flush_type);
357 }
358 }
359 }
360
361 return 0;
362}
363
364static uint64_t gmc_v11_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
365 unsigned vmid, uint64_t pd_addr)
366{
367 bool_Bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
368 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
369 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
370 unsigned eng = ring->vm_inv_eng;
371
372 /*
373 * It may lose gpuvm invalidate acknowldege state across power-gating
374 * off cycle, add semaphore acquire before invalidation and semaphore
375 * release after invalidation to avoid entering power gated state
376 * to WA the Issue
377 */
378
379 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
380 if (use_semaphore)
381 /* a read return value of 1 means semaphore acuqire */
382 amdgpu_ring_emit_reg_wait(ring,(ring)->funcs->emit_reg_wait((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0x1), (0x1))
383 hub->vm_inv_eng0_sem +(ring)->funcs->emit_reg_wait((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0x1), (0x1))
384 hub->eng_distance * eng, 0x1, 0x1)(ring)->funcs->emit_reg_wait((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0x1), (0x1));
385
386 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_lo32
+ (hub->ctx_addr_distance * vmid)), (((u32)(pd_addr))))
387 (hub->ctx_addr_distance * vmid),(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_lo32
+ (hub->ctx_addr_distance * vmid)), (((u32)(pd_addr))))
388 lower_32_bits(pd_addr))(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_lo32
+ (hub->ctx_addr_distance * vmid)), (((u32)(pd_addr))));
389
390 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_hi32
+ (hub->ctx_addr_distance * vmid)), (((u32)(((pd_addr) >>
16) >> 16))))
391 (hub->ctx_addr_distance * vmid),(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_hi32
+ (hub->ctx_addr_distance * vmid)), (((u32)(((pd_addr) >>
16) >> 16))))
392 upper_32_bits(pd_addr))(ring)->funcs->emit_wreg((ring), (hub->ctx0_ptb_addr_hi32
+ (hub->ctx_addr_distance * vmid)), (((u32)(((pd_addr) >>
16) >> 16))));
393
394 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
395 hub->eng_distance * eng,(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
396 hub->vm_inv_eng0_ack +(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
397 hub->eng_distance * eng,(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid))
398 req, 1 << vmid)(ring)->funcs->emit_reg_write_reg_wait((ring), (hub->
vm_inv_eng0_req + hub->eng_distance * eng), (hub->vm_inv_eng0_ack
+ hub->eng_distance * eng), (req), (1 << vmid));
399
400 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
401 if (use_semaphore)
402 /*
403 * add semaphore release after invalidation,
404 * write with 0 means semaphore release
405 */
406 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +(ring)->funcs->emit_wreg((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0))
407 hub->eng_distance * eng, 0)(ring)->funcs->emit_wreg((ring), (hub->vm_inv_eng0_sem
+ hub->eng_distance * eng), (0));
408
409 return pd_addr;
410}
411
412static void gmc_v11_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
413 unsigned pasid)
414{
415 struct amdgpu_device *adev = ring->adev;
416 uint32_t reg;
417
418 /* MES fw manages IH_VMID_x_LUT updating */
419 if (ring->is_mes_queue)
420 return;
421
422 if (ring->funcs->vmhub == AMDGPU_GFXHUB_00)
423 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT)(adev->reg_offset[OSSSYS_HWIP][0][0] + 0x0000) + vmid;
424 else
425 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM)(adev->reg_offset[OSSSYS_HWIP][0][0] + 0x0010) + vmid;
426
427 amdgpu_ring_emit_wreg(ring, reg, pasid)(ring)->funcs->emit_wreg((ring), (reg), (pasid));
428}
429
430/*
431 * PTE format:
432 * 63:59 reserved
433 * 58:57 reserved
434 * 56 F
435 * 55 L
436 * 54 reserved
437 * 53:52 SW
438 * 51 T
439 * 50:48 mtype
440 * 47:12 4k physical page base address
441 * 11:7 fragment
442 * 6 write
443 * 5 read
444 * 4 exe
445 * 3 Z
446 * 2 snooped
447 * 1 system
448 * 0 valid
449 *
450 * PDE format:
451 * 63:59 block fragment size
452 * 58:55 reserved
453 * 54 P
454 * 53:48 reserved
455 * 47:6 physical base address of PD or PTE
456 * 5:3 reserved
457 * 2 C
458 * 1 system
459 * 0 valid
460 */
461
462static uint64_t gmc_v11_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
463{
464 switch (flags) {
465 case AMDGPU_VM_MTYPE_DEFAULT(0 << 5):
466 return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC)((uint64_t)(MTYPE_NC) << 48);
467 case AMDGPU_VM_MTYPE_NC(1 << 5):
468 return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC)((uint64_t)(MTYPE_NC) << 48);
469 case AMDGPU_VM_MTYPE_WC(2 << 5):
470 return AMDGPU_PTE_MTYPE_NV10(MTYPE_WC)((uint64_t)(MTYPE_WC) << 48);
471 case AMDGPU_VM_MTYPE_CC(3 << 5):
472 return AMDGPU_PTE_MTYPE_NV10(MTYPE_CC)((uint64_t)(MTYPE_CC) << 48);
473 case AMDGPU_VM_MTYPE_UC(4 << 5):
474 return AMDGPU_PTE_MTYPE_NV10(MTYPE_UC)((uint64_t)(MTYPE_UC) << 48);
475 default:
476 return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC)((uint64_t)(MTYPE_NC) << 48);
477 }
478}
479
480static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level,
481 uint64_t *addr, uint64_t *flags)
482{
483 if (!(*flags & AMDGPU_PDE_PTE(1ULL << 54)) && !(*flags & AMDGPU_PTE_SYSTEM(1ULL << 1)))
484 *addr = adev->vm_manager.vram_base_offset + *addr -
485 adev->gmc.vram_start;
486 BUG_ON(*addr & 0xFFFF00000000003FULL)((!(*addr & 0xFFFF00000000003FULL)) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/gmc_v11_0.c"
, 486, "!(*addr & 0xFFFF00000000003FULL)"));
487
488 if (!adev->gmc.translate_further)
489 return;
490
491 if (level == AMDGPU_VM_PDB1) {
492 /* Set the block fragment size */
493 if (!(*flags & AMDGPU_PDE_PTE(1ULL << 54)))
494 *flags |= AMDGPU_PDE_BFS(0x9)((uint64_t)0x9 << 59);
495
496 } else if (level == AMDGPU_VM_PDB0) {
497 if (*flags & AMDGPU_PDE_PTE(1ULL << 54))
498 *flags &= ~AMDGPU_PDE_PTE(1ULL << 54);
499 else
500 *flags |= AMDGPU_PTE_TF(1ULL << 56);
501 }
502}
503
504static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev,
505 struct amdgpu_bo_va_mapping *mapping,
506 uint64_t *flags)
507{
508 *flags &= ~AMDGPU_PTE_EXECUTABLE(1ULL << 4);
509 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE(1ULL << 4);
510
511 *flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK((uint64_t)(7ULL) << 48);
512 *flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK((uint64_t)(7ULL) << 48));
513
514 *flags &= ~AMDGPU_PTE_NOALLOC(1ULL << 58);
515 *flags |= (mapping->flags & AMDGPU_PTE_NOALLOC(1ULL << 58));
516
517 if (mapping->flags & AMDGPU_PTE_PRT(1ULL << 51)) {
518 *flags |= AMDGPU_PTE_PRT(1ULL << 51);
519 *flags |= AMDGPU_PTE_SNOOPED(1ULL << 2);
520 *flags |= AMDGPU_PTE_LOG(1ULL << 55);
521 *flags |= AMDGPU_PTE_SYSTEM(1ULL << 1);
522 *flags &= ~AMDGPU_PTE_VALID(1ULL << 0);
523 }
524}
525
526static unsigned gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev)
527{
528 u32 d1vga_control = RREG32_SOC15(DCE, 0, regD1VGA_CONTROL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[DCE_HWIP][0][
1] + 0x000c, 0, DCE_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[DCE_HWIP][0][1] + 0x000c), 0));
529 unsigned size;
530
531 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)(((d1vga_control) & 0x00000001L) >> 0x0)) {
532 size = AMDGPU_VBIOS_VGA_ALLOCATION(9 * 1024 * 1024);
533 } else {
534 u32 viewport;
535 u32 pitch;
536
537 viewport = RREG32_SOC15(DCE, 0, regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[DCE_HWIP][0][
2] + 0x05ea, 0, DCE_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[DCE_HWIP][0][2] + 0x05ea), 0));
538 pitch = RREG32_SOC15(DCE, 0, regHUBPREQ0_DCSURF_SURFACE_PITCH)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[DCE_HWIP][0][
2] + 0x0607, 0, DCE_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[DCE_HWIP][0][2] + 0x0607), 0));
539 size = (REG_GET_FIELD(viewport,(((viewport) & 0x3FFF0000L) >> 0x10)
540 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT)(((viewport) & 0x3FFF0000L) >> 0x10) *
541 REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH)(((pitch) & 0x00003FFFL) >> 0x0) *
542 4);
543 }
544
545 return size;
546}
547
548static const struct amdgpu_gmc_funcs gmc_v11_0_gmc_funcs = {
549 .flush_gpu_tlb = gmc_v11_0_flush_gpu_tlb,
550 .flush_gpu_tlb_pasid = gmc_v11_0_flush_gpu_tlb_pasid,
551 .emit_flush_gpu_tlb = gmc_v11_0_emit_flush_gpu_tlb,
552 .emit_pasid_mapping = gmc_v11_0_emit_pasid_mapping,
553 .map_mtype = gmc_v11_0_map_mtype,
554 .get_vm_pde = gmc_v11_0_get_vm_pde,
555 .get_vm_pte = gmc_v11_0_get_vm_pte,
556 .get_vbios_fb_size = gmc_v11_0_get_vbios_fb_size,
557};
558
559static void gmc_v11_0_set_gmc_funcs(struct amdgpu_device *adev)
560{
561 adev->gmc.gmc_funcs = &gmc_v11_0_gmc_funcs;
562}
563
564static void gmc_v11_0_set_umc_funcs(struct amdgpu_device *adev)
565{
566 switch (adev->ip_versions[UMC_HWIP][0]) {
567 case IP_VERSION(8, 10, 0)(((8) << 16) | ((10) << 8) | (0)):
568 adev->umc.channel_inst_num = UMC_V8_10_CHANNEL_INSTANCE_NUM2;
569 adev->umc.umc_inst_num = UMC_V8_10_UMC_INSTANCE_NUM2;
570 adev->umc.node_inst_num = adev->gmc.num_umc;
571 adev->umc.max_ras_err_cnt_per_query = UMC_V8_10_TOTAL_CHANNEL_NUM(adev)(2 * 2 * (adev)->umc.node_inst_num);
572 adev->umc.channel_offs = UMC_V8_10_PER_CHANNEL_OFFSET0x400;
573 adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl[0][0][0];
574 adev->umc.ras = &umc_v8_10_ras;
575 break;
576 case IP_VERSION(8, 11, 0)(((8) << 16) | ((11) << 8) | (0)):
577 break;
578 default:
579 break;
580 }
581
582 if (adev->umc.ras) {
583 amdgpu_ras_register_ras_block(adev, &adev->umc.ras->ras_block);
584
585 strlcpy(adev->umc.ras->ras_block.ras_comm.name, "umc",
586 sizeof(adev->umc.ras->ras_block.ras_comm.name));
587 adev->umc.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__UMC;
588 adev->umc.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
589 adev->umc.ras_if = &adev->umc.ras->ras_block.ras_comm;
590
591 /* If don't define special ras_late_init function, use default ras_late_init */
592 if (!adev->umc.ras->ras_block.ras_late_init)
593 adev->umc.ras->ras_block.ras_late_init = amdgpu_umc_ras_late_init;
594
595 /* If not define special ras_cb function, use default ras_cb */
596 if (!adev->umc.ras->ras_block.ras_cb)
597 adev->umc.ras->ras_block.ras_cb = amdgpu_umc_process_ras_data_cb;
598 }
599}
600
601
602static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev)
603{
604 switch (adev->ip_versions[MMHUB_HWIP][0]) {
605 case IP_VERSION(3, 0, 1)(((3) << 16) | ((0) << 8) | (1)):
606 adev->mmhub.funcs = &mmhub_v3_0_1_funcs;
607 break;
608 case IP_VERSION(3, 0, 2)(((3) << 16) | ((0) << 8) | (2)):
609 adev->mmhub.funcs = &mmhub_v3_0_2_funcs;
610 break;
611 default:
612 adev->mmhub.funcs = &mmhub_v3_0_funcs;
613 break;
614 }
615}
616
617static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev)
618{
619 switch (adev->ip_versions[GC_HWIP][0]) {
620 case IP_VERSION(11, 0, 3)(((11) << 16) | ((0) << 8) | (3)):
621 adev->gfxhub.funcs = &gfxhub_v3_0_3_funcs;
622 break;
623 default:
624 adev->gfxhub.funcs = &gfxhub_v3_0_funcs;
625 break;
626 }
627}
628
629static int gmc_v11_0_early_init(void *handle)
630{
631 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
632
633 gmc_v11_0_set_gfxhub_funcs(adev);
634 gmc_v11_0_set_mmhub_funcs(adev);
635 gmc_v11_0_set_gmc_funcs(adev);
636 gmc_v11_0_set_irq_funcs(adev);
637 gmc_v11_0_set_umc_funcs(adev);
638
639 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
640 adev->gmc.shared_aperture_end =
641 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
642 adev->gmc.private_aperture_start = 0x1000000000000000ULL;
643 adev->gmc.private_aperture_end =
644 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
645
646 return 0;
647}
648
649static int gmc_v11_0_late_init(void *handle)
650{
651 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
652 int r;
653
654 r = amdgpu_gmc_allocate_vm_inv_eng(adev);
655 if (r)
656 return r;
657
658 r = amdgpu_gmc_ras_late_init(adev);
659 if (r)
660 return r;
661
662 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
663}
664
665static void gmc_v11_0_vram_gtt_location(struct amdgpu_device *adev,
666 struct amdgpu_gmc *mc)
667{
668 u64 base = 0;
669
670 base = adev->mmhub.funcs->get_fb_location(adev);
671
672 amdgpu_gmc_vram_location(adev, &adev->gmc, base);
673 amdgpu_gmc_gart_location(adev, mc);
674
675 /* base offset of vram pages */
676 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2)))
677 adev->vm_manager.vram_base_offset = 0;
678 else
679 adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev);
680}
681
682/**
683 * gmc_v11_0_mc_init - initialize the memory controller driver params
684 *
685 * @adev: amdgpu_device pointer
686 *
687 * Look up the amount of vram, vram width, and decide how to place
688 * vram and gart within the GPU's physical address space.
689 * Returns 0 for success.
690 */
691static int gmc_v11_0_mc_init(struct amdgpu_device *adev)
692{
693 int r;
694
695 /* size in MB on si */
696 adev->gmc.mc_vram_size =
697 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
698 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
699
700 if (!(adev->flags & AMD_IS_APU)) {
701 r = amdgpu_device_resize_fb_bar(adev);
702 if (r)
703 return r;
704 }
705 adev->gmc.aper_base = adev->fb_aper_offset;
706 adev->gmc.aper_size = adev->fb_aper_size;
707
708#ifdef CONFIG_X86_641
709 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)((adev)->virt.caps & (1 << 3))) {
710 adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev);
711 adev->gmc.aper_size = adev->gmc.real_vram_size;
712 }
713#endif
714 /* In case the PCI BAR is larger than the actual amount of vram */
715 adev->gmc.visible_vram_size = adev->gmc.aper_size;
716 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
717 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
718
719 /* set the gart size */
720 if (amdgpu_gart_size == -1) {
721 adev->gmc.gart_size = 512ULL << 20;
722 } else
723 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
724
725 gmc_v11_0_vram_gtt_location(adev, &adev->gmc);
726
727 return 0;
728}
729
730static int gmc_v11_0_gart_init(struct amdgpu_device *adev)
731{
732 int r;
733
734 if (adev->gart.bo) {
735 WARN(1, "PCIE GART already initialized\n")({ int __ret = !!(1); if (__ret) printf("PCIE GART already initialized\n"
); __builtin_expect(!!(__ret), 0); });
736 return 0;
737 }
738
739 /* Initialize common gart structure */
740 r = amdgpu_gart_init(adev);
741 if (r)
742 return r;
743
744 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
745 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC)((uint64_t)(MTYPE_UC) << 48) |
746 AMDGPU_PTE_EXECUTABLE(1ULL << 4);
747
748 return amdgpu_gart_table_vram_alloc(adev);
749}
750
751static int gmc_v11_0_sw_init(void *handle)
752{
753 int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
754 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
755
756 adev->mmhub.funcs->init(adev);
757
758 mtx_init(&adev->gmc.invalidate_lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
gmc.invalidate_lock), ((((0x0)) > 0x0 && ((0x0)) <
0x9) ? 0x9 : ((0x0)))); } while (0);
759
760 r = amdgpu_atomfirmware_get_vram_info(adev,
Value stored to 'r' is never read
761 &vram_width, &vram_type, &vram_vendor);
762 adev->gmc.vram_width = vram_width;
763
764 adev->gmc.vram_type = vram_type;
765 adev->gmc.vram_vendor = vram_vendor;
766
767 switch (adev->ip_versions[GC_HWIP][0]) {
768 case IP_VERSION(11, 0, 0)(((11) << 16) | ((0) << 8) | (0)):
769 case IP_VERSION(11, 0, 1)(((11) << 16) | ((0) << 8) | (1)):
770 case IP_VERSION(11, 0, 2)(((11) << 16) | ((0) << 8) | (2)):
771 case IP_VERSION(11, 0, 3)(((11) << 16) | ((0) << 8) | (3)):
772 case IP_VERSION(11, 0, 4)(((11) << 16) | ((0) << 8) | (4)):
773 adev->num_vmhubs = 2;
774 /*
775 * To fulfill 4-level page support,
776 * vm size is 256TB (48bit), maximum size,
777 * block size 512 (9bit)
778 */
779 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
780 break;
781 default:
782 break;
783 }
784
785 /* This interrupt is VMC page fault.*/
786 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VMC,
787 VMC_1_0__SRCID__VM_FAULT0,
788 &adev->gmc.vm_fault);
789
790 if (r)
791 return r;
792
793 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
794 UTCL2_1_0__SRCID__FAULT0,
795 &adev->gmc.vm_fault);
796 if (r)
797 return r;
798
799 if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
800 /* interrupt sent to DF. */
801 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0,
802 &adev->gmc.ecc_irq);
803 if (r)
804 return r;
805 }
806
807 /*
808 * Set the internal MC address mask This is the max address of the GPU's
809 * internal address space.
810 */
811 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
812
813 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)(((44) == 64) ? ~0ULL : (1ULL<<(44)) -1));
814 if (r) {
815 printk(KERN_WARNING"\0014" "amdgpu: No suitable DMA available.\n");
816 return r;
817 }
818
819 adev->need_swiotlb = drm_need_swiotlb(44);
820
821 r = gmc_v11_0_mc_init(adev);
822 if (r)
823 return r;
824
825 amdgpu_gmc_get_vbios_allocations(adev);
826
827 /* Memory manager */
828 r = amdgpu_bo_init(adev);
829 if (r)
830 return r;
831
832 r = gmc_v11_0_gart_init(adev);
833 if (r)
834 return r;
835
836 /*
837 * number of VMs
838 * VMID 0 is reserved for System
839 * amdgpu graphics/compute will use VMIDs 1-7
840 * amdkfd will use VMIDs 8-15
841 */
842 adev->vm_manager.first_kfd_vmid = 8;
843
844 amdgpu_vm_manager_init(adev);
845
846 return 0;
847}
848
849/**
850 * gmc_v11_0_gart_fini - vm fini callback
851 *
852 * @adev: amdgpu_device pointer
853 *
854 * Tears down the driver GART/VM setup (CIK).
855 */
856static void gmc_v11_0_gart_fini(struct amdgpu_device *adev)
857{
858 amdgpu_gart_table_vram_free(adev);
859}
860
861static int gmc_v11_0_sw_fini(void *handle)
862{
863 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
864
865 amdgpu_vm_manager_fini(adev);
866 gmc_v11_0_gart_fini(adev);
867 amdgpu_gem_force_release(adev);
868 amdgpu_bo_fini(adev);
869
870 return 0;
871}
872
873static void gmc_v11_0_init_golden_registers(struct amdgpu_device *adev)
874{
875}
876
877/**
878 * gmc_v11_0_gart_enable - gart enable
879 *
880 * @adev: amdgpu_device pointer
881 */
882static int gmc_v11_0_gart_enable(struct amdgpu_device *adev)
883{
884 int r;
885 bool_Bool value;
886
887 if (adev->gart.bo == NULL((void *)0)) {
888 dev_err(adev->dev, "No VRAM object for PCIE GART.\n")printf("drm:pid%d:%s *ERROR* " "No VRAM object for PCIE GART.\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__);
889 return -EINVAL22;
890 }
891
892 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
893
894 r = adev->mmhub.funcs->gart_enable(adev);
895 if (r)
896 return r;
897
898 /* Flush HDP after it is initialized */
899 adev->hdp.funcs->flush_hdp(adev, NULL((void *)0));
900
901 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS2) ?
902 false0 : true1;
903
904 adev->mmhub.funcs->set_fault_enable_default(adev, value);
905 gmc_v11_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB_01, 0);
906
907 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",printk("\0016" "[" "drm" "] " "PCIE GART of %uM enabled (table at 0x%016llX).\n"
, (unsigned)(adev->gmc.gart_size >> 20), (unsigned long
long)amdgpu_bo_gpu_offset(adev->gart.bo))
908 (unsigned)(adev->gmc.gart_size >> 20),printk("\0016" "[" "drm" "] " "PCIE GART of %uM enabled (table at 0x%016llX).\n"
, (unsigned)(adev->gmc.gart_size >> 20), (unsigned long
long)amdgpu_bo_gpu_offset(adev->gart.bo))
909 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo))printk("\0016" "[" "drm" "] " "PCIE GART of %uM enabled (table at 0x%016llX).\n"
, (unsigned)(adev->gmc.gart_size >> 20), (unsigned long
long)amdgpu_bo_gpu_offset(adev->gart.bo));
910
911 return 0;
912}
913
914static int gmc_v11_0_hw_init(void *handle)
915{
916 int r;
917 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
918
919 /* The sequence of these two function calls matters.*/
920 gmc_v11_0_init_golden_registers(adev);
921
922 r = gmc_v11_0_gart_enable(adev);
923 if (r)
924 return r;
925
926 if (adev->umc.funcs && adev->umc.funcs->init_registers)
927 adev->umc.funcs->init_registers(adev);
928
929 return 0;
930}
931
932/**
933 * gmc_v11_0_gart_disable - gart disable
934 *
935 * @adev: amdgpu_device pointer
936 *
937 * This disables all VM page table.
938 */
939static void gmc_v11_0_gart_disable(struct amdgpu_device *adev)
940{
941 adev->mmhub.funcs->gart_disable(adev);
942}
943
944static int gmc_v11_0_hw_fini(void *handle)
945{
946 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
947
948 if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
949 /* full access mode, so don't touch any GMC register */
950 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n")___drm_dbg(((void *)0), DRM_UT_CORE, "For SRIOV client, shouldn't do anything.\n"
);
951 return 0;
952 }
953
954 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
955 gmc_v11_0_gart_disable(adev);
956
957 return 0;
958}
959
960static int gmc_v11_0_suspend(void *handle)
961{
962 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
963
964 gmc_v11_0_hw_fini(adev);
965
966 return 0;
967}
968
969static int gmc_v11_0_resume(void *handle)
970{
971 int r;
972 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
973
974 r = gmc_v11_0_hw_init(adev);
975 if (r)
976 return r;
977
978 amdgpu_vmid_reset_all(adev);
979
980 return 0;
981}
982
983static bool_Bool gmc_v11_0_is_idle(void *handle)
984{
985 /* MC is always ready in GMC v11.*/
986 return true1;
987}
988
989static int gmc_v11_0_wait_for_idle(void *handle)
990{
991 /* There is no need to wait for MC idle in GMC v11.*/
992 return 0;
993}
994
995static int gmc_v11_0_soft_reset(void *handle)
996{
997 return 0;
998}
999
1000static int gmc_v11_0_set_clockgating_state(void *handle,
1001 enum amd_clockgating_state state)
1002{
1003 int r;
1004 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1005
1006 r = adev->mmhub.funcs->set_clockgating(adev, state);
1007 if (r)
1008 return r;
1009
1010 return athub_v3_0_set_clockgating(adev, state);
1011}
1012
1013static void gmc_v11_0_get_clockgating_state(void *handle, u64 *flags)
1014{
1015 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1016
1017 adev->mmhub.funcs->get_clockgating(adev, flags);
1018
1019 athub_v3_0_get_clockgating(adev, flags);
1020}
1021
1022static int gmc_v11_0_set_powergating_state(void *handle,
1023 enum amd_powergating_state state)
1024{
1025 return 0;
1026}
1027
1028const struct amd_ip_funcs gmc_v11_0_ip_funcs = {
1029 .name = "gmc_v11_0",
1030 .early_init = gmc_v11_0_early_init,
1031 .sw_init = gmc_v11_0_sw_init,
1032 .hw_init = gmc_v11_0_hw_init,
1033 .late_init = gmc_v11_0_late_init,
1034 .sw_fini = gmc_v11_0_sw_fini,
1035 .hw_fini = gmc_v11_0_hw_fini,
1036 .suspend = gmc_v11_0_suspend,
1037 .resume = gmc_v11_0_resume,
1038 .is_idle = gmc_v11_0_is_idle,
1039 .wait_for_idle = gmc_v11_0_wait_for_idle,
1040 .soft_reset = gmc_v11_0_soft_reset,
1041 .set_clockgating_state = gmc_v11_0_set_clockgating_state,
1042 .set_powergating_state = gmc_v11_0_set_powergating_state,
1043 .get_clockgating_state = gmc_v11_0_get_clockgating_state,
1044};
1045
1046const struct amdgpu_ip_block_version gmc_v11_0_ip_block = {
1047 .type = AMD_IP_BLOCK_TYPE_GMC,
1048 .major = 11,
1049 .minor = 0,
1050 .rev = 0,
1051 .funcs = &gmc_v11_0_ip_funcs,
1052};