Bug Summary

File:dev/pci/drm/amd/amdgpu/mes_v11_0.c
Warning:line 821, column 2
Value stored to 'data' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name mes_v11_0.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c
1/*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/firmware.h>
25#include <linux/module.h>
26#include "amdgpu.h"
27#include "soc15_common.h"
28#include "soc21.h"
29#include "gc/gc_11_0_0_offset.h"
30#include "gc/gc_11_0_0_sh_mask.h"
31#include "gc/gc_11_0_0_default.h"
32#include "v11_structs.h"
33#include "mes_v11_api_def.h"
34
35MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin");
36MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes_2.bin");
37MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin");
38MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");
39MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes_2.bin");
40MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");
41MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");
42MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes_2.bin");
43MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");
44MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin");
45MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes_2.bin");
46MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin");
47MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes.bin");
48MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes_2.bin");
49MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes1.bin");
50
51static int mes_v11_0_hw_fini(void *handle);
52static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);
53static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev);
54
55#define MES_EOP_SIZE2048 2048
56
57static void mes_v11_0_ring_set_wptr(struct amdgpu_ring *ring)
58{
59 struct amdgpu_device *adev = ring->adev;
60
61 if (ring->use_doorbell) {
62 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,({ typeof(*((atomic64_t *)ring->wptr_cpu_addr)) __tmp = ((
ring->wptr)); *(volatile typeof(*((atomic64_t *)ring->wptr_cpu_addr
)) *)&(*((atomic64_t *)ring->wptr_cpu_addr)) = __tmp; __tmp
; })
63 ring->wptr)({ typeof(*((atomic64_t *)ring->wptr_cpu_addr)) __tmp = ((
ring->wptr)); *(volatile typeof(*((atomic64_t *)ring->wptr_cpu_addr
)) *)&(*((atomic64_t *)ring->wptr_cpu_addr)) = __tmp; __tmp
; });
64 WDOORBELL64(ring->doorbell_index, ring->wptr)amdgpu_mm_wdoorbell64(adev, (ring->doorbell_index), (ring->
wptr));
65 } else {
66 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c"
, 66); } while (0);
67 }
68}
69
70static u64 mes_v11_0_ring_get_rptr(struct amdgpu_ring *ring)
71{
72 return *ring->rptr_cpu_addr;
73}
74
75static u64 mes_v11_0_ring_get_wptr(struct amdgpu_ring *ring)
76{
77 u64 wptr;
78
79 if (ring->use_doorbell)
80 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr)({ typeof(*((atomic64_t *)ring->wptr_cpu_addr)) __tmp = *(
volatile typeof(*((atomic64_t *)ring->wptr_cpu_addr)) *)&
(*((atomic64_t *)ring->wptr_cpu_addr)); membar_datadep_consumer
(); __tmp; });
81 else
82 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c"
, 82); } while (0);
83 return wptr;
84}
85
86static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = {
87 .type = AMDGPU_RING_TYPE_MES,
88 .align_mask = 1,
89 .nop = 0,
90 .support_64bit_ptrs = true1,
91 .get_rptr = mes_v11_0_ring_get_rptr,
92 .get_wptr = mes_v11_0_ring_get_wptr,
93 .set_wptr = mes_v11_0_ring_set_wptr,
94 .insert_nop = amdgpu_ring_insert_nop,
95};
96
97static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
98 void *pkt, int size,
99 int api_status_off)
100{
101 int ndw = size / 4;
102 signed long r;
103 union MESAPI__ADD_QUEUE *x_pkt = pkt;
104 struct MES_API_STATUS *api_status;
105 struct amdgpu_device *adev = mes->adev;
106 struct amdgpu_ring *ring = &mes->ring;
107 unsigned long flags;
108 signed long timeout = adev->usec_timeout;
109
110 if (amdgpu_emu_mode) {
111 timeout *= 100;
112 } else if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) {
113 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
114 timeout = 15 * 600 * 1000;
115 }
116 BUG_ON(size % 4 != 0)((!(size % 4 != 0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c"
, 116, "!(size % 4 != 0)"));
117
118 spin_lock_irqsave(&mes->ring_lock, flags)do { flags = 0; mtx_enter(&mes->ring_lock); } while (0
);
119 if (amdgpu_ring_alloc(ring, ndw)) {
120 spin_unlock_irqrestore(&mes->ring_lock, flags)do { (void)(flags); mtx_leave(&mes->ring_lock); } while
(0);
121 return -ENOMEM12;
122 }
123
124 api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
125 api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;
126 api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;
127
128 amdgpu_ring_write_multiple(ring, pkt, ndw);
129 amdgpu_ring_commit(ring);
130 spin_unlock_irqrestore(&mes->ring_lock, flags)do { (void)(flags); mtx_leave(&mes->ring_lock); } while
(0);
131
132 DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode)___drm_dbg(((void *)0), DRM_UT_CORE, "MES msg=%d was emitted\n"
, x_pkt->header.opcode);
133
134 r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq,
135 timeout);
136 if (r < 1) {
137 DRM_ERROR("MES failed to response msg=%d\n",__drm_err("MES failed to response msg=%d\n", x_pkt->header
.opcode)
138 x_pkt->header.opcode)__drm_err("MES failed to response msg=%d\n", x_pkt->header
.opcode);
139 return -ETIMEDOUT60;
140 }
141
142 return 0;
143}
144
145static int convert_to_mes_queue_type(int queue_type)
146{
147 if (queue_type == AMDGPU_RING_TYPE_GFX)
148 return MES_QUEUE_TYPE_GFX;
149 else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
150 return MES_QUEUE_TYPE_COMPUTE;
151 else if (queue_type == AMDGPU_RING_TYPE_SDMA)
152 return MES_QUEUE_TYPE_SDMA;
153 else
154 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c"
, 154); } while (0);
155 return -1;
156}
157
158static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
159 struct mes_add_queue_input *input)
160{
161 struct amdgpu_device *adev = mes->adev;
162 union MESAPI__ADD_QUEUE mes_add_queue_pkt;
163 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_00];
164 uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
165
166 memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt))__builtin_memset((&mes_add_queue_pkt), (0), (sizeof(mes_add_queue_pkt
)));
167
168 mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
169 mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
170 mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
171
172 mes_add_queue_pkt.process_id = input->process_id;
173 mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
174 mes_add_queue_pkt.process_va_start = input->process_va_start;
175 mes_add_queue_pkt.process_va_end = input->process_va_end;
176 mes_add_queue_pkt.process_quantum = input->process_quantum;
177 mes_add_queue_pkt.process_context_addr = input->process_context_addr;
178 mes_add_queue_pkt.gang_quantum = input->gang_quantum;
179 mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
180 mes_add_queue_pkt.inprocess_gang_priority =
181 input->inprocess_gang_priority;
182 mes_add_queue_pkt.gang_global_priority_level =
183 input->gang_global_priority_level;
184 mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
185 mes_add_queue_pkt.mqd_addr = input->mqd_addr;
186
187 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK0x00fff000) >>
188 AMDGPU_MES_API_VERSION_SHIFT12) >= 2)
189 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
190 else
191 mes_add_queue_pkt.wptr_addr = input->wptr_addr;
192
193 mes_add_queue_pkt.queue_type =
194 convert_to_mes_queue_type(input->queue_type);
195 mes_add_queue_pkt.paging = input->paging;
196 mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
197 mes_add_queue_pkt.gws_base = input->gws_base;
198 mes_add_queue_pkt.gws_size = input->gws_size;
199 mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
200 mes_add_queue_pkt.tma_addr = input->tma_addr;
201 mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
202
203 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
204 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
205 mes_add_queue_pkt.gds_size = input->queue_size;
206
207 if (!(((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK0x00000fff) >= 4) &&
208 (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)(((11) << 16) | ((0) << 8) | (0))) &&
209 (adev->ip_versions[GC_HWIP][0] <= IP_VERSION(11, 0, 3)(((11) << 16) | ((0) << 8) | (3)))))
210 mes_add_queue_pkt.trap_en = 1;
211
212 /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
213 mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
214 mes_add_queue_pkt.gds_size = input->queue_size;
215
216 return mes_v11_0_submit_pkt_and_poll_completion(mes,
217 &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
218 offsetof(union MESAPI__ADD_QUEUE, api_status)__builtin_offsetof(union MESAPI__ADD_QUEUE, api_status));
219}
220
221static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,
222 struct mes_remove_queue_input *input)
223{
224 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
225
226 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt))__builtin_memset((&mes_remove_queue_pkt), (0), (sizeof(mes_remove_queue_pkt
)));
227
228 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
229 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
230 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
231
232 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
233 mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
234
235 return mes_v11_0_submit_pkt_and_poll_completion(mes,
236 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
237 offsetof(union MESAPI__REMOVE_QUEUE, api_status)__builtin_offsetof(union MESAPI__REMOVE_QUEUE, api_status));
238}
239
240static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,
241 struct mes_unmap_legacy_queue_input *input)
242{
243 union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
244
245 memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt))__builtin_memset((&mes_remove_queue_pkt), (0), (sizeof(mes_remove_queue_pkt
)));
246
247 mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
248 mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
249 mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
250
251 mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
252 mes_remove_queue_pkt.gang_context_addr = 0;
253
254 mes_remove_queue_pkt.pipe_id = input->pipe_id;
255 mes_remove_queue_pkt.queue_id = input->queue_id;
256
257 if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
258 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
259 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
260 mes_remove_queue_pkt.tf_data =
261 lower_32_bits(input->trail_fence_data)((u32)(input->trail_fence_data));
262 } else {
263 mes_remove_queue_pkt.unmap_legacy_queue = 1;
264 mes_remove_queue_pkt.queue_type =
265 convert_to_mes_queue_type(input->queue_type);
266 }
267
268 return mes_v11_0_submit_pkt_and_poll_completion(mes,
269 &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
270 offsetof(union MESAPI__REMOVE_QUEUE, api_status)__builtin_offsetof(union MESAPI__REMOVE_QUEUE, api_status));
271}
272
273static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,
274 struct mes_suspend_gang_input *input)
275{
276 return 0;
277}
278
279static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,
280 struct mes_resume_gang_input *input)
281{
282 return 0;
283}
284
285static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)
286{
287 union MESAPI__QUERY_MES_STATUS mes_status_pkt;
288
289 memset(&mes_status_pkt, 0, sizeof(mes_status_pkt))__builtin_memset((&mes_status_pkt), (0), (sizeof(mes_status_pkt
)));
290
291 mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
292 mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
293 mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
294
295 return mes_v11_0_submit_pkt_and_poll_completion(mes,
296 &mes_status_pkt, sizeof(mes_status_pkt),
297 offsetof(union MESAPI__QUERY_MES_STATUS, api_status)__builtin_offsetof(union MESAPI__QUERY_MES_STATUS, api_status
));
298}
299
300static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
301 struct mes_misc_op_input *input)
302{
303 union MESAPI__MISC misc_pkt;
304
305 memset(&misc_pkt, 0, sizeof(misc_pkt))__builtin_memset((&misc_pkt), (0), (sizeof(misc_pkt)));
306
307 misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
308 misc_pkt.header.opcode = MES_SCH_API_MISC;
309 misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
310
311 switch (input->op) {
312 case MES_MISC_OP_READ_REG:
313 misc_pkt.opcode = MESAPI_MISC__READ_REG;
314 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
315 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
316 break;
317 case MES_MISC_OP_WRITE_REG:
318 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
319 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
320 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
321 break;
322 case MES_MISC_OP_WRM_REG_WAIT:
323 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
324 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
325 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
326 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
327 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
328 misc_pkt.wait_reg_mem.reg_offset2 = 0;
329 break;
330 case MES_MISC_OP_WRM_REG_WR_WAIT:
331 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
332 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
333 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
334 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
335 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
336 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
337 break;
338 default:
339 DRM_ERROR("unsupported misc op (%d) \n", input->op)__drm_err("unsupported misc op (%d) \n", input->op);
340 return -EINVAL22;
341 }
342
343 return mes_v11_0_submit_pkt_and_poll_completion(mes,
344 &misc_pkt, sizeof(misc_pkt),
345 offsetof(union MESAPI__MISC, api_status)__builtin_offsetof(union MESAPI__MISC, api_status));
346}
347
348static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
349{
350 int i;
351 struct amdgpu_device *adev = mes->adev;
352 union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
353
354 memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt))__builtin_memset((&mes_set_hw_res_pkt), (0), (sizeof(mes_set_hw_res_pkt
)));
355
356 mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
357 mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
358 mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
359
360 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
361 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
362 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
363 mes_set_hw_res_pkt.paging_vmid = 0;
364 mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;
365 mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
366 mes->query_status_fence_gpu_addr;
367
368 for (i = 0; i < MAX_COMPUTE_PIPES; i++)
369 mes_set_hw_res_pkt.compute_hqd_mask[i] =
370 mes->compute_hqd_mask[i];
371
372 for (i = 0; i < MAX_GFX_PIPES; i++)
373 mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];
374
375 for (i = 0; i < MAX_SDMA_PIPES; i++)
376 mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
377
378 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
379 mes_set_hw_res_pkt.aggregated_doorbells[i] =
380 mes->aggregated_doorbells[i];
381
382 for (i = 0; i < 5; i++) {
383 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
384 mes_set_hw_res_pkt.mmhub_base[i] =
385 adev->reg_offset[MMHUB_HWIP][0][i];
386 mes_set_hw_res_pkt.osssys_base[i] =
387 adev->reg_offset[OSSSYS_HWIP][0][i];
388 }
389
390 mes_set_hw_res_pkt.disable_reset = 1;
391 mes_set_hw_res_pkt.disable_mes_log = 1;
392 mes_set_hw_res_pkt.use_different_vmid_compute = 1;
393 mes_set_hw_res_pkt.enable_reg_active_poll = 1;
394 mes_set_hw_res_pkt.oversubscription_timer = 50;
395
396 return mes_v11_0_submit_pkt_and_poll_completion(mes,
397 &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
398 offsetof(union MESAPI_SET_HW_RESOURCES, api_status)__builtin_offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
399}
400
401static void mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
402{
403 struct amdgpu_device *adev = mes->adev;
404 uint32_t data;
405
406 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x283c, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x283c), 0));
407 data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK0x0FFFFFFCL |
408 CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK0x40000000L |
409 CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK0x80000000L);
410 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
411 CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT0x2;
412 data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT0x1e;
413 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x283c), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x283c)), (data), 0));
414
415 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x283d, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x283d), 0));
416 data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK0x0FFFFFFCL |
417 CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK0x40000000L |
418 CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK0x80000000L);
419 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
420 CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT0x2;
421 data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT0x1e;
422 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x283d), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x283d)), (data), 0));
423
424 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x283e, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x283e), 0));
425 data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK0x0FFFFFFCL |
426 CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK0x40000000L |
427 CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK0x80000000L);
428 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
429 CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT0x2;
430 data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT0x1e;
431 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x283e), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x283e)), (data), 0));
432
433 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x283f, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x283f), 0));
434 data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK0x0FFFFFFCL |
435 CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK0x40000000L |
436 CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK0x80000000L);
437 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
438 CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT0x2;
439 data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT0x1e;
440 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x283f), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x283f)), (data), 0));
441
442 data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2840, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2840), 0));
443 data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK0x0FFFFFFCL |
444 CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK0x40000000L |
445 CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK0x80000000L);
446 data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
447 CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT0x2;
448 data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT0x1e;
449 WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2840), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x2840)), (data), 0));
450
451 data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT0xf;
452 WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1e9f), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][0] + 0x1e9f)), (data), 0));
453}
454
455static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
456 .add_hw_queue = mes_v11_0_add_hw_queue,
457 .remove_hw_queue = mes_v11_0_remove_hw_queue,
458 .unmap_legacy_queue = mes_v11_0_unmap_legacy_queue,
459 .suspend_gang = mes_v11_0_suspend_gang,
460 .resume_gang = mes_v11_0_resume_gang,
461 .misc_op = mes_v11_0_misc_op,
462};
463
464static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
465 enum admgpu_mes_pipe pipe)
466{
467 int r;
468 const struct mes_firmware_header_v1_0 *mes_hdr;
469 const __le32 *fw_data;
470 unsigned fw_size;
471
472 mes_hdr = (const struct mes_firmware_header_v1_0 *)
473 adev->mes.fw[pipe]->data;
474
475 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
476 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)((__uint32_t)(mes_hdr->mes_ucode_offset_bytes)));
477 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes)((__uint32_t)(mes_hdr->mes_ucode_size_bytes));
478
479 r = amdgpu_bo_create_reserved(adev, fw_size,
480 PAGE_SIZE(1 << 12), AMDGPU_GEM_DOMAIN_VRAM0x4,
481 &adev->mes.ucode_fw_obj[pipe],
482 &adev->mes.ucode_fw_gpu_addr[pipe],
483 (void **)&adev->mes.ucode_fw_ptr[pipe]);
484 if (r) {
485 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r)printf("drm:pid%d:%s *ERROR* " "(%d) failed to create mes fw bo\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
486 return r;
487 }
488
489 memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size)__builtin_memcpy((adev->mes.ucode_fw_ptr[pipe]), (fw_data)
, (fw_size));
490
491 amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
492 amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
493
494 return 0;
495}
496
497static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
498 enum admgpu_mes_pipe pipe)
499{
500 int r;
501 const struct mes_firmware_header_v1_0 *mes_hdr;
502 const __le32 *fw_data;
503 unsigned fw_size;
504
505 mes_hdr = (const struct mes_firmware_header_v1_0 *)
506 adev->mes.fw[pipe]->data;
507
508 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
509 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)((__uint32_t)(mes_hdr->mes_ucode_data_offset_bytes)));
510 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes)((__uint32_t)(mes_hdr->mes_ucode_data_size_bytes));
511
512 r = amdgpu_bo_create_reserved(adev, fw_size,
513 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM0x4,
514 &adev->mes.data_fw_obj[pipe],
515 &adev->mes.data_fw_gpu_addr[pipe],
516 (void **)&adev->mes.data_fw_ptr[pipe]);
517 if (r) {
518 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r)printf("drm:pid%d:%s *ERROR* " "(%d) failed to create mes data fw bo\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
519 return r;
520 }
521
522 memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size)__builtin_memcpy((adev->mes.data_fw_ptr[pipe]), (fw_data),
(fw_size));
523
524 amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
525 amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
526
527 return 0;
528}
529
530static void mes_v11_0_free_ucode_buffers(struct amdgpu_device *adev,
531 enum admgpu_mes_pipe pipe)
532{
533 amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
534 &adev->mes.data_fw_gpu_addr[pipe],
535 (void **)&adev->mes.data_fw_ptr[pipe]);
536
537 amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
538 &adev->mes.ucode_fw_gpu_addr[pipe],
539 (void **)&adev->mes.ucode_fw_ptr[pipe]);
540}
541
542static void mes_v11_0_enable(struct amdgpu_device *adev, bool_Bool enable)
543{
544 uint64_t ucode_addr;
545 uint32_t pipe, data = 0;
546
547 if (enable) {
548 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2807, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2807), 0));
549 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1)(((data) & ~0x00010000L) | (0x00010000L & ((1) <<
0x10)));
550 data = REG_SET_FIELD(data, CP_MES_CNTL,(((data) & ~0x00020000L) | (0x00020000L & ((adev->
enable_mes_kiq ? 1 : 0) << 0x11)))
551 MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0)(((data) & ~0x00020000L) | (0x00020000L & ((adev->
enable_mes_kiq ? 1 : 0) << 0x11)));
552 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2807), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x2807)), (data), 0));
553
554 mutex_lock(&adev->srbm_mutex)rw_enter_write(&adev->srbm_mutex);
555 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
556 if (!adev->enable_mes_kiq &&
557 pipe == AMDGPU_MES_KIQ_PIPE)
558 continue;
559
560 soc21_grbm_select(adev, 3, pipe, 0, 0);
561
562 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
563 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2800), ((u32)(ucode_addr)), 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][1] + 0x2800)), (((u32
)(ucode_addr))), 0))
564 lower_32_bits(ucode_addr))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2800), ((u32)(ucode_addr)), 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][1] + 0x2800)), (((u32
)(ucode_addr))), 0));
565 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x289d), ((u32)(((ucode_addr) >> 16) >> 16))
, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset
[GC_HWIP][0][1] + 0x289d)), (((u32)(((ucode_addr) >> 16
) >> 16))), 0))
566 upper_32_bits(ucode_addr))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x289d), ((u32)(((ucode_addr) >> 16) >> 16))
, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset
[GC_HWIP][0][1] + 0x289d)), (((u32)(((ucode_addr) >> 16
) >> 16))), 0));
567 }
568 soc21_grbm_select(adev, 0, 0, 0, 0);
569 mutex_unlock(&adev->srbm_mutex)rw_exit_write(&adev->srbm_mutex);
570
571 /* unhalt MES and activate pipe0 */
572 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1)(((0) & ~0x04000000L) | (0x04000000L & ((1) << 0x1a
)));
573 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE,(((data) & ~0x08000000L) | (0x08000000L & ((adev->
enable_mes_kiq ? 1 : 0) << 0x1b)))
574 adev->enable_mes_kiq ? 1 : 0)(((data) & ~0x08000000L) | (0x08000000L & ((adev->
enable_mes_kiq ? 1 : 0) << 0x1b)));
575 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2807), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x2807)), (data), 0));
576
577 if (amdgpu_emu_mode)
578 drm_msleep(100)mdelay(100);
579 else
580 udelay(50);
581 } else {
582 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2807, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2807), 0));
583 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0)(((data) & ~0x04000000L) | (0x04000000L & ((0) <<
0x1a)));
584 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0)(((data) & ~0x08000000L) | (0x08000000L & ((0) <<
0x1b)));
585 data = REG_SET_FIELD(data, CP_MES_CNTL,(((data) & ~0x00000010L) | (0x00000010L & ((1) <<
0x4)))
586 MES_INVALIDATE_ICACHE, 1)(((data) & ~0x00000010L) | (0x00000010L & ((1) <<
0x4)));
587 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1)(((data) & ~0x00010000L) | (0x00010000L & ((1) <<
0x10)));
588 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET,(((data) & ~0x00020000L) | (0x00020000L & ((adev->
enable_mes_kiq ? 1 : 0) << 0x11)))
589 adev->enable_mes_kiq ? 1 : 0)(((data) & ~0x00020000L) | (0x00020000L & ((adev->
enable_mes_kiq ? 1 : 0) << 0x11)));
590 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1)(((data) & ~0x40000000L) | (0x40000000L & ((1) <<
0x1e)));
591 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2807), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x2807)), (data), 0));
592 }
593}
594
595/* This function is for backdoor MES firmware */
596static int mes_v11_0_load_microcode(struct amdgpu_device *adev,
597 enum admgpu_mes_pipe pipe, bool_Bool prime_icache)
598{
599 int r;
600 uint32_t data;
601 uint64_t ucode_addr;
602
603 mes_v11_0_enable(adev, false0);
604
605 if (!adev->mes.fw[pipe])
606 return -EINVAL22;
607
608 r = mes_v11_0_allocate_ucode_buffer(adev, pipe);
609 if (r)
610 return r;
611
612 r = mes_v11_0_allocate_ucode_data_buffer(adev, pipe);
613 if (r) {
614 mes_v11_0_free_ucode_buffers(adev, pipe);
615 return r;
616 }
617
618 mutex_lock(&adev->srbm_mutex)rw_enter_write(&adev->srbm_mutex);
619 /* me=3, pipe=0, queue=0 */
620 soc21_grbm_select(adev, 3, pipe, 0, 0);
621
622 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5852), 0, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][1] + 0x5852)), (0), 0));
623
624 /* set ucode start address */
625 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
626 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2800), ((u32)(ucode_addr)), 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][1] + 0x2800)), (((u32
)(ucode_addr))), 0))
627 lower_32_bits(ucode_addr))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2800), ((u32)(ucode_addr)), 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][1] + 0x2800)), (((u32
)(ucode_addr))), 0));
628 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x289d), ((u32)(((ucode_addr) >> 16) >> 16))
, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset
[GC_HWIP][0][1] + 0x289d)), (((u32)(((ucode_addr) >> 16
) >> 16))), 0))
629 upper_32_bits(ucode_addr))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x289d), ((u32)(((ucode_addr) >> 16) >> 16))
, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset
[GC_HWIP][0][1] + 0x289d)), (((u32)(((ucode_addr) >> 16
) >> 16))), 0));
630
631 /* set ucode fimrware address */
632 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5850), ((u32)(adev->mes.ucode_fw_gpu_addr[pipe])), 0
, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP
][0][1] + 0x5850)), (((u32)(adev->mes.ucode_fw_gpu_addr[pipe
]))), 0))
633 lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5850), ((u32)(adev->mes.ucode_fw_gpu_addr[pipe])), 0
, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP
][0][1] + 0x5850)), (((u32)(adev->mes.ucode_fw_gpu_addr[pipe
]))), 0));
634 WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5851), ((u32)(((adev->mes.ucode_fw_gpu_addr[pipe]) >>
16) >> 16)), 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x5851)), (((u32)(((adev
->mes.ucode_fw_gpu_addr[pipe]) >> 16) >> 16)))
, 0))
635 upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5851), ((u32)(((adev->mes.ucode_fw_gpu_addr[pipe]) >>
16) >> 16)), 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x5851)), (((u32)(((adev
->mes.ucode_fw_gpu_addr[pipe]) >> 16) >> 16)))
, 0));
636
637 /* set ucode instruction cache boundary to 2M-1 */
638 WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x585b), 0x1FFFFF, 0, GC_HWIP) : amdgpu_device_wreg(adev
, ((adev->reg_offset[GC_HWIP][0][1] + 0x585b)), (0x1FFFFF)
, 0));
639
640 /* set ucode data firmware address */
641 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5854), ((u32)(adev->mes.data_fw_gpu_addr[pipe])), 0
, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP
][0][1] + 0x5854)), (((u32)(adev->mes.data_fw_gpu_addr[pipe
]))), 0))
642 lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5854), ((u32)(adev->mes.data_fw_gpu_addr[pipe])), 0
, GC_HWIP) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP
][0][1] + 0x5854)), (((u32)(adev->mes.data_fw_gpu_addr[pipe
]))), 0));
643 WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5855), ((u32)(((adev->mes.data_fw_gpu_addr[pipe]) >>
16) >> 16)), 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x5855)), (((u32)(((adev
->mes.data_fw_gpu_addr[pipe]) >> 16) >> 16))),
0))
644 upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]))((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x5855), ((u32)(((adev->mes.data_fw_gpu_addr[pipe]) >>
16) >> 16)), 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x5855)), (((u32)(((adev
->mes.data_fw_gpu_addr[pipe]) >> 16) >> 16))),
0));
645
646 /* Set 0x3FFFF (256K-1) to CP_MES_MDBOUND_LO */
647 WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x3FFFF)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x585d), 0x3FFFF, 0, GC_HWIP) : amdgpu_device_wreg(adev,
((adev->reg_offset[GC_HWIP][0][1] + 0x585d)), (0x3FFFF), 0
));
648
649 if (prime_icache) {
650 /* invalidate ICACHE */
651 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2820, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2820), 0));
652 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0)(((data) & ~0x00000010L) | (0x00000010L & ((0) <<
0x4)));
653 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1)(((data) & ~0x00000001L) | (0x00000001L & ((1) <<
0x0)));
654 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2820), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x2820)), (data), 0));
655
656 /* prime the ICACHE. */
657 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2820, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2820), 0));
658 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1)(((data) & ~0x00000010L) | (0x00000010L & ((1) <<
0x4)));
659 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x2820), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][1] + 0x2820)), (data), 0));
660 }
661
662 soc21_grbm_select(adev, 0, 0, 0, 0);
663 mutex_unlock(&adev->srbm_mutex)rw_exit_write(&adev->srbm_mutex);
664
665 return 0;
666}
667
668static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev,
669 enum admgpu_mes_pipe pipe)
670{
671 int r;
672 u32 *eop;
673
674 r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE2048, PAGE_SIZE(1 << 12),
675 AMDGPU_GEM_DOMAIN_GTT0x2,
676 &adev->mes.eop_gpu_obj[pipe],
677 &adev->mes.eop_gpu_addr[pipe],
678 (void **)&eop);
679 if (r) {
680 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r)printf("drm:pid%d:%s *WARNING* " "(%d) create EOP bo failed\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
681 return r;
682 }
683
684 memset(eop, 0,__builtin_memset((eop), (0), (adev->mes.eop_gpu_obj[pipe]->
tbo.base.size))
685 adev->mes.eop_gpu_obj[pipe]->tbo.base.size)__builtin_memset((eop), (0), (adev->mes.eop_gpu_obj[pipe]->
tbo.base.size));
686
687 amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
688 amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
689
690 return 0;
691}
692
693static int mes_v11_0_mqd_init(struct amdgpu_ring *ring)
694{
695 struct v11_compute_mqd *mqd = ring->mqd_ptr;
696 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
697 uint32_t tmp;
698
699 mqd->header = 0xC0310800;
700 mqd->compute_pipelinestat_enable = 0x00000001;
701 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
702 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
703 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
704 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
705 mqd->compute_misc_reserved = 0x00000007;
706
707 eop_base_addr = ring->eop_gpu_addr >> 8;
708
709 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
710 tmp = regCP_HQD_EOP_CONTROL_DEFAULT0x00000006;
711 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,(((tmp) & ~0x0000003FL) | (0x0000003FL & (((drm_order
(2048 / 4) - 1)) << 0x0)))
712 (order_base_2(MES_EOP_SIZE / 4) - 1))(((tmp) & ~0x0000003FL) | (0x0000003FL & (((drm_order
(2048 / 4) - 1)) << 0x0)));
713
714 mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr)((u32)(eop_base_addr));
715 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr)((u32)(((eop_base_addr) >> 16) >> 16));
716 mqd->cp_hqd_eop_control = tmp;
717
718 /* disable the queue if it's active */
719 ring->wptr = 0;
720 mqd->cp_hqd_pq_rptr = 0;
721 mqd->cp_hqd_pq_wptr_lo = 0;
722 mqd->cp_hqd_pq_wptr_hi = 0;
723
724 /* set the pointer to the MQD */
725 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
726 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr)((u32)(((ring->mqd_gpu_addr) >> 16) >> 16));
727
728 /* set MQD vmid to 0 */
729 tmp = regCP_MQD_CONTROL_DEFAULT0x00000100;
730 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0)(((tmp) & ~0x0000000FL) | (0x0000000FL & ((0) <<
0x0)));
731 mqd->cp_mqd_control = tmp;
732
733 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
734 hqd_gpu_addr = ring->gpu_addr >> 8;
735 mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr)((u32)(hqd_gpu_addr));
736 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr)((u32)(((hqd_gpu_addr) >> 16) >> 16));
737
738 /* set the wb address whether it's enabled or not */
739 wb_gpu_addr = ring->rptr_gpu_addr;
740 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
741 mqd->cp_hqd_pq_rptr_report_addr_hi =
742 upper_32_bits(wb_gpu_addr)((u32)(((wb_gpu_addr) >> 16) >> 16)) & 0xffff;
743
744 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
745 wb_gpu_addr = ring->wptr_gpu_addr;
746 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
747 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr)((u32)(((wb_gpu_addr) >> 16) >> 16)) & 0xffff;
748
749 /* set up the HQD, this is similar to CP_RB0_CNTL */
750 tmp = regCP_HQD_PQ_CONTROL_DEFAULT0x00308509;
751 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,(((tmp) & ~0x0000003FL) | (0x0000003FL & (((drm_order
(ring->ring_size / 4) - 1)) << 0x0)))
752 (order_base_2(ring->ring_size / 4) - 1))(((tmp) & ~0x0000003FL) | (0x0000003FL & (((drm_order
(ring->ring_size / 4) - 1)) << 0x0)));
753 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,(((tmp) & ~0x00003F00L) | (0x00003F00L & ((((drm_order
(4096 / 4) - 1) << 8)) << 0x8)))
754 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8))(((tmp) & ~0x00003F00L) | (0x00003F00L & ((((drm_order
(4096 / 4) - 1) << 8)) << 0x8)));
755 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1)(((tmp) & ~0x10000000L) | (0x10000000L & ((1) <<
0x1c)));
756 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0)(((tmp) & ~0x20000000L) | (0x20000000L & ((0) <<
0x1d)));
757 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1)(((tmp) & ~0x40000000L) | (0x40000000L & ((1) <<
0x1e)));
758 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1)(((tmp) & ~0x80000000L) | (0x80000000L & ((1) <<
0x1f)));
759 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1)(((tmp) & ~0x08000000L) | (0x08000000L & ((1) <<
0x1b)));
760 mqd->cp_hqd_pq_control = tmp;
761
762 /* enable doorbell */
763 tmp = 0;
764 if (ring->use_doorbell) {
765 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,(((tmp) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ((ring->doorbell_index
) << 0x2)))
766 DOORBELL_OFFSET, ring->doorbell_index)(((tmp) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ((ring->doorbell_index
) << 0x2)));
767 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,(((tmp) & ~0x40000000L) | (0x40000000L & ((1) <<
0x1e)))
768 DOORBELL_EN, 1)(((tmp) & ~0x40000000L) | (0x40000000L & ((1) <<
0x1e)));
769 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,(((tmp) & ~0x10000000L) | (0x10000000L & ((0) <<
0x1c)))
770 DOORBELL_SOURCE, 0)(((tmp) & ~0x10000000L) | (0x10000000L & ((0) <<
0x1c)));
771 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,(((tmp) & ~0x80000000L) | (0x80000000L & ((0) <<
0x1f)))
772 DOORBELL_HIT, 0)(((tmp) & ~0x80000000L) | (0x80000000L & ((0) <<
0x1f)));
773 }
774 else
775 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,(((tmp) & ~0x40000000L) | (0x40000000L & ((0) <<
0x1e)))
776 DOORBELL_EN, 0)(((tmp) & ~0x40000000L) | (0x40000000L & ((0) <<
0x1e)));
777 mqd->cp_hqd_pq_doorbell_control = tmp;
778
779 mqd->cp_hqd_vmid = 0;
780 /* activate the queue */
781 mqd->cp_hqd_active = 1;
782
783 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT0x0be05501;
784 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,(((tmp) & ~0x0003FF00L) | (0x0003FF00L & ((0x55) <<
0x8)))
785 PRELOAD_SIZE, 0x55)(((tmp) & ~0x0003FF00L) | (0x0003FF00L & ((0x55) <<
0x8)));
786 mqd->cp_hqd_persistent_state = tmp;
787
788 mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT0x00300000;
789 mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT0x00000000;
790 mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT0x00000000;
791
792 return 0;
793}
794
795static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring)
796{
797 struct v11_compute_mqd *mqd = ring->mqd_ptr;
798 struct amdgpu_device *adev = ring->adev;
799 uint32_t data = 0;
800
801 mutex_lock(&adev->srbm_mutex)rw_enter_write(&adev->srbm_mutex);
802 soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
803
804 /* set CP_HQD_VMID.VMID = 0. */
805 data = RREG32_SOC15(GC, 0, regCP_HQD_VMID)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][0
] + 0x1fac, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][0] + 0x1fac), 0));
806 data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0)(((data) & ~0x0000000FL) | (0x0000000FL & ((0) <<
0x0)));
807 WREG32_SOC15(GC, 0, regCP_HQD_VMID, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fac), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][0] + 0x1fac)), (data), 0));
808
809 /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
810 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][0
] + 0x1fb8, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][0] + 0x1fb8), 0));
811 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,(((data) & ~0x40000000L) | (0x40000000L & ((0) <<
0x1e)))
812 DOORBELL_EN, 0)(((data) & ~0x40000000L) | (0x40000000L & ((0) <<
0x1e)));
813 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb8), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][0] + 0x1fb8)), (data), 0));
814
815 /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
816 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fa9), mqd->cp_mqd_base_addr_lo, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1fa9)), (mqd->
cp_mqd_base_addr_lo), 0));
817 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1faa), mqd->cp_mqd_base_addr_hi, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1faa)), (mqd->
cp_mqd_base_addr_hi), 0));
818
819 /* set CP_MQD_CONTROL.VMID=0 */
820 data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][0
] + 0x1fcb, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][0] + 0x1fcb), 0));
821 data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0)(((data) & ~0x0000000FL) | (0x0000000FL & ((0) <<
0x0)));
Value stored to 'data' is never read
822 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fcb), 0, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][0] + 0x1fcb)), (0), 0));
823
824 /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
825 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb1), mqd->cp_hqd_pq_base_lo, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1fb1)), (mqd->
cp_hqd_pq_base_lo), 0));
826 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb2), mqd->cp_hqd_pq_base_hi, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1fb2)), (mqd->
cp_hqd_pq_base_hi), 0));
827
828 /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
829 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb4), mqd->cp_hqd_pq_rptr_report_addr_lo, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb4)), (mqd->cp_hqd_pq_rptr_report_addr_lo), 0))
830 mqd->cp_hqd_pq_rptr_report_addr_lo)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb4), mqd->cp_hqd_pq_rptr_report_addr_lo, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb4)), (mqd->cp_hqd_pq_rptr_report_addr_lo), 0));
831 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb5), mqd->cp_hqd_pq_rptr_report_addr_hi, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb5)), (mqd->cp_hqd_pq_rptr_report_addr_hi), 0))
832 mqd->cp_hqd_pq_rptr_report_addr_hi)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb5), mqd->cp_hqd_pq_rptr_report_addr_hi, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb5)), (mqd->cp_hqd_pq_rptr_report_addr_hi), 0));
833
834 /* set CP_HQD_PQ_CONTROL */
835 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fba), mqd->cp_hqd_pq_control, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1fba)), (mqd->
cp_hqd_pq_control), 0));
836
837 /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
838 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb6), mqd->cp_hqd_pq_wptr_poll_addr_lo, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb6)), (mqd->cp_hqd_pq_wptr_poll_addr_lo), 0))
839 mqd->cp_hqd_pq_wptr_poll_addr_lo)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb6), mqd->cp_hqd_pq_wptr_poll_addr_lo, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb6)), (mqd->cp_hqd_pq_wptr_poll_addr_lo), 0));
840 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb7), mqd->cp_hqd_pq_wptr_poll_addr_hi, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb7)), (mqd->cp_hqd_pq_wptr_poll_addr_hi), 0))
841 mqd->cp_hqd_pq_wptr_poll_addr_hi)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb7), mqd->cp_hqd_pq_wptr_poll_addr_hi, 0, GC_HWIP
) : amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0
][0] + 0x1fb7)), (mqd->cp_hqd_pq_wptr_poll_addr_hi), 0));
842
843 /* set CP_HQD_PQ_DOORBELL_CONTROL */
844 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb8), mqd->cp_hqd_pq_doorbell_control, 0, GC_HWIP)
: amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0]
[0] + 0x1fb8)), (mqd->cp_hqd_pq_doorbell_control), 0))
845 mqd->cp_hqd_pq_doorbell_control)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb8), mqd->cp_hqd_pq_doorbell_control, 0, GC_HWIP)
: amdgpu_device_wreg(adev, ((adev->reg_offset[GC_HWIP][0]
[0] + 0x1fb8)), (mqd->cp_hqd_pq_doorbell_control), 0));
846
847 /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
848 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fad), mqd->cp_hqd_persistent_state, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1fad)), (mqd->
cp_hqd_persistent_state), 0));
849
850 /* set CP_HQD_ACTIVE.ACTIVE=1 */
851 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fab), mqd->cp_hqd_active, 0, GC_HWIP) : amdgpu_device_wreg
(adev, ((adev->reg_offset[GC_HWIP][0][0] + 0x1fab)), (mqd->
cp_hqd_active), 0));
852
853 soc21_grbm_select(adev, 0, 0, 0, 0);
854 mutex_unlock(&adev->srbm_mutex)rw_exit_write(&adev->srbm_mutex);
855}
856
857static int mes_v11_0_kiq_enable_queue(struct amdgpu_device *adev)
858{
859 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
860 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
861 int r;
862
863 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
864 return -EINVAL22;
865
866 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
867 if (r) {
868 DRM_ERROR("Failed to lock KIQ (%d).\n", r)__drm_err("Failed to lock KIQ (%d).\n", r);
869 return r;
870 }
871
872 kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);
873
874 r = amdgpu_ring_test_ring(kiq_ring)(kiq_ring)->funcs->test_ring((kiq_ring));
875 if (r) {
876 DRM_ERROR("kfq enable failed\n")__drm_err("kfq enable failed\n");
877 kiq_ring->sched.ready = false0;
878 }
879 return r;
880}
881
882static int mes_v11_0_queue_init(struct amdgpu_device *adev,
883 enum admgpu_mes_pipe pipe)
884{
885 struct amdgpu_ring *ring;
886 int r;
887
888 if (pipe == AMDGPU_MES_KIQ_PIPE)
889 ring = &adev->gfx.kiq.ring;
890 else if (pipe == AMDGPU_MES_SCHED_PIPE)
891 ring = &adev->mes.ring;
892 else
893 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c"
, 893); } while (0);
894
895 if ((pipe == AMDGPU_MES_SCHED_PIPE) &&
896 (amdgpu_in_reset(adev) || adev->in_suspend)) {
897 *(ring->wptr_cpu_addr) = 0;
898 *(ring->rptr_cpu_addr) = 0;
899 amdgpu_ring_clear_ring(ring);
900 }
901
902 r = mes_v11_0_mqd_init(ring);
903 if (r)
904 return r;
905
906 if (pipe == AMDGPU_MES_SCHED_PIPE) {
907 r = mes_v11_0_kiq_enable_queue(adev);
908 if (r)
909 return r;
910 } else {
911 mes_v11_0_queue_init_register(ring);
912 }
913
914 /* get MES scheduler/KIQ versions */
915 mutex_lock(&adev->srbm_mutex)rw_enter_write(&adev->srbm_mutex);
916 soc21_grbm_select(adev, 3, pipe, 0, 0);
917
918 if (pipe == AMDGPU_MES_SCHED_PIPE)
919 adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2849, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2849), 0));
920 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
921 adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x2849, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x2849), 0));
922
923 soc21_grbm_select(adev, 0, 0, 0, 0);
924 mutex_unlock(&adev->srbm_mutex)rw_exit_write(&adev->srbm_mutex);
925
926 return 0;
927}
928
929static int mes_v11_0_ring_init(struct amdgpu_device *adev)
930{
931 struct amdgpu_ring *ring;
932
933 ring = &adev->mes.ring;
934
935 ring->funcs = &mes_v11_0_ring_funcs;
936
937 ring->me = 3;
938 ring->pipe = 0;
939 ring->queue = 0;
940
941 ring->ring_obj = NULL((void *)0);
942 ring->use_doorbell = true1;
943 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
944 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];
945 ring->no_scheduler = true1;
946 snprintf(ring->name, sizeof(ring->name), "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
947
948 return amdgpu_ring_init(adev, ring, 1024, NULL((void *)0), 0,
949 AMDGPU_RING_PRIO_DEFAULT, NULL((void *)0));
950}
951
952static int mes_v11_0_kiq_ring_init(struct amdgpu_device *adev)
953{
954 struct amdgpu_ring *ring;
955
956 mtx_init(&adev->gfx.kiq.ring_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&adev->
gfx.kiq.ring_lock), ((((0x9)) > 0x0 && ((0x9)) <
0x9) ? 0x9 : ((0x9)))); } while (0);
957
958 ring = &adev->gfx.kiq.ring;
959
960 ring->me = 3;
961 ring->pipe = 1;
962 ring->queue = 0;
963
964 ring->adev = NULL((void *)0);
965 ring->ring_obj = NULL((void *)0);
966 ring->use_doorbell = true1;
967 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
968 ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
969 ring->no_scheduler = true1;
970 snprintf(ring->name, sizeof(ring->name), "mes_kiq_%d.%d.%d",
971 ring->me, ring->pipe, ring->queue);
972
973 return amdgpu_ring_init(adev, ring, 1024, NULL((void *)0), 0,
974 AMDGPU_RING_PRIO_DEFAULT, NULL((void *)0));
975}
976
977static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev,
978 enum admgpu_mes_pipe pipe)
979{
980 int r, mqd_size = sizeof(struct v11_compute_mqd);
981 struct amdgpu_ring *ring;
982
983 if (pipe == AMDGPU_MES_KIQ_PIPE)
984 ring = &adev->gfx.kiq.ring;
985 else if (pipe == AMDGPU_MES_SCHED_PIPE)
986 ring = &adev->mes.ring;
987 else
988 BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/mes_v11_0.c"
, 988); } while (0);
989
990 if (ring->mqd_obj)
991 return 0;
992
993 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE(1 << 12),
994 AMDGPU_GEM_DOMAIN_GTT0x2, &ring->mqd_obj,
995 &ring->mqd_gpu_addr, &ring->mqd_ptr);
996 if (r) {
997 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r)printf("drm:pid%d:%s *WARNING* " "failed to create ring mqd bo (%d)"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , r);
998 return r;
999 }
1000
1001 memset(ring->mqd_ptr, 0, mqd_size)__builtin_memset((ring->mqd_ptr), (0), (mqd_size));
1002
1003 /* prepare MQD backup */
1004 adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL(0x0001 | 0x0004));
1005 if (!adev->mes.mqd_backup[pipe])
1006 dev_warn(adev->dev,printf("drm:pid%d:%s *WARNING* " "no memory to create MQD backup for ring %s\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , ring->
name)
1007 "no memory to create MQD backup for ring %s\n",printf("drm:pid%d:%s *WARNING* " "no memory to create MQD backup for ring %s\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , ring->
name)
1008 ring->name)printf("drm:pid%d:%s *WARNING* " "no memory to create MQD backup for ring %s\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , ring->
name);
1009
1010 return 0;
1011}
1012
1013static int mes_v11_0_sw_init(void *handle)
1014{
1015 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1016 int pipe, r;
1017
1018 adev->mes.adev = adev;
1019 adev->mes.funcs = &mes_v11_0_funcs;
1020 adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;
1021 adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;
1022
1023 r = amdgpu_mes_init(adev);
1024 if (r)
1025 return r;
1026
1027 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1028 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1029 continue;
1030
1031 r = mes_v11_0_allocate_eop_buf(adev, pipe);
1032 if (r)
1033 return r;
1034
1035 r = mes_v11_0_mqd_sw_init(adev, pipe);
1036 if (r)
1037 return r;
1038 }
1039
1040 if (adev->enable_mes_kiq) {
1041 r = mes_v11_0_kiq_ring_init(adev);
1042 if (r)
1043 return r;
1044 }
1045
1046 r = mes_v11_0_ring_init(adev);
1047 if (r)
1048 return r;
1049
1050 return 0;
1051}
1052
1053static int mes_v11_0_sw_fini(void *handle)
1054{
1055 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1056 int pipe;
1057
1058 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
1059 amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
1060
1061 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1062 kfree(adev->mes.mqd_backup[pipe]);
1063
1064 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1065 &adev->mes.eop_gpu_addr[pipe],
1066 NULL((void *)0));
1067 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1068 }
1069
1070 amdgpu_bo_free_kernel(&adev->gfx.kiq.ring.mqd_obj,
1071 &adev->gfx.kiq.ring.mqd_gpu_addr,
1072 &adev->gfx.kiq.ring.mqd_ptr);
1073
1074 amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,
1075 &adev->mes.ring.mqd_gpu_addr,
1076 &adev->mes.ring.mqd_ptr);
1077
1078 amdgpu_ring_fini(&adev->gfx.kiq.ring);
1079 amdgpu_ring_fini(&adev->mes.ring);
1080
1081 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1082 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1083 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1084 }
1085
1086 amdgpu_mes_fini(adev);
1087 return 0;
1088}
1089
1090static void mes_v11_0_kiq_dequeue_sched(struct amdgpu_device *adev)
1091{
1092 uint32_t data;
1093 int i;
1094
1095 mutex_lock(&adev->srbm_mutex)rw_enter_write(&adev->srbm_mutex);
1096 soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
1097
1098 /* disable the queue if it's active */
1099 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][0
] + 0x1fab, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][0] + 0x1fab), 0)) & 1) {
1100 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fc1), 1, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][0] + 0x1fc1)), (1), 0));
1101 for (i = 0; i < adev->usec_timeout; i++) {
1102 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][0
] + 0x1fab, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][0] + 0x1fab), 0)) & 1))
1103 break;
1104 udelay(1);
1105 }
1106 }
1107 data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][0
] + 0x1fb8, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][0] + 0x1fb8), 0));
1108 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,(((data) & ~0x40000000L) | (0x40000000L & ((0) <<
0x1e)))
1109 DOORBELL_EN, 0)(((data) & ~0x40000000L) | (0x40000000L & ((0) <<
0x1e)));
1110 data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,(((data) & ~0x80000000L) | (0x80000000L & ((1) <<
0x1f)))
1111 DOORBELL_HIT, 1)(((data) & ~0x80000000L) | (0x80000000L & ((1) <<
0x1f)));
1112 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb8), data, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((
adev->reg_offset[GC_HWIP][0][0] + 0x1fb8)), (data), 0));
1113
1114 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb8), 0, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][0] + 0x1fb8)), (0), 0));
1115
1116 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fdf), 0, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][0] + 0x1fdf)), (0), 0));
1117 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fe0), 0, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][0] + 0x1fe0)), (0), 0));
1118 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
0] + 0x1fb3), 0, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][0] + 0x1fb3)), (0), 0));
1119
1120 soc21_grbm_select(adev, 0, 0, 0, 0);
1121 mutex_unlock(&adev->srbm_mutex)rw_exit_write(&adev->srbm_mutex);
1122
1123 adev->mes.ring.sched.ready = false0;
1124}
1125
1126static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)
1127{
1128 uint32_t tmp;
1129 struct amdgpu_device *adev = ring->adev;
1130
1131 /* tell RLC which is KIQ queue */
1132 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_rreg(adev, adev->reg_offset[GC_HWIP][0][1
] + 0x098a, 0, GC_HWIP) : amdgpu_device_rreg(adev, (adev->
reg_offset[GC_HWIP][0][1] + 0x098a), 0));
1133 tmp &= 0xffffff00;
1134 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1135 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x098a), tmp, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][1] + 0x098a)), (tmp), 0));
1136 tmp |= 0x80;
1137 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp)((((adev)->virt.caps & (1 << 2)) && adev
->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported
) ? amdgpu_sriov_wreg(adev, (adev->reg_offset[GC_HWIP][0][
1] + 0x098a), tmp, 0, GC_HWIP) : amdgpu_device_wreg(adev, ((adev
->reg_offset[GC_HWIP][0][1] + 0x098a)), (tmp), 0));
1138}
1139
1140static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
1141{
1142 int r = 0;
1143
1144 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1145
1146 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false0);
1147 if (r) {
1148 DRM_ERROR("failed to load MES fw, r=%d\n", r)__drm_err("failed to load MES fw, r=%d\n", r);
1149 return r;
1150 }
1151
1152 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true1);
1153 if (r) {
1154 DRM_ERROR("failed to load MES kiq fw, r=%d\n", r)__drm_err("failed to load MES kiq fw, r=%d\n", r);
1155 return r;
1156 }
1157
1158 }
1159
1160 mes_v11_0_enable(adev, true1);
1161
1162 mes_v11_0_kiq_setting(&adev->gfx.kiq.ring);
1163
1164 r = mes_v11_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1165 if (r)
1166 goto failure;
1167
1168 return r;
1169
1170failure:
1171 mes_v11_0_hw_fini(adev);
1172 return r;
1173}
1174
1175static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)
1176{
1177 if (adev->mes.ring.sched.ready)
1178 mes_v11_0_kiq_dequeue_sched(adev);
1179
1180 mes_v11_0_enable(adev, false0);
1181 return 0;
1182}
1183
1184static int mes_v11_0_hw_init(void *handle)
1185{
1186 int r;
1187 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1188
1189 if (!adev->enable_mes_kiq) {
1190 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1191 r = mes_v11_0_load_microcode(adev,
1192 AMDGPU_MES_SCHED_PIPE, true1);
1193 if (r) {
1194 DRM_ERROR("failed to MES fw, r=%d\n", r)__drm_err("failed to MES fw, r=%d\n", r);
1195 return r;
1196 }
1197 }
1198
1199 mes_v11_0_enable(adev, true1);
1200 }
1201
1202 r = mes_v11_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1203 if (r)
1204 goto failure;
1205
1206 r = mes_v11_0_set_hw_resources(&adev->mes);
1207 if (r)
1208 goto failure;
1209
1210 mes_v11_0_init_aggregated_doorbell(&adev->mes);
1211
1212 r = mes_v11_0_query_sched_status(&adev->mes);
1213 if (r) {
1214 DRM_ERROR("MES is busy\n")__drm_err("MES is busy\n");
1215 goto failure;
1216 }
1217
1218 /*
1219 * Disable KIQ ring usage from the driver once MES is enabled.
1220 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1221 * with MES enabled.
1222 */
1223 adev->gfx.kiq.ring.sched.ready = false0;
1224 adev->mes.ring.sched.ready = true1;
1225
1226 return 0;
1227
1228failure:
1229 mes_v11_0_hw_fini(adev);
1230 return r;
1231}
1232
1233static int mes_v11_0_hw_fini(void *handle)
1234{
1235 return 0;
1236}
1237
1238static int mes_v11_0_suspend(void *handle)
1239{
1240 int r;
1241 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1242
1243 r = amdgpu_mes_suspend(adev);
1244 if (r)
1245 return r;
1246
1247 return mes_v11_0_hw_fini(adev);
1248}
1249
1250static int mes_v11_0_resume(void *handle)
1251{
1252 int r;
1253 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1254
1255 r = mes_v11_0_hw_init(adev);
1256 if (r)
1257 return r;
1258
1259 return amdgpu_mes_resume(adev);
1260}
1261
1262static int mes_v11_0_early_init(void *handle)
1263{
1264 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1265 int pipe, r;
1266
1267 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1268 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1269 continue;
1270 r = amdgpu_mes_init_microcode(adev, pipe);
1271 if (r)
1272 return r;
1273 }
1274
1275 return 0;
1276}
1277
1278static int mes_v11_0_late_init(void *handle)
1279{
1280 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1281
1282 /* it's only intended for use in mes_self_test case, not for s0ix and reset */
1283 if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend &&
1284 (adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3)(((11) << 16) | ((0) << 8) | (3))))
1285 amdgpu_mes_self_test(adev);
1286
1287 return 0;
1288}
1289
1290static const struct amd_ip_funcs mes_v11_0_ip_funcs = {
1291 .name = "mes_v11_0",
1292 .early_init = mes_v11_0_early_init,
1293 .late_init = mes_v11_0_late_init,
1294 .sw_init = mes_v11_0_sw_init,
1295 .sw_fini = mes_v11_0_sw_fini,
1296 .hw_init = mes_v11_0_hw_init,
1297 .hw_fini = mes_v11_0_hw_fini,
1298 .suspend = mes_v11_0_suspend,
1299 .resume = mes_v11_0_resume,
1300};
1301
1302const struct amdgpu_ip_block_version mes_v11_0_ip_block = {
1303 .type = AMD_IP_BLOCK_TYPE_MES,
1304 .major = 11,
1305 .minor = 0,
1306 .rev = 0,
1307 .funcs = &mes_v11_0_ip_funcs,
1308};