File: | dev/pci/drm/amd/amdgpu/sdma_v4_0.c |
Warning: | line 548, column 50 Access to field 'data' results in a dereference of a null pointer (loaded from field 'fw') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* | |||
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |||
3 | * | |||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |||
5 | * copy of this software and associated documentation files (the "Software"), | |||
6 | * to deal in the Software without restriction, including without limitation | |||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |||
8 | * and/or sell copies of the Software, and to permit persons to whom the | |||
9 | * Software is furnished to do so, subject to the following conditions: | |||
10 | * | |||
11 | * The above copyright notice and this permission notice shall be included in | |||
12 | * all copies or substantial portions of the Software. | |||
13 | * | |||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |||
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |||
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |||
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |||
20 | * OTHER DEALINGS IN THE SOFTWARE. | |||
21 | * | |||
22 | */ | |||
23 | ||||
24 | #include <linux/delay.h> | |||
25 | #include <linux/firmware.h> | |||
26 | #include <linux/module.h> | |||
27 | #include <linux/pci.h> | |||
28 | ||||
29 | #include "amdgpu.h" | |||
30 | #include "amdgpu_ucode.h" | |||
31 | #include "amdgpu_trace.h" | |||
32 | ||||
33 | #include "sdma0/sdma0_4_2_offset.h" | |||
34 | #include "sdma0/sdma0_4_2_sh_mask.h" | |||
35 | #include "sdma1/sdma1_4_2_offset.h" | |||
36 | #include "sdma1/sdma1_4_2_sh_mask.h" | |||
37 | #include "sdma2/sdma2_4_2_2_offset.h" | |||
38 | #include "sdma2/sdma2_4_2_2_sh_mask.h" | |||
39 | #include "sdma3/sdma3_4_2_2_offset.h" | |||
40 | #include "sdma3/sdma3_4_2_2_sh_mask.h" | |||
41 | #include "sdma4/sdma4_4_2_2_offset.h" | |||
42 | #include "sdma4/sdma4_4_2_2_sh_mask.h" | |||
43 | #include "sdma5/sdma5_4_2_2_offset.h" | |||
44 | #include "sdma5/sdma5_4_2_2_sh_mask.h" | |||
45 | #include "sdma6/sdma6_4_2_2_offset.h" | |||
46 | #include "sdma6/sdma6_4_2_2_sh_mask.h" | |||
47 | #include "sdma7/sdma7_4_2_2_offset.h" | |||
48 | #include "sdma7/sdma7_4_2_2_sh_mask.h" | |||
49 | #include "hdp/hdp_4_0_offset.h" | |||
50 | #include "sdma0/sdma0_4_1_default.h" | |||
51 | ||||
52 | #include "soc15_common.h" | |||
53 | #include "soc15.h" | |||
54 | #include "vega10_sdma_pkt_open.h" | |||
55 | ||||
56 | #include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h" | |||
57 | #include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h" | |||
58 | ||||
59 | #include "amdgpu_ras.h" | |||
60 | ||||
61 | MODULE_FIRMWARE("amdgpu/vega10_sdma.bin"); | |||
62 | MODULE_FIRMWARE("amdgpu/vega10_sdma1.bin"); | |||
63 | MODULE_FIRMWARE("amdgpu/vega12_sdma.bin"); | |||
64 | MODULE_FIRMWARE("amdgpu/vega12_sdma1.bin"); | |||
65 | MODULE_FIRMWARE("amdgpu/vega20_sdma.bin"); | |||
66 | MODULE_FIRMWARE("amdgpu/vega20_sdma1.bin"); | |||
67 | MODULE_FIRMWARE("amdgpu/raven_sdma.bin"); | |||
68 | MODULE_FIRMWARE("amdgpu/picasso_sdma.bin"); | |||
69 | MODULE_FIRMWARE("amdgpu/raven2_sdma.bin"); | |||
70 | MODULE_FIRMWARE("amdgpu/arcturus_sdma.bin"); | |||
71 | MODULE_FIRMWARE("amdgpu/renoir_sdma.bin"); | |||
72 | MODULE_FIRMWARE("amdgpu/green_sardine_sdma.bin"); | |||
73 | ||||
74 | #define SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK0x000000F8L 0x000000F8L | |||
75 | #define SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK0xFC000000L 0xFC000000L | |||
76 | ||||
77 | #define WREG32_SDMA(instance, offset, value)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (instance ), (offset))), (value), 0) \ | |||
78 | WREG32(sdma_v4_0_get_reg_offset(adev, (instance), (offset)), value)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (instance ), (offset))), (value), 0) | |||
79 | #define RREG32_SDMA(instance, offset)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (instance ), (offset))), 0) \ | |||
80 | RREG32(sdma_v4_0_get_reg_offset(adev, (instance), (offset)))amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (instance ), (offset))), 0) | |||
81 | ||||
82 | static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev); | |||
83 | static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev); | |||
84 | static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev); | |||
85 | static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev); | |||
86 | static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev); | |||
87 | ||||
88 | static const struct soc15_reg_golden golden_settings_sdma_4[] = { | |||
89 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07){ SDMA0_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831d07 }, | |||
90 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xff000ff0, 0x3f000100){ SDMA0_HWIP, 0, 0, 0x001b, 0xff000ff0, 0x3f000100 }, | |||
91 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0100, 0x00000100){ SDMA0_HWIP, 0, 0, 0x008a, 0x800f0100, 0x00000100 }, | |||
92 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0087, 0xfffffff7, 0x00403000 }, | |||
93 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_IB_CNTL, 0x800f0100, 0x00000100){ SDMA0_HWIP, 0, 0, 0x00ea, 0x800f0100, 0x00000100 }, | |||
94 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA0_HWIP, 0, 0, 0x00e7, 0x0000fff0, 0x00403000 }, | |||
95 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003ff006, 0x0003c000){ SDMA0_HWIP, 0, 0, 0x001a, 0x003ff006, 0x0003c000 }, | |||
96 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0100, 0x00000100){ SDMA0_HWIP, 0, 0, 0x014a, 0x800f0100, 0x00000100 }, | |||
97 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0147, 0x0000fff0, 0x00403000 }, | |||
98 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0100, 0x00000100){ SDMA0_HWIP, 0, 0, 0x01aa, 0x800f0100, 0x00000100 }, | |||
99 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA0_HWIP, 0, 0, 0x01a7, 0x0000fff0, 0x00403000 }, | |||
100 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0){ SDMA0_HWIP, 0, 0, 0x0048, 0x000003ff, 0x000003c0 }, | |||
101 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x00000000){ SDMA0_HWIP, 0, 0, 0x003d, 0xfc000000, 0x00000000 }, | |||
102 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100){ SDMA1_HWIP, 0, 0, 0x001b, 0xffffffff, 0x3f000100 }, | |||
103 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_IB_CNTL, 0x800f0100, 0x00000100){ SDMA1_HWIP, 0, 0, 0x008a, 0x800f0100, 0x00000100 }, | |||
104 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0087, 0x0000fff0, 0x00403000 }, | |||
105 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_IB_CNTL, 0x800f0100, 0x00000100){ SDMA1_HWIP, 0, 0, 0x00ea, 0x800f0100, 0x00000100 }, | |||
106 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA1_HWIP, 0, 0, 0x00e7, 0x0000fff0, 0x00403000 }, | |||
107 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_POWER_CNTL, 0x003ff000, 0x0003c000){ SDMA1_HWIP, 0, 0, 0x001a, 0x003ff000, 0x0003c000 }, | |||
108 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_IB_CNTL, 0x800f0100, 0x00000100){ SDMA1_HWIP, 0, 0, 0x014a, 0x800f0100, 0x00000100 }, | |||
109 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0147, 0x0000fff0, 0x00403000 }, | |||
110 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_IB_CNTL, 0x800f0100, 0x00000100){ SDMA1_HWIP, 0, 0, 0x01aa, 0x800f0100, 0x00000100 }, | |||
111 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000){ SDMA1_HWIP, 0, 0, 0x01a7, 0x0000fff0, 0x00403000 }, | |||
112 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0){ SDMA1_HWIP, 0, 0, 0x0048, 0x000003ff, 0x000003c0 }, | |||
113 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_WATERMK, 0xfc000000, 0x00000000){ SDMA1_HWIP, 0, 0, 0x003d, 0xfc000000, 0x00000000 } | |||
114 | }; | |||
115 | ||||
116 | static const struct soc15_reg_golden golden_settings_sdma_vg10[] = { | |||
117 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104002){ SDMA0_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00104002 }, | |||
118 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002){ SDMA0_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00104002 }, | |||
119 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA0_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
120 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07){ SDMA1_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831d07 }, | |||
121 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104002){ SDMA1_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00104002 }, | |||
122 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002){ SDMA1_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00104002 }, | |||
123 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA1_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
124 | }; | |||
125 | ||||
126 | static const struct soc15_reg_golden golden_settings_sdma_vg12[] = { | |||
127 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104001){ SDMA0_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00104001 }, | |||
128 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104001){ SDMA0_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00104001 }, | |||
129 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA0_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
130 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07){ SDMA1_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831d07 }, | |||
131 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104001){ SDMA1_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00104001 }, | |||
132 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104001){ SDMA1_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00104001 }, | |||
133 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA1_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
134 | }; | |||
135 | ||||
136 | static const struct soc15_reg_golden golden_settings_sdma_4_1[] = { | |||
137 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07){ SDMA0_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831d07 }, | |||
138 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100){ SDMA0_HWIP, 0, 0, 0x001b, 0xffffffff, 0x3f000100 }, | |||
139 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100){ SDMA0_HWIP, 0, 0, 0x008a, 0x800f0111, 0x00000100 }, | |||
140 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0087, 0xfffffff7, 0x00403000 }, | |||
141 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0xfc3fffff, 0x40000051){ SDMA0_HWIP, 0, 0, 0x001a, 0xfc3fffff, 0x40000051 }, | |||
142 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100){ SDMA0_HWIP, 0, 0, 0x014a, 0x800f0111, 0x00000100 }, | |||
143 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0147, 0xfffffff7, 0x00403000 }, | |||
144 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100){ SDMA0_HWIP, 0, 0, 0x01aa, 0x800f0111, 0x00000100 }, | |||
145 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x01a7, 0xfffffff7, 0x00403000 }, | |||
146 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003e0){ SDMA0_HWIP, 0, 0, 0x0048, 0x000003ff, 0x000003e0 }, | |||
147 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x00000000){ SDMA0_HWIP, 0, 0, 0x003d, 0xfc000000, 0x00000000 } | |||
148 | }; | |||
149 | ||||
150 | static const struct soc15_reg_golden golden_settings_sdma0_4_2_init[] = { | |||
151 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0147, 0xfffffff0, 0x00403000 }, | |||
152 | }; | |||
153 | ||||
154 | static const struct soc15_reg_golden golden_settings_sdma0_4_2[] = | |||
155 | { | |||
156 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA0_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
157 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100){ SDMA0_HWIP, 0, 0, 0x001b, 0xffffffff, 0x3f000100 }, | |||
158 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA0_HWIP, 0, 0, 0x001e, 0x0000773f, 0x00004002 }, | |||
159 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA0_HWIP, 0, 0, 0x001f, 0x0000773f, 0x00004002 }, | |||
160 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x0089, 0xfffffffd, 0x00000001 }, | |||
161 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0087, 0xfffffff7, 0x00403000 }, | |||
162 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x00e9, 0xfffffffd, 0x00000001 }, | |||
163 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x00e7, 0xfffffff7, 0x00403000 }, | |||
164 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RD_BURST_CNTL, 0x0000000f, 0x00000003){ SDMA0_HWIP, 0, 0, 0x0027, 0x0000000f, 0x00000003 }, | |||
165 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x0149, 0xfffffffd, 0x00000001 }, | |||
166 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0147, 0xfffffff0, 0x00403000 }, | |||
167 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x01a9, 0xfffffffd, 0x00000001 }, | |||
168 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x01a7, 0xfffffff7, 0x00403000 }, | |||
169 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC2_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x0209, 0xfffffffd, 0x00000001 }, | |||
170 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0207, 0xfffffff7, 0x00403000 }, | |||
171 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC3_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x0269, 0xfffffffd, 0x00000001 }, | |||
172 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0267, 0xfffffff7, 0x00403000 }, | |||
173 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC4_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x02c9, 0xfffffffd, 0x00000001 }, | |||
174 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x02c7, 0xfffffff7, 0x00403000 }, | |||
175 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC5_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x0329, 0xfffffffd, 0x00000001 }, | |||
176 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0327, 0xfffffff7, 0x00403000 }, | |||
177 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC6_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x0389, 0xfffffffd, 0x00000001 }, | |||
178 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0387, 0xfffffff7, 0x00403000 }, | |||
179 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA0_HWIP, 0, 0, 0x03e9, 0xfffffffd, 0x00000001 }, | |||
180 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x03e7, 0xfffffff7, 0x00403000 }, | |||
181 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0){ SDMA0_HWIP, 0, 0, 0x0048, 0x000003ff, 0x000003c0 }, | |||
182 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA0_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
183 | }; | |||
184 | ||||
185 | static const struct soc15_reg_golden golden_settings_sdma1_4_2[] = { | |||
186 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA1_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
187 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100){ SDMA1_HWIP, 0, 0, 0x001b, 0xffffffff, 0x3f000100 }, | |||
188 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA1_HWIP, 0, 0, 0x001e, 0x0000773f, 0x00004002 }, | |||
189 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA1_HWIP, 0, 0, 0x001f, 0x0000773f, 0x00004002 }, | |||
190 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x0089, 0xfffffffd, 0x00000001 }, | |||
191 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0087, 0xfffffff7, 0x00403000 }, | |||
192 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x00e9, 0xfffffffd, 0x00000001 }, | |||
193 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x00e7, 0xfffffff7, 0x00403000 }, | |||
194 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RD_BURST_CNTL, 0x0000000f, 0x00000003){ SDMA1_HWIP, 0, 0, 0x0027, 0x0000000f, 0x00000003 }, | |||
195 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x0149, 0xfffffffd, 0x00000001 }, | |||
196 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0147, 0xfffffff0, 0x00403000 }, | |||
197 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x01a9, 0xfffffffd, 0x00000001 }, | |||
198 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x01a7, 0xfffffff7, 0x00403000 }, | |||
199 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC2_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x0209, 0xfffffffd, 0x00000001 }, | |||
200 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0207, 0xfffffff7, 0x00403000 }, | |||
201 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC3_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x0269, 0xfffffffd, 0x00000001 }, | |||
202 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0267, 0xfffffff7, 0x00403000 }, | |||
203 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC4_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x02c9, 0xfffffffd, 0x00000001 }, | |||
204 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x02c7, 0xfffffff7, 0x00403000 }, | |||
205 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC5_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x0329, 0xfffffffd, 0x00000001 }, | |||
206 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0327, 0xfffffff7, 0x00403000 }, | |||
207 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC6_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x0389, 0xfffffffd, 0x00000001 }, | |||
208 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x0387, 0xfffffff7, 0x00403000 }, | |||
209 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001){ SDMA1_HWIP, 0, 0, 0x03e9, 0xfffffffd, 0x00000001 }, | |||
210 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA1_HWIP, 0, 0, 0x03e7, 0xfffffff7, 0x00403000 }, | |||
211 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0){ SDMA1_HWIP, 0, 0, 0x0048, 0x000003ff, 0x000003c0 }, | |||
212 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA1_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
213 | }; | |||
214 | ||||
215 | static const struct soc15_reg_golden golden_settings_sdma_rv1[] = | |||
216 | { | |||
217 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002){ SDMA0_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00000002 }, | |||
218 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002){ SDMA0_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00000002 } | |||
219 | }; | |||
220 | ||||
221 | static const struct soc15_reg_golden golden_settings_sdma_rv2[] = | |||
222 | { | |||
223 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00003001){ SDMA0_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00003001 }, | |||
224 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00003001){ SDMA0_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00003001 } | |||
225 | }; | |||
226 | ||||
227 | static const struct soc15_reg_golden golden_settings_sdma_arct[] = | |||
228 | { | |||
229 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA0_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
230 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA0_HWIP, 0, 0, 0x001e, 0x0000773f, 0x00004002 }, | |||
231 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA0_HWIP, 0, 0, 0x001f, 0x0000773f, 0x00004002 }, | |||
232 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA0_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
233 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA1_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
234 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA1_HWIP, 0, 0, 0x001e, 0x0000773f, 0x00004002 }, | |||
235 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA1_HWIP, 0, 0, 0x001f, 0x0000773f, 0x00004002 }, | |||
236 | SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA1_HWIP, 0, 0, 0x0047, 0xffffffff, 0x00010001 }, | |||
237 | SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA2_HWIP, 0, 1, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
238 | SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA2_HWIP, 0, 1, 0x001e, 0x0000773f, 0x00004002 }, | |||
239 | SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA2_HWIP, 0, 1, 0x001f, 0x0000773f, 0x00004002 }, | |||
240 | SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA2_HWIP, 0, 1, 0x0047, 0xffffffff, 0x00010001 }, | |||
241 | SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA3_HWIP, 0, 1, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
242 | SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA3_HWIP, 0, 1, 0x001e, 0x0000773f, 0x00004002 }, | |||
243 | SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA3_HWIP, 0, 1, 0x001f, 0x0000773f, 0x00004002 }, | |||
244 | SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA3_HWIP, 0, 1, 0x0047, 0xffffffff, 0x00010001 }, | |||
245 | SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA4_HWIP, 0, 1, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
246 | SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA4_HWIP, 0, 1, 0x001e, 0x0000773f, 0x00004002 }, | |||
247 | SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA4_HWIP, 0, 1, 0x001f, 0x0000773f, 0x00004002 }, | |||
248 | SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA4_HWIP, 0, 1, 0x0047, 0xffffffff, 0x00010001 }, | |||
249 | SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA5_HWIP, 0, 1, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
250 | SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA5_HWIP, 0, 1, 0x001e, 0x0000773f, 0x00004002 }, | |||
251 | SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA5_HWIP, 0, 1, 0x001f, 0x0000773f, 0x00004002 }, | |||
252 | SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA5_HWIP, 0, 1, 0x0047, 0xffffffff, 0x00010001 }, | |||
253 | SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA6_HWIP, 0, 1, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
254 | SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA6_HWIP, 0, 1, 0x001e, 0x0000773f, 0x00004002 }, | |||
255 | SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA6_HWIP, 0, 1, 0x001f, 0x0000773f, 0x00004002 }, | |||
256 | SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA6_HWIP, 0, 1, 0x0047, 0xffffffff, 0x00010001 }, | |||
257 | SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA7_HWIP, 0, 1, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
258 | SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_GB_ADDR_CONFIG, 0x0000773f, 0x00004002){ SDMA7_HWIP, 0, 1, 0x001e, 0x0000773f, 0x00004002 }, | |||
259 | SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002){ SDMA7_HWIP, 0, 1, 0x001f, 0x0000773f, 0x00004002 }, | |||
260 | SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_UTCL1_TIMEOUT, 0xffffffff, 0x00010001){ SDMA7_HWIP, 0, 1, 0x0047, 0xffffffff, 0x00010001 } | |||
261 | }; | |||
262 | ||||
263 | static const struct soc15_reg_golden golden_settings_sdma_4_3[] = { | |||
264 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07){ SDMA0_HWIP, 0, 0, 0x001d, 0xfe931f07, 0x02831f07 }, | |||
265 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100){ SDMA0_HWIP, 0, 0, 0x001b, 0xffffffff, 0x3f000100 }, | |||
266 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002){ SDMA0_HWIP, 0, 0, 0x001e, 0x0018773f, 0x00000002 }, | |||
267 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002){ SDMA0_HWIP, 0, 0, 0x001f, 0x0018773f, 0x00000002 }, | |||
268 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0087, 0xfffffff7, 0x00403000 }, | |||
269 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003fff07, 0x40000051){ SDMA0_HWIP, 0, 0, 0x001a, 0x003fff07, 0x40000051 }, | |||
270 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x0147, 0xfffffff7, 0x00403000 }, | |||
271 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000){ SDMA0_HWIP, 0, 0, 0x01a7, 0xfffffff7, 0x00403000 }, | |||
272 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003e0){ SDMA0_HWIP, 0, 0, 0x0048, 0x000003ff, 0x000003e0 }, | |||
273 | SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x03fbe1fe){ SDMA0_HWIP, 0, 0, 0x003d, 0xfc000000, 0x03fbe1fe } | |||
274 | }; | |||
275 | ||||
276 | static const struct soc15_ras_field_entry sdma_v4_0_ras_fields[] = { | |||
277 | { "SDMA_UCODE_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
278 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UCODE_BUF_SED)0x00000001L, 0x0, | |||
279 | 0, 0, | |||
280 | }, | |||
281 | { "SDMA_RB_CMD_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
282 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_RB_CMD_BUF_SED)0x00000004L, 0x2, | |||
283 | 0, 0, | |||
284 | }, | |||
285 | { "SDMA_IB_CMD_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
286 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_IB_CMD_BUF_SED)0x00000008L, 0x3, | |||
287 | 0, 0, | |||
288 | }, | |||
289 | { "SDMA_UTCL1_RD_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
290 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UTCL1_RD_FIFO_SED)0x00000010L, 0x4, | |||
291 | 0, 0, | |||
292 | }, | |||
293 | { "SDMA_UTCL1_RDBST_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
294 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UTCL1_RDBST_FIFO_SED)0x00000020L, 0x5, | |||
295 | 0, 0, | |||
296 | }, | |||
297 | { "SDMA_DATA_LUT_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
298 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_DATA_LUT_FIFO_SED)0x00000040L, 0x6, | |||
299 | 0, 0, | |||
300 | }, | |||
301 | { "SDMA_MBANK_DATA_BUF0_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
302 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF0_SED)0x00000080L, 0x7, | |||
303 | 0, 0, | |||
304 | }, | |||
305 | { "SDMA_MBANK_DATA_BUF1_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
306 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF1_SED)0x00000100L, 0x8, | |||
307 | 0, 0, | |||
308 | }, | |||
309 | { "SDMA_MBANK_DATA_BUF2_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
310 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF2_SED)0x00000200L, 0x9, | |||
311 | 0, 0, | |||
312 | }, | |||
313 | { "SDMA_MBANK_DATA_BUF3_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
314 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF3_SED)0x00000400L, 0xa, | |||
315 | 0, 0, | |||
316 | }, | |||
317 | { "SDMA_MBANK_DATA_BUF4_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
318 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF4_SED)0x00000800L, 0xb, | |||
319 | 0, 0, | |||
320 | }, | |||
321 | { "SDMA_MBANK_DATA_BUF5_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
322 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF5_SED)0x00001000L, 0xc, | |||
323 | 0, 0, | |||
324 | }, | |||
325 | { "SDMA_MBANK_DATA_BUF6_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
326 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF6_SED)0x00002000L, 0xd, | |||
327 | 0, 0, | |||
328 | }, | |||
329 | { "SDMA_MBANK_DATA_BUF7_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
330 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF7_SED)0x00004000L, 0xe, | |||
331 | 0, 0, | |||
332 | }, | |||
333 | { "SDMA_MBANK_DATA_BUF8_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
334 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF8_SED)0x00008000L, 0xf, | |||
335 | 0, 0, | |||
336 | }, | |||
337 | { "SDMA_MBANK_DATA_BUF9_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
338 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF9_SED)0x00010000L, 0x10, | |||
339 | 0, 0, | |||
340 | }, | |||
341 | { "SDMA_MBANK_DATA_BUF10_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
342 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF10_SED)0x00020000L, 0x11, | |||
343 | 0, 0, | |||
344 | }, | |||
345 | { "SDMA_MBANK_DATA_BUF11_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
346 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF11_SED)0x00040000L, 0x12, | |||
347 | 0, 0, | |||
348 | }, | |||
349 | { "SDMA_MBANK_DATA_BUF12_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
350 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF12_SED)0x00080000L, 0x13, | |||
351 | 0, 0, | |||
352 | }, | |||
353 | { "SDMA_MBANK_DATA_BUF13_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
354 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF13_SED)0x00100000L, 0x14, | |||
355 | 0, 0, | |||
356 | }, | |||
357 | { "SDMA_MBANK_DATA_BUF14_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
358 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF14_SED)0x00200000L, 0x15, | |||
359 | 0, 0, | |||
360 | }, | |||
361 | { "SDMA_MBANK_DATA_BUF15_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
362 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF15_SED)0x00400000L, 0x16, | |||
363 | 0, 0, | |||
364 | }, | |||
365 | { "SDMA_SPLIT_DAT_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
366 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_SPLIT_DAT_BUF_SED)0x00800000L, 0x17, | |||
367 | 0, 0, | |||
368 | }, | |||
369 | { "SDMA_MC_WR_ADDR_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER)SDMA0_HWIP, 0, 0, 0x0036, | |||
370 | SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MC_WR_ADDR_FIFO_SED)0x01000000L, 0x18, | |||
371 | 0, 0, | |||
372 | }, | |||
373 | }; | |||
374 | ||||
375 | static u32 sdma_v4_0_get_reg_offset(struct amdgpu_device *adev, | |||
376 | u32 instance, u32 offset) | |||
377 | { | |||
378 | switch (instance) { | |||
379 | case 0: | |||
380 | return (adev->reg_offset[SDMA0_HWIP][0][0] + offset); | |||
381 | case 1: | |||
382 | return (adev->reg_offset[SDMA1_HWIP][0][0] + offset); | |||
383 | case 2: | |||
384 | return (adev->reg_offset[SDMA2_HWIP][0][1] + offset); | |||
385 | case 3: | |||
386 | return (adev->reg_offset[SDMA3_HWIP][0][1] + offset); | |||
387 | case 4: | |||
388 | return (adev->reg_offset[SDMA4_HWIP][0][1] + offset); | |||
389 | case 5: | |||
390 | return (adev->reg_offset[SDMA5_HWIP][0][1] + offset); | |||
391 | case 6: | |||
392 | return (adev->reg_offset[SDMA6_HWIP][0][1] + offset); | |||
393 | case 7: | |||
394 | return (adev->reg_offset[SDMA7_HWIP][0][1] + offset); | |||
395 | default: | |||
396 | break; | |||
397 | } | |||
398 | return 0; | |||
399 | } | |||
400 | ||||
401 | static unsigned sdma_v4_0_seq_to_irq_id(int seq_num) | |||
402 | { | |||
403 | switch (seq_num) { | |||
404 | case 0: | |||
405 | return SOC15_IH_CLIENTID_SDMA0; | |||
406 | case 1: | |||
407 | return SOC15_IH_CLIENTID_SDMA1; | |||
408 | case 2: | |||
409 | return SOC15_IH_CLIENTID_SDMA2; | |||
410 | case 3: | |||
411 | return SOC15_IH_CLIENTID_SDMA3; | |||
412 | case 4: | |||
413 | return SOC15_IH_CLIENTID_SDMA4; | |||
414 | case 5: | |||
415 | return SOC15_IH_CLIENTID_SDMA5; | |||
416 | case 6: | |||
417 | return SOC15_IH_CLIENTID_SDMA6; | |||
418 | case 7: | |||
419 | return SOC15_IH_CLIENTID_SDMA7; | |||
420 | default: | |||
421 | break; | |||
422 | } | |||
423 | return -EINVAL22; | |||
424 | } | |||
425 | ||||
426 | static int sdma_v4_0_irq_id_to_seq(unsigned client_id) | |||
427 | { | |||
428 | switch (client_id) { | |||
429 | case SOC15_IH_CLIENTID_SDMA0: | |||
430 | return 0; | |||
431 | case SOC15_IH_CLIENTID_SDMA1: | |||
432 | return 1; | |||
433 | case SOC15_IH_CLIENTID_SDMA2: | |||
434 | return 2; | |||
435 | case SOC15_IH_CLIENTID_SDMA3: | |||
436 | return 3; | |||
437 | case SOC15_IH_CLIENTID_SDMA4: | |||
438 | return 4; | |||
439 | case SOC15_IH_CLIENTID_SDMA5: | |||
440 | return 5; | |||
441 | case SOC15_IH_CLIENTID_SDMA6: | |||
442 | return 6; | |||
443 | case SOC15_IH_CLIENTID_SDMA7: | |||
444 | return 7; | |||
445 | default: | |||
446 | break; | |||
447 | } | |||
448 | return -EINVAL22; | |||
449 | } | |||
450 | ||||
451 | static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev) | |||
452 | { | |||
453 | switch (adev->asic_type) { | |||
454 | case CHIP_VEGA10: | |||
455 | soc15_program_register_sequence(adev, | |||
456 | golden_settings_sdma_4, | |||
457 | ARRAY_SIZE(golden_settings_sdma_4)(sizeof((golden_settings_sdma_4)) / sizeof((golden_settings_sdma_4 )[0]))); | |||
458 | soc15_program_register_sequence(adev, | |||
459 | golden_settings_sdma_vg10, | |||
460 | ARRAY_SIZE(golden_settings_sdma_vg10)(sizeof((golden_settings_sdma_vg10)) / sizeof((golden_settings_sdma_vg10 )[0]))); | |||
461 | break; | |||
462 | case CHIP_VEGA12: | |||
463 | soc15_program_register_sequence(adev, | |||
464 | golden_settings_sdma_4, | |||
465 | ARRAY_SIZE(golden_settings_sdma_4)(sizeof((golden_settings_sdma_4)) / sizeof((golden_settings_sdma_4 )[0]))); | |||
466 | soc15_program_register_sequence(adev, | |||
467 | golden_settings_sdma_vg12, | |||
468 | ARRAY_SIZE(golden_settings_sdma_vg12)(sizeof((golden_settings_sdma_vg12)) / sizeof((golden_settings_sdma_vg12 )[0]))); | |||
469 | break; | |||
470 | case CHIP_VEGA20: | |||
471 | soc15_program_register_sequence(adev, | |||
472 | golden_settings_sdma0_4_2_init, | |||
473 | ARRAY_SIZE(golden_settings_sdma0_4_2_init)(sizeof((golden_settings_sdma0_4_2_init)) / sizeof((golden_settings_sdma0_4_2_init )[0]))); | |||
474 | soc15_program_register_sequence(adev, | |||
475 | golden_settings_sdma0_4_2, | |||
476 | ARRAY_SIZE(golden_settings_sdma0_4_2)(sizeof((golden_settings_sdma0_4_2)) / sizeof((golden_settings_sdma0_4_2 )[0]))); | |||
477 | soc15_program_register_sequence(adev, | |||
478 | golden_settings_sdma1_4_2, | |||
479 | ARRAY_SIZE(golden_settings_sdma1_4_2)(sizeof((golden_settings_sdma1_4_2)) / sizeof((golden_settings_sdma1_4_2 )[0]))); | |||
480 | break; | |||
481 | case CHIP_ARCTURUS: | |||
482 | soc15_program_register_sequence(adev, | |||
483 | golden_settings_sdma_arct, | |||
484 | ARRAY_SIZE(golden_settings_sdma_arct)(sizeof((golden_settings_sdma_arct)) / sizeof((golden_settings_sdma_arct )[0]))); | |||
485 | break; | |||
486 | case CHIP_RAVEN: | |||
487 | soc15_program_register_sequence(adev, | |||
488 | golden_settings_sdma_4_1, | |||
489 | ARRAY_SIZE(golden_settings_sdma_4_1)(sizeof((golden_settings_sdma_4_1)) / sizeof((golden_settings_sdma_4_1 )[0]))); | |||
490 | if (adev->apu_flags & AMD_APU_IS_RAVEN2) | |||
491 | soc15_program_register_sequence(adev, | |||
492 | golden_settings_sdma_rv2, | |||
493 | ARRAY_SIZE(golden_settings_sdma_rv2)(sizeof((golden_settings_sdma_rv2)) / sizeof((golden_settings_sdma_rv2 )[0]))); | |||
494 | else | |||
495 | soc15_program_register_sequence(adev, | |||
496 | golden_settings_sdma_rv1, | |||
497 | ARRAY_SIZE(golden_settings_sdma_rv1)(sizeof((golden_settings_sdma_rv1)) / sizeof((golden_settings_sdma_rv1 )[0]))); | |||
498 | break; | |||
499 | case CHIP_RENOIR: | |||
500 | soc15_program_register_sequence(adev, | |||
501 | golden_settings_sdma_4_3, | |||
502 | ARRAY_SIZE(golden_settings_sdma_4_3)(sizeof((golden_settings_sdma_4_3)) / sizeof((golden_settings_sdma_4_3 )[0]))); | |||
503 | break; | |||
504 | default: | |||
505 | break; | |||
506 | } | |||
507 | } | |||
508 | ||||
509 | static void sdma_v4_0_setup_ulv(struct amdgpu_device *adev) | |||
510 | { | |||
511 | int i; | |||
512 | ||||
513 | /* | |||
514 | * The only chips with SDMAv4 and ULV are VG10 and VG20. | |||
515 | * Server SKUs take a different hysteresis setting from other SKUs. | |||
516 | */ | |||
517 | switch (adev->asic_type) { | |||
518 | case CHIP_VEGA10: | |||
519 | if (adev->pdev->device == 0x6860) | |||
520 | break; | |||
521 | return; | |||
522 | case CHIP_VEGA20: | |||
523 | if (adev->pdev->device == 0x66a1) | |||
524 | break; | |||
525 | return; | |||
526 | default: | |||
527 | return; | |||
528 | } | |||
529 | ||||
530 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
531 | uint32_t temp; | |||
532 | ||||
533 | temp = RREG32_SDMA(i, mmSDMA0_ULV_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x005e))), 0); | |||
534 | temp = REG_SET_FIELD(temp, SDMA0_ULV_CNTL, HYSTERESIS, 0x0)(((temp) & ~0x0000001FL) | (0x0000001FL & ((0x0) << 0x0))); | |||
535 | WREG32_SDMA(i, mmSDMA0_ULV_CNTL, temp)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x005e))), (temp), 0); | |||
536 | } | |||
537 | } | |||
538 | ||||
539 | static int sdma_v4_0_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) | |||
540 | { | |||
541 | int err = 0; | |||
542 | const struct sdma_firmware_header_v1_0 *hdr; | |||
543 | ||||
544 | err = amdgpu_ucode_validate(sdma_inst->fw); | |||
545 | if (err) | |||
546 | return err; | |||
547 | ||||
548 | hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data; | |||
| ||||
549 | sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version)((__uint32_t)(hdr->header.ucode_version)); | |||
550 | sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version)((__uint32_t)(hdr->ucode_feature_version)); | |||
551 | ||||
552 | if (sdma_inst->feature_version >= 20) | |||
553 | sdma_inst->burst_nop = true1; | |||
554 | ||||
555 | return 0; | |||
556 | } | |||
557 | ||||
558 | static void sdma_v4_0_destroy_inst_ctx(struct amdgpu_device *adev) | |||
559 | { | |||
560 | int i; | |||
561 | ||||
562 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
563 | release_firmware(adev->sdma.instance[i].fw); | |||
564 | adev->sdma.instance[i].fw = NULL((void *)0); | |||
565 | ||||
566 | /* arcturus shares the same FW memory across | |||
567 | all SDMA isntances */ | |||
568 | if (adev->asic_type == CHIP_ARCTURUS) | |||
569 | break; | |||
570 | } | |||
571 | ||||
572 | memset((void*)adev->sdma.instance, 0,__builtin_memset(((void*)adev->sdma.instance), (0), (sizeof (struct amdgpu_sdma_instance) * 8)) | |||
573 | sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES)__builtin_memset(((void*)adev->sdma.instance), (0), (sizeof (struct amdgpu_sdma_instance) * 8)); | |||
574 | } | |||
575 | ||||
576 | /** | |||
577 | * sdma_v4_0_init_microcode - load ucode images from disk | |||
578 | * | |||
579 | * @adev: amdgpu_device pointer | |||
580 | * | |||
581 | * Use the firmware interface to load the ucode images into | |||
582 | * the driver (not loaded into hw). | |||
583 | * Returns 0 on success, error on failure. | |||
584 | */ | |||
585 | ||||
586 | // emulation only, won't work on real chip | |||
587 | // vega10 real chip need to use PSP to load firmware | |||
588 | static int sdma_v4_0_init_microcode(struct amdgpu_device *adev) | |||
589 | { | |||
590 | const char *chip_name; | |||
591 | char fw_name[30]; | |||
592 | int err = 0, i; | |||
593 | struct amdgpu_firmware_info *info = NULL((void *)0); | |||
594 | const struct common_firmware_header *header = NULL((void *)0); | |||
595 | ||||
596 | if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) | |||
597 | return 0; | |||
598 | ||||
599 | DRM_DEBUG("\n")__drm_dbg(DRM_UT_CORE, "\n"); | |||
600 | ||||
601 | switch (adev->asic_type) { | |||
602 | case CHIP_VEGA10: | |||
603 | chip_name = "vega10"; | |||
604 | break; | |||
605 | case CHIP_VEGA12: | |||
606 | chip_name = "vega12"; | |||
607 | break; | |||
608 | case CHIP_VEGA20: | |||
609 | chip_name = "vega20"; | |||
610 | break; | |||
611 | case CHIP_RAVEN: | |||
612 | if (adev->apu_flags & AMD_APU_IS_RAVEN2) | |||
613 | chip_name = "raven2"; | |||
614 | else if (adev->apu_flags & AMD_APU_IS_PICASSO) | |||
615 | chip_name = "picasso"; | |||
616 | else | |||
617 | chip_name = "raven"; | |||
618 | break; | |||
619 | case CHIP_ARCTURUS: | |||
620 | chip_name = "arcturus"; | |||
621 | break; | |||
622 | case CHIP_RENOIR: | |||
623 | if (adev->apu_flags & AMD_APU_IS_RENOIR) | |||
624 | chip_name = "renoir"; | |||
625 | else | |||
626 | chip_name = "green_sardine"; | |||
627 | break; | |||
628 | default: | |||
629 | BUG()do { panic("BUG at %s:%d", "/usr/src/sys/dev/pci/drm/amd/amdgpu/sdma_v4_0.c" , 629); } while (0); | |||
630 | } | |||
631 | ||||
632 | snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name); | |||
633 | ||||
634 | err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev); | |||
635 | if (err) | |||
636 | goto out; | |||
637 | ||||
638 | err = sdma_v4_0_init_inst_ctx(&adev->sdma.instance[0]); | |||
639 | if (err) | |||
640 | goto out; | |||
641 | ||||
642 | for (i = 1; i < adev->sdma.num_instances; i++) { | |||
643 | if (adev->asic_type == CHIP_ARCTURUS) { | |||
644 | /* Acturus will leverage the same FW memory | |||
645 | for every SDMA instance */ | |||
646 | memcpy((void*)&adev->sdma.instance[i],__builtin_memcpy(((void*)&adev->sdma.instance[i]), ((void *)&adev->sdma.instance[0]), (sizeof(struct amdgpu_sdma_instance ))) | |||
647 | (void*)&adev->sdma.instance[0],__builtin_memcpy(((void*)&adev->sdma.instance[i]), ((void *)&adev->sdma.instance[0]), (sizeof(struct amdgpu_sdma_instance ))) | |||
648 | sizeof(struct amdgpu_sdma_instance))__builtin_memcpy(((void*)&adev->sdma.instance[i]), ((void *)&adev->sdma.instance[0]), (sizeof(struct amdgpu_sdma_instance ))); | |||
649 | } | |||
650 | else { | |||
651 | snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i); | |||
652 | ||||
653 | err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); | |||
654 | if (err) | |||
655 | goto out; | |||
656 | ||||
657 | err = sdma_v4_0_init_inst_ctx(&adev->sdma.instance[i]); | |||
658 | if (err) | |||
659 | goto out; | |||
660 | } | |||
661 | } | |||
662 | ||||
663 | DRM_DEBUG("psp_load == '%s'\n",__drm_dbg(DRM_UT_CORE, "psp_load == '%s'\n", adev->firmware .load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false") | |||
664 | adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false")__drm_dbg(DRM_UT_CORE, "psp_load == '%s'\n", adev->firmware .load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); | |||
665 | ||||
666 | if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { | |||
667 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
668 | info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; | |||
669 | info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; | |||
670 | info->fw = adev->sdma.instance[i].fw; | |||
671 | header = (const struct common_firmware_header *)info->fw->data; | |||
672 | adev->firmware.fw_size += | |||
673 | roundup2(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE)(((((__uint32_t)(header->ucode_size_bytes))) + (((1 << 12)) - 1)) & (~((__typeof(((__uint32_t)(header->ucode_size_bytes ))))((1 << 12)) - 1))); | |||
674 | } | |||
675 | } | |||
676 | ||||
677 | out: | |||
678 | if (err) { | |||
679 | DRM_ERROR("sdma_v4_0: Failed to load firmware \"%s\"\n", fw_name)__drm_err("sdma_v4_0: Failed to load firmware \"%s\"\n", fw_name ); | |||
680 | sdma_v4_0_destroy_inst_ctx(adev); | |||
681 | } | |||
682 | return err; | |||
683 | } | |||
684 | ||||
685 | /** | |||
686 | * sdma_v4_0_ring_get_rptr - get the current read pointer | |||
687 | * | |||
688 | * @ring: amdgpu ring pointer | |||
689 | * | |||
690 | * Get the current rptr from the hardware (VEGA10+). | |||
691 | */ | |||
692 | static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring) | |||
693 | { | |||
694 | u64 *rptr; | |||
695 | ||||
696 | /* XXX check if swapping is necessary on BE */ | |||
697 | rptr = ((u64 *)&ring->adev->wb.wb[ring->rptr_offs]); | |||
698 | ||||
699 | DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr)__drm_dbg(DRM_UT_CORE, "rptr before shift == 0x%016llx\n", *rptr ); | |||
700 | return ((*rptr) >> 2); | |||
701 | } | |||
702 | ||||
703 | /** | |||
704 | * sdma_v4_0_ring_get_wptr - get the current write pointer | |||
705 | * | |||
706 | * @ring: amdgpu ring pointer | |||
707 | * | |||
708 | * Get the current wptr from the hardware (VEGA10+). | |||
709 | */ | |||
710 | static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring) | |||
711 | { | |||
712 | struct amdgpu_device *adev = ring->adev; | |||
713 | u64 wptr; | |||
714 | ||||
715 | if (ring->use_doorbell) { | |||
716 | /* XXX check if swapping is necessary on BE */ | |||
717 | wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]))({ typeof(*((u64 *)&adev->wb.wb[ring->wptr_offs])) __tmp = *(volatile typeof(*((u64 *)&adev->wb.wb[ring->wptr_offs ])) *)&(*((u64 *)&adev->wb.wb[ring->wptr_offs]) ); membar_datadep_consumer(); __tmp; }); | |||
718 | DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr)__drm_dbg(DRM_UT_CORE, "wptr/doorbell before shift == 0x%016llx\n" , wptr); | |||
719 | } else { | |||
720 | wptr = RREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR_HI)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x0086))), 0); | |||
721 | wptr = wptr << 32; | |||
722 | wptr |= RREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x0085))), 0); | |||
723 | DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",__drm_dbg(DRM_UT_CORE, "wptr before shift [%i] wptr == 0x%016llx\n" , ring->me, wptr) | |||
724 | ring->me, wptr)__drm_dbg(DRM_UT_CORE, "wptr before shift [%i] wptr == 0x%016llx\n" , ring->me, wptr); | |||
725 | } | |||
726 | ||||
727 | return wptr >> 2; | |||
728 | } | |||
729 | ||||
730 | /** | |||
731 | * sdma_v4_0_page_ring_set_wptr - commit the write pointer | |||
732 | * | |||
733 | * @ring: amdgpu ring pointer | |||
734 | * | |||
735 | * Write the wptr back to the hardware (VEGA10+). | |||
736 | */ | |||
737 | static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring) | |||
738 | { | |||
739 | struct amdgpu_device *adev = ring->adev; | |||
740 | ||||
741 | DRM_DEBUG("Setting write pointer\n")__drm_dbg(DRM_UT_CORE, "Setting write pointer\n"); | |||
742 | if (ring->use_doorbell) { | |||
743 | u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs]; | |||
744 | ||||
745 | DRM_DEBUG("Using doorbell -- "__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))) | |||
746 | "wptr_offs == 0x%08x "__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))) | |||
747 | "lower_32_bits(ring->wptr) << 2 == 0x%08x "__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))) | |||
748 | "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))) | |||
749 | ring->wptr_offs,__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))) | |||
750 | lower_32_bits(ring->wptr << 2),__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))) | |||
751 | upper_32_bits(ring->wptr << 2))__drm_dbg(DRM_UT_CORE, "Using doorbell -- " "wptr_offs == 0x%08x " "lower_32_bits(ring->wptr) << 2 == 0x%08x " "upper_32_bits(ring->wptr) << 2 == 0x%08x\n" , ring->wptr_offs, ((u32)(ring->wptr << 2)), ((u32 )(((ring->wptr << 2) >> 16) >> 16))); | |||
752 | /* XXX check if swapping is necessary on BE */ | |||
753 | WRITE_ONCE(*wb, (ring->wptr << 2))({ typeof(*wb) __tmp = ((ring->wptr << 2)); *(volatile typeof(*wb) *)&(*wb) = __tmp; __tmp; }); | |||
754 | DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",__drm_dbg(DRM_UT_CORE, "calling WDOORBELL64(0x%08x, 0x%016llx)\n" , ring->doorbell_index, ring->wptr << 2) | |||
755 | ring->doorbell_index, ring->wptr << 2)__drm_dbg(DRM_UT_CORE, "calling WDOORBELL64(0x%08x, 0x%016llx)\n" , ring->doorbell_index, ring->wptr << 2); | |||
756 | WDOORBELL64(ring->doorbell_index, ring->wptr << 2)amdgpu_mm_wdoorbell64(adev, (ring->doorbell_index), (ring-> wptr << 2)); | |||
757 | } else { | |||
758 | DRM_DEBUG("Not using doorbell -- "__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))) | |||
759 | "mmSDMA%i_GFX_RB_WPTR == 0x%08x "__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))) | |||
760 | "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))) | |||
761 | ring->me,__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))) | |||
762 | lower_32_bits(ring->wptr << 2),__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))) | |||
763 | ring->me,__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))) | |||
764 | upper_32_bits(ring->wptr << 2))__drm_dbg(DRM_UT_CORE, "Not using doorbell -- " "mmSDMA%i_GFX_RB_WPTR == 0x%08x " "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", ring->me, ((u32)(ring ->wptr << 2)), ring->me, ((u32)(((ring->wptr << 2) >> 16) >> 16))); | |||
765 | WREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x0085))), (((u32)(ring->wptr << 2))), 0) | |||
766 | lower_32_bits(ring->wptr << 2))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x0085))), (((u32)(ring->wptr << 2))), 0); | |||
767 | WREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR_HI,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x0086))), (((u32)(((ring->wptr << 2) >> 16) >> 16))), 0) | |||
768 | upper_32_bits(ring->wptr << 2))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x0086))), (((u32)(((ring->wptr << 2) >> 16) >> 16))), 0); | |||
769 | } | |||
770 | } | |||
771 | ||||
772 | /** | |||
773 | * sdma_v4_0_page_ring_get_wptr - get the current write pointer | |||
774 | * | |||
775 | * @ring: amdgpu ring pointer | |||
776 | * | |||
777 | * Get the current wptr from the hardware (VEGA10+). | |||
778 | */ | |||
779 | static uint64_t sdma_v4_0_page_ring_get_wptr(struct amdgpu_ring *ring) | |||
780 | { | |||
781 | struct amdgpu_device *adev = ring->adev; | |||
782 | u64 wptr; | |||
783 | ||||
784 | if (ring->use_doorbell) { | |||
785 | /* XXX check if swapping is necessary on BE */ | |||
786 | wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]))({ typeof(*((u64 *)&adev->wb.wb[ring->wptr_offs])) __tmp = *(volatile typeof(*((u64 *)&adev->wb.wb[ring->wptr_offs ])) *)&(*((u64 *)&adev->wb.wb[ring->wptr_offs]) ); membar_datadep_consumer(); __tmp; }); | |||
787 | } else { | |||
788 | wptr = RREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR_HI)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x00e6))), 0); | |||
789 | wptr = wptr << 32; | |||
790 | wptr |= RREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x00e5))), 0); | |||
791 | } | |||
792 | ||||
793 | return wptr >> 2; | |||
794 | } | |||
795 | ||||
796 | /** | |||
797 | * sdma_v4_0_ring_set_wptr - commit the write pointer | |||
798 | * | |||
799 | * @ring: amdgpu ring pointer | |||
800 | * | |||
801 | * Write the wptr back to the hardware (VEGA10+). | |||
802 | */ | |||
803 | static void sdma_v4_0_page_ring_set_wptr(struct amdgpu_ring *ring) | |||
804 | { | |||
805 | struct amdgpu_device *adev = ring->adev; | |||
806 | ||||
807 | if (ring->use_doorbell) { | |||
808 | u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs]; | |||
809 | ||||
810 | /* XXX check if swapping is necessary on BE */ | |||
811 | WRITE_ONCE(*wb, (ring->wptr << 2))({ typeof(*wb) __tmp = ((ring->wptr << 2)); *(volatile typeof(*wb) *)&(*wb) = __tmp; __tmp; }); | |||
812 | WDOORBELL64(ring->doorbell_index, ring->wptr << 2)amdgpu_mm_wdoorbell64(adev, (ring->doorbell_index), (ring-> wptr << 2)); | |||
813 | } else { | |||
814 | uint64_t wptr = ring->wptr << 2; | |||
815 | ||||
816 | WREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x00e5))), (((u32)(wptr))), 0) | |||
817 | lower_32_bits(wptr))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x00e5))), (((u32)(wptr))), 0); | |||
818 | WREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR_HI,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x00e6))), (((u32)(((wptr) >> 16) >> 16 ))), 0) | |||
819 | upper_32_bits(wptr))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (ring ->me), (0x00e6))), (((u32)(((wptr) >> 16) >> 16 ))), 0); | |||
820 | } | |||
821 | } | |||
822 | ||||
823 | static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) | |||
824 | { | |||
825 | struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); | |||
826 | int i; | |||
827 | ||||
828 | for (i = 0; i < count; i++) | |||
829 | if (sdma && sdma->burst_nop && (i == 0)) | |||
830 | amdgpu_ring_write(ring, ring->funcs->nop | | |||
831 | SDMA_PKT_NOP_HEADER_COUNT(count - 1)(((count - 1) & 0x00003FFF) << 16)); | |||
832 | else | |||
833 | amdgpu_ring_write(ring, ring->funcs->nop); | |||
834 | } | |||
835 | ||||
836 | /** | |||
837 | * sdma_v4_0_ring_emit_ib - Schedule an IB on the DMA engine | |||
838 | * | |||
839 | * @ring: amdgpu ring pointer | |||
840 | * @ib: IB object to schedule | |||
841 | * | |||
842 | * Schedule an IB in the DMA ring (VEGA10). | |||
843 | */ | |||
844 | static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring, | |||
845 | struct amdgpu_job *job, | |||
846 | struct amdgpu_ib *ib, | |||
847 | uint32_t flags) | |||
848 | { | |||
849 | unsigned vmid = AMDGPU_JOB_GET_VMID(job)((job) ? (job)->vmid : 0); | |||
850 | ||||
851 | /* IB packet must end on a 8 DW boundary */ | |||
852 | sdma_v4_0_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)((u32)(ring->wptr))) & 7); | |||
853 | ||||
854 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT)(((4) & 0x000000FF) << 0) | | |||
855 | SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf)(((vmid & 0xf) & 0x0000000F) << 16)); | |||
856 | /* base must be 32 byte aligned */ | |||
857 | amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)((u32)(ib->gpu_addr)) & 0xffffffe0); | |||
858 | amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)((u32)(((ib->gpu_addr) >> 16) >> 16))); | |||
859 | amdgpu_ring_write(ring, ib->length_dw); | |||
860 | amdgpu_ring_write(ring, 0); | |||
861 | amdgpu_ring_write(ring, 0); | |||
862 | ||||
863 | } | |||
864 | ||||
865 | static void sdma_v4_0_wait_reg_mem(struct amdgpu_ring *ring, | |||
866 | int mem_space, int hdp, | |||
867 | uint32_t addr0, uint32_t addr1, | |||
868 | uint32_t ref, uint32_t mask, | |||
869 | uint32_t inv) | |||
870 | { | |||
871 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM)(((8) & 0x000000FF) << 0) | | |||
872 | SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp)(((hdp) & 0x00000001) << 26) | | |||
873 | SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space)(((mem_space) & 0x00000001) << 31) | | |||
874 | SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)(((3) & 0x00000007) << 28)); /* == */ | |||
875 | if (mem_space) { | |||
876 | /* memory */ | |||
877 | amdgpu_ring_write(ring, addr0); | |||
878 | amdgpu_ring_write(ring, addr1); | |||
879 | } else { | |||
880 | /* registers */ | |||
881 | amdgpu_ring_write(ring, addr0 << 2); | |||
882 | amdgpu_ring_write(ring, addr1 << 2); | |||
883 | } | |||
884 | amdgpu_ring_write(ring, ref); /* reference */ | |||
885 | amdgpu_ring_write(ring, mask); /* mask */ | |||
886 | amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff)(((0xfff) & 0x00000FFF) << 16) | | |||
887 | SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)(((inv) & 0x0000FFFF) << 0)); /* retry count, poll interval */ | |||
888 | } | |||
889 | ||||
890 | /** | |||
891 | * sdma_v4_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring | |||
892 | * | |||
893 | * @ring: amdgpu ring pointer | |||
894 | * | |||
895 | * Emit an hdp flush packet on the requested DMA ring. | |||
896 | */ | |||
897 | static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) | |||
898 | { | |||
899 | struct amdgpu_device *adev = ring->adev; | |||
900 | u32 ref_and_mask = 0; | |||
901 | const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; | |||
902 | ||||
903 | ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me; | |||
904 | ||||
905 | sdma_v4_0_wait_reg_mem(ring, 0, 1, | |||
906 | adev->nbio.funcs->get_hdp_flush_done_offset(adev), | |||
907 | adev->nbio.funcs->get_hdp_flush_req_offset(adev), | |||
908 | ref_and_mask, ref_and_mask, 10); | |||
909 | } | |||
910 | ||||
911 | /** | |||
912 | * sdma_v4_0_ring_emit_fence - emit a fence on the DMA ring | |||
913 | * | |||
914 | * @ring: amdgpu ring pointer | |||
915 | * @fence: amdgpu fence object | |||
916 | * | |||
917 | * Add a DMA fence packet to the ring to write | |||
918 | * the fence seq number and DMA trap packet to generate | |||
919 | * an interrupt if needed (VEGA10). | |||
920 | */ | |||
921 | static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, | |||
922 | unsigned flags) | |||
923 | { | |||
924 | bool_Bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT(1 << 0); | |||
925 | /* write the fence */ | |||
926 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)(((5) & 0x000000FF) << 0)); | |||
927 | /* zero in first two bits */ | |||
928 | BUG_ON(addr & 0x3)((!(addr & 0x3)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/sdma_v4_0.c" , 928, "!(addr & 0x3)")); | |||
929 | amdgpu_ring_write(ring, lower_32_bits(addr)((u32)(addr))); | |||
930 | amdgpu_ring_write(ring, upper_32_bits(addr)((u32)(((addr) >> 16) >> 16))); | |||
931 | amdgpu_ring_write(ring, lower_32_bits(seq)((u32)(seq))); | |||
932 | ||||
933 | /* optionally write high bits as well */ | |||
934 | if (write64bit) { | |||
935 | addr += 4; | |||
936 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)(((5) & 0x000000FF) << 0)); | |||
937 | /* zero in first two bits */ | |||
938 | BUG_ON(addr & 0x3)((!(addr & 0x3)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/sdma_v4_0.c" , 938, "!(addr & 0x3)")); | |||
939 | amdgpu_ring_write(ring, lower_32_bits(addr)((u32)(addr))); | |||
940 | amdgpu_ring_write(ring, upper_32_bits(addr)((u32)(((addr) >> 16) >> 16))); | |||
941 | amdgpu_ring_write(ring, upper_32_bits(seq)((u32)(((seq) >> 16) >> 16))); | |||
942 | } | |||
943 | ||||
944 | /* generate an interrupt */ | |||
945 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP)(((6) & 0x000000FF) << 0)); | |||
946 | amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0)(((0) & 0x0FFFFFFF) << 0)); | |||
947 | } | |||
948 | ||||
949 | ||||
950 | /** | |||
951 | * sdma_v4_0_gfx_stop - stop the gfx async dma engines | |||
952 | * | |||
953 | * @adev: amdgpu_device pointer | |||
954 | * | |||
955 | * Stop the gfx async dma ring buffers (VEGA10). | |||
956 | */ | |||
957 | static void sdma_v4_0_gfx_stop(struct amdgpu_device *adev) | |||
958 | { | |||
959 | struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES8]; | |||
960 | u32 rb_cntl, ib_cntl; | |||
961 | int i, unset = 0; | |||
962 | ||||
963 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
964 | sdma[i] = &adev->sdma.instance[i].ring; | |||
965 | ||||
966 | if ((adev->mman.buffer_funcs_ring == sdma[i]) && unset != 1) { | |||
967 | amdgpu_ttm_set_buffer_funcs_status(adev, false0); | |||
968 | unset = 1; | |||
969 | } | |||
970 | ||||
971 | rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0080))), 0); | |||
972 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0)(((rb_cntl) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))); | |||
973 | WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0080))), (rb_cntl), 0); | |||
974 | ib_cntl = RREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x008a))), 0); | |||
975 | ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0)(((ib_cntl) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))); | |||
976 | WREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL, ib_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x008a))), (ib_cntl), 0); | |||
977 | } | |||
978 | } | |||
979 | ||||
980 | /** | |||
981 | * sdma_v4_0_rlc_stop - stop the compute async dma engines | |||
982 | * | |||
983 | * @adev: amdgpu_device pointer | |||
984 | * | |||
985 | * Stop the compute async dma queues (VEGA10). | |||
986 | */ | |||
987 | static void sdma_v4_0_rlc_stop(struct amdgpu_device *adev) | |||
988 | { | |||
989 | /* XXX todo */ | |||
990 | } | |||
991 | ||||
992 | /** | |||
993 | * sdma_v4_0_page_stop - stop the page async dma engines | |||
994 | * | |||
995 | * @adev: amdgpu_device pointer | |||
996 | * | |||
997 | * Stop the page async dma ring buffers (VEGA10). | |||
998 | */ | |||
999 | static void sdma_v4_0_page_stop(struct amdgpu_device *adev) | |||
1000 | { | |||
1001 | struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES8]; | |||
1002 | u32 rb_cntl, ib_cntl; | |||
1003 | int i; | |||
1004 | bool_Bool unset = false0; | |||
1005 | ||||
1006 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1007 | sdma[i] = &adev->sdma.instance[i].page; | |||
1008 | ||||
1009 | if ((adev->mman.buffer_funcs_ring == sdma[i]) && | |||
1010 | (!unset)) { | |||
1011 | amdgpu_ttm_set_buffer_funcs_status(adev, false0); | |||
1012 | unset = true1; | |||
1013 | } | |||
1014 | ||||
1015 | rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e0))), 0); | |||
1016 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,(((rb_cntl) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))) | |||
1017 | RB_ENABLE, 0)(((rb_cntl) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))); | |||
1018 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e0))), (rb_cntl), 0); | |||
1019 | ib_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00ea))), 0); | |||
1020 | ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL,(((ib_cntl) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))) | |||
1021 | IB_ENABLE, 0)(((ib_cntl) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))); | |||
1022 | WREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL, ib_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00ea))), (ib_cntl), 0); | |||
1023 | } | |||
1024 | } | |||
1025 | ||||
1026 | /** | |||
1027 | * sdma_v4_0_ctx_switch_enable - stop the async dma engines context switch | |||
1028 | * | |||
1029 | * @adev: amdgpu_device pointer | |||
1030 | * @enable: enable/disable the DMA MEs context switch. | |||
1031 | * | |||
1032 | * Halt or unhalt the async dma engines context switch (VEGA10). | |||
1033 | */ | |||
1034 | static void sdma_v4_0_ctx_switch_enable(struct amdgpu_device *adev, bool_Bool enable) | |||
1035 | { | |||
1036 | u32 f32_cntl, phase_quantum = 0; | |||
1037 | int i; | |||
1038 | ||||
1039 | if (amdgpu_sdma_phase_quantum) { | |||
1040 | unsigned value = amdgpu_sdma_phase_quantum; | |||
1041 | unsigned unit = 0; | |||
1042 | ||||
1043 | while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK0x00FFFF00L >> | |||
1044 | SDMA0_PHASE0_QUANTUM__VALUE__SHIFT0x8)) { | |||
1045 | value = (value + 1) >> 1; | |||
1046 | unit++; | |||
1047 | } | |||
1048 | if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK0x0000000FL >> | |||
1049 | SDMA0_PHASE0_QUANTUM__UNIT__SHIFT0x0)) { | |||
1050 | value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK0x00FFFF00L >> | |||
1051 | SDMA0_PHASE0_QUANTUM__VALUE__SHIFT0x8); | |||
1052 | unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK0x0000000FL >> | |||
1053 | SDMA0_PHASE0_QUANTUM__UNIT__SHIFT0x0); | |||
1054 | WARN_ONCE(1,({ static int __warned; int __ret = !!(1); if (__ret && !__warned) { printf("clamping sdma_phase_quantum to %uK clock cycles\n" , value << unit); __warned = 1; } __builtin_expect(!!(__ret ), 0); }) | |||
1055 | "clamping sdma_phase_quantum to %uK clock cycles\n",({ static int __warned; int __ret = !!(1); if (__ret && !__warned) { printf("clamping sdma_phase_quantum to %uK clock cycles\n" , value << unit); __warned = 1; } __builtin_expect(!!(__ret ), 0); }) | |||
1056 | value << unit)({ static int __warned; int __ret = !!(1); if (__ret && !__warned) { printf("clamping sdma_phase_quantum to %uK clock cycles\n" , value << unit); __warned = 1; } __builtin_expect(!!(__ret ), 0); }); | |||
1057 | } | |||
1058 | phase_quantum = | |||
1059 | value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT0x8 | | |||
1060 | unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT0x0; | |||
1061 | } | |||
1062 | ||||
1063 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1064 | f32_cntl = RREG32_SDMA(i, mmSDMA0_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001c))), 0); | |||
1065 | f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,(((f32_cntl) & ~0x00040000L) | (0x00040000L & ((enable ? 1 : 0) << 0x12))) | |||
1066 | AUTO_CTXSW_ENABLE, enable ? 1 : 0)(((f32_cntl) & ~0x00040000L) | (0x00040000L & ((enable ? 1 : 0) << 0x12))); | |||
1067 | if (enable && amdgpu_sdma_phase_quantum) { | |||
1068 | WREG32_SDMA(i, mmSDMA0_PHASE0_QUANTUM, phase_quantum)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x002c))), (phase_quantum), 0); | |||
1069 | WREG32_SDMA(i, mmSDMA0_PHASE1_QUANTUM, phase_quantum)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x002d))), (phase_quantum), 0); | |||
1070 | WREG32_SDMA(i, mmSDMA0_PHASE2_QUANTUM, phase_quantum)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x004f))), (phase_quantum), 0); | |||
1071 | } | |||
1072 | WREG32_SDMA(i, mmSDMA0_CNTL, f32_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001c))), (f32_cntl), 0); | |||
1073 | ||||
1074 | /* | |||
1075 | * Enable SDMA utilization. Its only supported on | |||
1076 | * Arcturus for the moment and firmware version 14 | |||
1077 | * and above. | |||
1078 | */ | |||
1079 | if (adev->asic_type == CHIP_ARCTURUS && | |||
1080 | adev->sdma.instance[i].fw_version >= 14) | |||
1081 | WREG32_SDMA(i, mmSDMA0_PUB_DUMMY_REG2, enable)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0053))), (enable), 0); | |||
1082 | } | |||
1083 | ||||
1084 | } | |||
1085 | ||||
1086 | /** | |||
1087 | * sdma_v4_0_enable - stop the async dma engines | |||
1088 | * | |||
1089 | * @adev: amdgpu_device pointer | |||
1090 | * @enable: enable/disable the DMA MEs. | |||
1091 | * | |||
1092 | * Halt or unhalt the async dma engines (VEGA10). | |||
1093 | */ | |||
1094 | static void sdma_v4_0_enable(struct amdgpu_device *adev, bool_Bool enable) | |||
1095 | { | |||
1096 | u32 f32_cntl; | |||
1097 | int i; | |||
1098 | ||||
1099 | if (!enable) { | |||
1100 | sdma_v4_0_gfx_stop(adev); | |||
1101 | sdma_v4_0_rlc_stop(adev); | |||
1102 | if (adev->sdma.has_page_queue) | |||
1103 | sdma_v4_0_page_stop(adev); | |||
1104 | } | |||
1105 | ||||
1106 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1107 | f32_cntl = RREG32_SDMA(i, mmSDMA0_F32_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x002a))), 0); | |||
1108 | f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1)(((f32_cntl) & ~0x00000001L) | (0x00000001L & ((enable ? 0 : 1) << 0x0))); | |||
1109 | WREG32_SDMA(i, mmSDMA0_F32_CNTL, f32_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x002a))), (f32_cntl), 0); | |||
1110 | } | |||
1111 | } | |||
1112 | ||||
1113 | /** | |||
1114 | * sdma_v4_0_rb_cntl - get parameters for rb_cntl | |||
1115 | */ | |||
1116 | static uint32_t sdma_v4_0_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl) | |||
1117 | { | |||
1118 | /* Set ring buffer size in dwords */ | |||
1119 | uint32_t rb_bufsz = order_base_2(ring->ring_size / 4)drm_order(ring->ring_size / 4); | |||
1120 | ||||
1121 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz)(((rb_cntl) & ~0x0000003EL) | (0x0000003EL & ((rb_bufsz ) << 0x1))); | |||
1122 | #ifdef __BIG_ENDIAN | |||
1123 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1)(((rb_cntl) & ~0x00000200L) | (0x00000200L & ((1) << 0x9))); | |||
1124 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,(((rb_cntl) & ~0x00002000L) | (0x00002000L & ((1) << 0xd))) | |||
1125 | RPTR_WRITEBACK_SWAP_ENABLE, 1)(((rb_cntl) & ~0x00002000L) | (0x00002000L & ((1) << 0xd))); | |||
1126 | #endif | |||
1127 | return rb_cntl; | |||
1128 | } | |||
1129 | ||||
1130 | /** | |||
1131 | * sdma_v4_0_gfx_resume - setup and start the async dma engines | |||
1132 | * | |||
1133 | * @adev: amdgpu_device pointer | |||
1134 | * @i: instance to resume | |||
1135 | * | |||
1136 | * Set up the gfx DMA ring buffers and enable them (VEGA10). | |||
1137 | * Returns 0 for success, error for failure. | |||
1138 | */ | |||
1139 | static void sdma_v4_0_gfx_resume(struct amdgpu_device *adev, unsigned int i) | |||
1140 | { | |||
1141 | struct amdgpu_ring *ring = &adev->sdma.instance[i].ring; | |||
1142 | u32 rb_cntl, ib_cntl, wptr_poll_cntl; | |||
1143 | u32 wb_offset; | |||
1144 | u32 doorbell; | |||
1145 | u32 doorbell_offset; | |||
1146 | u64 wptr_gpu_addr; | |||
1147 | ||||
1148 | wb_offset = (ring->rptr_offs * 4); | |||
1149 | ||||
1150 | rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0080))), 0); | |||
1151 | rb_cntl = sdma_v4_0_rb_cntl(ring, rb_cntl); | |||
1152 | WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0080))), (rb_cntl), 0); | |||
1153 | ||||
1154 | /* Initialize the ring buffer's read and write pointers */ | |||
1155 | WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0083))), (0), 0); | |||
1156 | WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_HI, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0084))), (0), 0); | |||
1157 | WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0085))), (0), 0); | |||
1158 | WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_HI, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0086))), (0), 0); | |||
1159 | ||||
1160 | /* set the wb address whether it's enabled or not */ | |||
1161 | WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_ADDR_HI,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0088))), (((u32)(((adev->wb.gpu_addr + wb_offset) >> 16) >> 16)) & 0xFFFFFFFF), 0) | |||
1162 | upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0088))), (((u32)(((adev->wb.gpu_addr + wb_offset) >> 16) >> 16)) & 0xFFFFFFFF), 0); | |||
1163 | WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_ADDR_LO,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0089))), (((u32)(adev->wb.gpu_addr + wb_offset)) & 0xFFFFFFFC), 0) | |||
1164 | lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0089))), (((u32)(adev->wb.gpu_addr + wb_offset)) & 0xFFFFFFFC), 0); | |||
1165 | ||||
1166 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,(((rb_cntl) & ~0x00001000L) | (0x00001000L & ((1) << 0xc))) | |||
1167 | RPTR_WRITEBACK_ENABLE, 1)(((rb_cntl) & ~0x00001000L) | (0x00001000L & ((1) << 0xc))); | |||
1168 | ||||
1169 | WREG32_SDMA(i, mmSDMA0_GFX_RB_BASE, ring->gpu_addr >> 8)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0081))), (ring->gpu_addr >> 8), 0); | |||
1170 | WREG32_SDMA(i, mmSDMA0_GFX_RB_BASE_HI, ring->gpu_addr >> 40)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0082))), (ring->gpu_addr >> 40), 0); | |||
1171 | ||||
1172 | ring->wptr = 0; | |||
1173 | ||||
1174 | /* before programing wptr to a less value, need set minor_ptr_update first */ | |||
1175 | WREG32_SDMA(i, mmSDMA0_GFX_MINOR_PTR_UPDATE, 1)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00b5))), (1), 0); | |||
1176 | ||||
1177 | doorbell = RREG32_SDMA(i, mmSDMA0_GFX_DOORBELL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0092))), 0); | |||
1178 | doorbell_offset = RREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00ab))), 0); | |||
1179 | ||||
1180 | doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE,(((doorbell) & ~0x10000000L) | (0x10000000L & ((ring-> use_doorbell) << 0x1c))) | |||
1181 | ring->use_doorbell)(((doorbell) & ~0x10000000L) | (0x10000000L & ((ring-> use_doorbell) << 0x1c))); | |||
1182 | doorbell_offset = REG_SET_FIELD(doorbell_offset,(((doorbell_offset) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ( (ring->doorbell_index) << 0x2))) | |||
1183 | SDMA0_GFX_DOORBELL_OFFSET,(((doorbell_offset) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ( (ring->doorbell_index) << 0x2))) | |||
1184 | OFFSET, ring->doorbell_index)(((doorbell_offset) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ( (ring->doorbell_index) << 0x2))); | |||
1185 | WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL, doorbell)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0092))), (doorbell), 0); | |||
1186 | WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET, doorbell_offset)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00ab))), (doorbell_offset), 0); | |||
1187 | ||||
1188 | sdma_v4_0_ring_set_wptr(ring); | |||
1189 | ||||
1190 | /* set minor_ptr_update to 0 after wptr programed */ | |||
1191 | WREG32_SDMA(i, mmSDMA0_GFX_MINOR_PTR_UPDATE, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00b5))), (0), 0); | |||
1192 | ||||
1193 | /* setup the wptr shadow polling */ | |||
1194 | wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); | |||
1195 | WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00b3))), (((u32)(wptr_gpu_addr))), 0) | |||
1196 | lower_32_bits(wptr_gpu_addr))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00b3))), (((u32)(wptr_gpu_addr))), 0); | |||
1197 | WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00b2))), (((u32)(((wptr_gpu_addr) >> 16) >> 16 ))), 0) | |||
1198 | upper_32_bits(wptr_gpu_addr))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00b2))), (((u32)(((wptr_gpu_addr) >> 16) >> 16 ))), 0); | |||
1199 | wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0087))), 0); | |||
1200 | wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,(((wptr_poll_cntl) & ~0x00000004L) | (0x00000004L & ( (((adev)->virt.caps & (1 << 2))? 1 : 0) << 0x2))) | |||
1201 | SDMA0_GFX_RB_WPTR_POLL_CNTL,(((wptr_poll_cntl) & ~0x00000004L) | (0x00000004L & ( (((adev)->virt.caps & (1 << 2))? 1 : 0) << 0x2))) | |||
1202 | F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0)(((wptr_poll_cntl) & ~0x00000004L) | (0x00000004L & ( (((adev)->virt.caps & (1 << 2))? 1 : 0) << 0x2))); | |||
1203 | WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0087))), (wptr_poll_cntl), 0); | |||
1204 | ||||
1205 | /* enable DMA RB */ | |||
1206 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1)(((rb_cntl) & ~0x00000001L) | (0x00000001L & ((1) << 0x0))); | |||
1207 | WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0080))), (rb_cntl), 0); | |||
1208 | ||||
1209 | ib_cntl = RREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x008a))), 0); | |||
1210 | ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1)(((ib_cntl) & ~0x00000001L) | (0x00000001L & ((1) << 0x0))); | |||
1211 | #ifdef __BIG_ENDIAN | |||
1212 | ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1)(((ib_cntl) & ~0x00000010L) | (0x00000010L & ((1) << 0x4))); | |||
1213 | #endif | |||
1214 | /* enable DMA IBs */ | |||
1215 | WREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL, ib_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x008a))), (ib_cntl), 0); | |||
1216 | ||||
1217 | ring->sched.ready = true1; | |||
1218 | } | |||
1219 | ||||
1220 | /** | |||
1221 | * sdma_v4_0_page_resume - setup and start the async dma engines | |||
1222 | * | |||
1223 | * @adev: amdgpu_device pointer | |||
1224 | * @i: instance to resume | |||
1225 | * | |||
1226 | * Set up the page DMA ring buffers and enable them (VEGA10). | |||
1227 | * Returns 0 for success, error for failure. | |||
1228 | */ | |||
1229 | static void sdma_v4_0_page_resume(struct amdgpu_device *adev, unsigned int i) | |||
1230 | { | |||
1231 | struct amdgpu_ring *ring = &adev->sdma.instance[i].page; | |||
1232 | u32 rb_cntl, ib_cntl, wptr_poll_cntl; | |||
1233 | u32 wb_offset; | |||
1234 | u32 doorbell; | |||
1235 | u32 doorbell_offset; | |||
1236 | u64 wptr_gpu_addr; | |||
1237 | ||||
1238 | wb_offset = (ring->rptr_offs * 4); | |||
1239 | ||||
1240 | rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e0))), 0); | |||
1241 | rb_cntl = sdma_v4_0_rb_cntl(ring, rb_cntl); | |||
1242 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e0))), (rb_cntl), 0); | |||
1243 | ||||
1244 | /* Initialize the ring buffer's read and write pointers */ | |||
1245 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e3))), (0), 0); | |||
1246 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_HI, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e4))), (0), 0); | |||
1247 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e5))), (0), 0); | |||
1248 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_HI, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e6))), (0), 0); | |||
1249 | ||||
1250 | /* set the wb address whether it's enabled or not */ | |||
1251 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_ADDR_HI,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e8))), (((u32)(((adev->wb.gpu_addr + wb_offset) >> 16) >> 16)) & 0xFFFFFFFF), 0) | |||
1252 | upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e8))), (((u32)(((adev->wb.gpu_addr + wb_offset) >> 16) >> 16)) & 0xFFFFFFFF), 0); | |||
1253 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_ADDR_LO,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e9))), (((u32)(adev->wb.gpu_addr + wb_offset)) & 0xFFFFFFFC), 0) | |||
1254 | lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e9))), (((u32)(adev->wb.gpu_addr + wb_offset)) & 0xFFFFFFFC), 0); | |||
1255 | ||||
1256 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,(((rb_cntl) & ~0x00001000L) | (0x00001000L & ((1) << 0xc))) | |||
1257 | RPTR_WRITEBACK_ENABLE, 1)(((rb_cntl) & ~0x00001000L) | (0x00001000L & ((1) << 0xc))); | |||
1258 | ||||
1259 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_BASE, ring->gpu_addr >> 8)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e1))), (ring->gpu_addr >> 8), 0); | |||
1260 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_BASE_HI, ring->gpu_addr >> 40)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e2))), (ring->gpu_addr >> 40), 0); | |||
1261 | ||||
1262 | ring->wptr = 0; | |||
1263 | ||||
1264 | /* before programing wptr to a less value, need set minor_ptr_update first */ | |||
1265 | WREG32_SDMA(i, mmSDMA0_PAGE_MINOR_PTR_UPDATE, 1)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0115))), (1), 0); | |||
1266 | ||||
1267 | doorbell = RREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00f2))), 0); | |||
1268 | doorbell_offset = RREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL_OFFSET)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x010b))), 0); | |||
1269 | ||||
1270 | doorbell = REG_SET_FIELD(doorbell, SDMA0_PAGE_DOORBELL, ENABLE,(((doorbell) & ~0x10000000L) | (0x10000000L & ((ring-> use_doorbell) << 0x1c))) | |||
1271 | ring->use_doorbell)(((doorbell) & ~0x10000000L) | (0x10000000L & ((ring-> use_doorbell) << 0x1c))); | |||
1272 | doorbell_offset = REG_SET_FIELD(doorbell_offset,(((doorbell_offset) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ( (ring->doorbell_index) << 0x2))) | |||
1273 | SDMA0_PAGE_DOORBELL_OFFSET,(((doorbell_offset) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ( (ring->doorbell_index) << 0x2))) | |||
1274 | OFFSET, ring->doorbell_index)(((doorbell_offset) & ~0x0FFFFFFCL) | (0x0FFFFFFCL & ( (ring->doorbell_index) << 0x2))); | |||
1275 | WREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL, doorbell)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00f2))), (doorbell), 0); | |||
1276 | WREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL_OFFSET, doorbell_offset)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x010b))), (doorbell_offset), 0); | |||
1277 | ||||
1278 | /* paging queue doorbell range is setup at sdma_v4_0_gfx_resume */ | |||
1279 | sdma_v4_0_page_ring_set_wptr(ring); | |||
1280 | ||||
1281 | /* set minor_ptr_update to 0 after wptr programed */ | |||
1282 | WREG32_SDMA(i, mmSDMA0_PAGE_MINOR_PTR_UPDATE, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0115))), (0), 0); | |||
1283 | ||||
1284 | /* setup the wptr shadow polling */ | |||
1285 | wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); | |||
1286 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0113))), (((u32)(wptr_gpu_addr))), 0) | |||
1287 | lower_32_bits(wptr_gpu_addr))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0113))), (((u32)(wptr_gpu_addr))), 0); | |||
1288 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0112))), (((u32)(((wptr_gpu_addr) >> 16) >> 16 ))), 0) | |||
1289 | upper_32_bits(wptr_gpu_addr))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0112))), (((u32)(((wptr_gpu_addr) >> 16) >> 16 ))), 0); | |||
1290 | wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e7))), 0); | |||
1291 | wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,(((wptr_poll_cntl) & ~0x00000004L) | (0x00000004L & ( (((adev)->virt.caps & (1 << 2))? 1 : 0) << 0x2))) | |||
1292 | SDMA0_PAGE_RB_WPTR_POLL_CNTL,(((wptr_poll_cntl) & ~0x00000004L) | (0x00000004L & ( (((adev)->virt.caps & (1 << 2))? 1 : 0) << 0x2))) | |||
1293 | F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0)(((wptr_poll_cntl) & ~0x00000004L) | (0x00000004L & ( (((adev)->virt.caps & (1 << 2))? 1 : 0) << 0x2))); | |||
1294 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e7))), (wptr_poll_cntl), 0); | |||
1295 | ||||
1296 | /* enable DMA RB */ | |||
1297 | rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL, RB_ENABLE, 1)(((rb_cntl) & ~0x00000001L) | (0x00000001L & ((1) << 0x0))); | |||
1298 | WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00e0))), (rb_cntl), 0); | |||
1299 | ||||
1300 | ib_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00ea))), 0); | |||
1301 | ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL, IB_ENABLE, 1)(((ib_cntl) & ~0x00000001L) | (0x00000001L & ((1) << 0x0))); | |||
1302 | #ifdef __BIG_ENDIAN | |||
1303 | ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1)(((ib_cntl) & ~0x00000010L) | (0x00000010L & ((1) << 0x4))); | |||
1304 | #endif | |||
1305 | /* enable DMA IBs */ | |||
1306 | WREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL, ib_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x00ea))), (ib_cntl), 0); | |||
1307 | ||||
1308 | ring->sched.ready = true1; | |||
1309 | } | |||
1310 | ||||
1311 | static void | |||
1312 | sdma_v4_1_update_power_gating(struct amdgpu_device *adev, bool_Bool enable) | |||
1313 | { | |||
1314 | uint32_t def, data; | |||
1315 | ||||
1316 | if (enable && (adev->pg_flags & AMD_PG_SUPPORT_SDMA(1 << 8))) { | |||
1317 | /* enable idle interrupt */ | |||
1318 | def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001c)), 0); | |||
1319 | data |= SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK0x10000000L; | |||
1320 | ||||
1321 | if (data != def) | |||
1322 | WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data)amdgpu_device_wreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001c)), (data), 0); | |||
1323 | } else { | |||
1324 | /* disable idle interrupt */ | |||
1325 | def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001c)), 0); | |||
1326 | data &= ~SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK0x10000000L; | |||
1327 | if (data != def) | |||
1328 | WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data)amdgpu_device_wreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001c)), (data), 0); | |||
1329 | } | |||
1330 | } | |||
1331 | ||||
1332 | static void sdma_v4_1_init_power_gating(struct amdgpu_device *adev) | |||
1333 | { | |||
1334 | uint32_t def, data; | |||
1335 | ||||
1336 | /* Enable HW based PG. */ | |||
1337 | def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001a)), 0); | |||
1338 | data |= SDMA0_POWER_CNTL__PG_CNTL_ENABLE_MASK0x00000001L; | |||
1339 | if (data != def) | |||
1340 | WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data)amdgpu_device_wreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001a)), (data), 0); | |||
1341 | ||||
1342 | /* enable interrupt */ | |||
1343 | def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001c)), 0); | |||
1344 | data |= SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK0x10000000L; | |||
1345 | if (data != def) | |||
1346 | WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data)amdgpu_device_wreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001c)), (data), 0); | |||
1347 | ||||
1348 | /* Configure hold time to filter in-valid power on/off request. Use default right now */ | |||
1349 | def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001a)), 0); | |||
1350 | data &= ~SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK0x000000F8L; | |||
1351 | data |= (mmSDMA0_POWER_CNTL_DEFAULT0x4003c050 & SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK0x000000F8L); | |||
1352 | /* Configure switch time for hysteresis purpose. Use default right now */ | |||
1353 | data &= ~SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK0xFC000000L; | |||
1354 | data |= (mmSDMA0_POWER_CNTL_DEFAULT0x4003c050 & SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK0xFC000000L); | |||
1355 | if(data != def) | |||
1356 | WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data)amdgpu_device_wreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001a)), (data), 0); | |||
1357 | } | |||
1358 | ||||
1359 | static void sdma_v4_0_init_pg(struct amdgpu_device *adev) | |||
1360 | { | |||
1361 | if (!(adev->pg_flags & AMD_PG_SUPPORT_SDMA(1 << 8))) | |||
1362 | return; | |||
1363 | ||||
1364 | switch (adev->asic_type) { | |||
1365 | case CHIP_RAVEN: | |||
1366 | case CHIP_RENOIR: | |||
1367 | sdma_v4_1_init_power_gating(adev); | |||
1368 | sdma_v4_1_update_power_gating(adev, true1); | |||
1369 | break; | |||
1370 | default: | |||
1371 | break; | |||
1372 | } | |||
1373 | } | |||
1374 | ||||
1375 | /** | |||
1376 | * sdma_v4_0_rlc_resume - setup and start the async dma engines | |||
1377 | * | |||
1378 | * @adev: amdgpu_device pointer | |||
1379 | * | |||
1380 | * Set up the compute DMA queues and enable them (VEGA10). | |||
1381 | * Returns 0 for success, error for failure. | |||
1382 | */ | |||
1383 | static int sdma_v4_0_rlc_resume(struct amdgpu_device *adev) | |||
1384 | { | |||
1385 | sdma_v4_0_init_pg(adev); | |||
1386 | ||||
1387 | return 0; | |||
1388 | } | |||
1389 | ||||
1390 | /** | |||
1391 | * sdma_v4_0_load_microcode - load the sDMA ME ucode | |||
1392 | * | |||
1393 | * @adev: amdgpu_device pointer | |||
1394 | * | |||
1395 | * Loads the sDMA0/1 ucode. | |||
1396 | * Returns 0 for success, -EINVAL if the ucode is not available. | |||
1397 | */ | |||
1398 | static int sdma_v4_0_load_microcode(struct amdgpu_device *adev) | |||
1399 | { | |||
1400 | const struct sdma_firmware_header_v1_0 *hdr; | |||
1401 | const __le32 *fw_data; | |||
1402 | u32 fw_size; | |||
1403 | int i, j; | |||
1404 | ||||
1405 | /* halt the MEs */ | |||
1406 | sdma_v4_0_enable(adev, false0); | |||
1407 | ||||
1408 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1409 | if (!adev->sdma.instance[i].fw) | |||
1410 | return -EINVAL22; | |||
1411 | ||||
1412 | hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; | |||
1413 | amdgpu_ucode_print_sdma_hdr(&hdr->header); | |||
1414 | fw_size = le32_to_cpu(hdr->header.ucode_size_bytes)((__uint32_t)(hdr->header.ucode_size_bytes)) / 4; | |||
1415 | ||||
1416 | fw_data = (const __le32 *) | |||
1417 | (adev->sdma.instance[i].fw->data + | |||
1418 | le32_to_cpu(hdr->header.ucode_array_offset_bytes)((__uint32_t)(hdr->header.ucode_array_offset_bytes))); | |||
1419 | ||||
1420 | WREG32_SDMA(i, mmSDMA0_UCODE_ADDR, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0000))), (0), 0); | |||
1421 | ||||
1422 | for (j = 0; j < fw_size; j++) | |||
1423 | WREG32_SDMA(i, mmSDMA0_UCODE_DATA,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0001))), (((__uint32_t)(*(__uint32_t *)(fw_data++)))), 0) | |||
1424 | le32_to_cpup(fw_data++))amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0001))), (((__uint32_t)(*(__uint32_t *)(fw_data++)))), 0); | |||
1425 | ||||
1426 | WREG32_SDMA(i, mmSDMA0_UCODE_ADDR,amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0000))), (adev->sdma.instance[i].fw_version), 0) | |||
1427 | adev->sdma.instance[i].fw_version)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0000))), (adev->sdma.instance[i].fw_version), 0); | |||
1428 | } | |||
1429 | ||||
1430 | return 0; | |||
1431 | } | |||
1432 | ||||
1433 | /** | |||
1434 | * sdma_v4_0_start - setup and start the async dma engines | |||
1435 | * | |||
1436 | * @adev: amdgpu_device pointer | |||
1437 | * | |||
1438 | * Set up the DMA engines and enable them (VEGA10). | |||
1439 | * Returns 0 for success, error for failure. | |||
1440 | */ | |||
1441 | static int sdma_v4_0_start(struct amdgpu_device *adev) | |||
1442 | { | |||
1443 | struct amdgpu_ring *ring; | |||
1444 | int i, r = 0; | |||
1445 | ||||
1446 | if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) { | |||
1447 | sdma_v4_0_ctx_switch_enable(adev, false0); | |||
1448 | sdma_v4_0_enable(adev, false0); | |||
1449 | } else { | |||
1450 | ||||
1451 | if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { | |||
1452 | r = sdma_v4_0_load_microcode(adev); | |||
1453 | if (r) | |||
1454 | return r; | |||
1455 | } | |||
1456 | ||||
1457 | /* unhalt the MEs */ | |||
1458 | sdma_v4_0_enable(adev, true1); | |||
1459 | /* enable sdma ring preemption */ | |||
1460 | sdma_v4_0_ctx_switch_enable(adev, true1); | |||
1461 | } | |||
1462 | ||||
1463 | /* start the gfx rings and rlc compute queues */ | |||
1464 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1465 | uint32_t temp; | |||
1466 | ||||
1467 | WREG32_SDMA(i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL, 0)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0021))), (0), 0); | |||
1468 | sdma_v4_0_gfx_resume(adev, i); | |||
1469 | if (adev->sdma.has_page_queue) | |||
1470 | sdma_v4_0_page_resume(adev, i); | |||
1471 | ||||
1472 | /* set utc l1 enable flag always to 1 */ | |||
1473 | temp = RREG32_SDMA(i, mmSDMA0_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001c))), 0); | |||
1474 | temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1)(((temp) & ~0x00000002L) | (0x00000002L & ((1) << 0x1))); | |||
1475 | WREG32_SDMA(i, mmSDMA0_CNTL, temp)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001c))), (temp), 0); | |||
1476 | ||||
1477 | if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) { | |||
1478 | /* unhalt engine */ | |||
1479 | temp = RREG32_SDMA(i, mmSDMA0_F32_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x002a))), 0); | |||
1480 | temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0)(((temp) & ~0x00000001L) | (0x00000001L & ((0) << 0x0))); | |||
1481 | WREG32_SDMA(i, mmSDMA0_F32_CNTL, temp)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x002a))), (temp), 0); | |||
1482 | } | |||
1483 | } | |||
1484 | ||||
1485 | if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) { | |||
1486 | sdma_v4_0_ctx_switch_enable(adev, true1); | |||
1487 | sdma_v4_0_enable(adev, true1); | |||
1488 | } else { | |||
1489 | r = sdma_v4_0_rlc_resume(adev); | |||
1490 | if (r) | |||
1491 | return r; | |||
1492 | } | |||
1493 | ||||
1494 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1495 | ring = &adev->sdma.instance[i].ring; | |||
1496 | ||||
1497 | r = amdgpu_ring_test_helper(ring); | |||
1498 | if (r) | |||
1499 | return r; | |||
1500 | ||||
1501 | if (adev->sdma.has_page_queue) { | |||
1502 | struct amdgpu_ring *page = &adev->sdma.instance[i].page; | |||
1503 | ||||
1504 | r = amdgpu_ring_test_helper(page); | |||
1505 | if (r) | |||
1506 | return r; | |||
1507 | ||||
1508 | if (adev->mman.buffer_funcs_ring == page) | |||
1509 | amdgpu_ttm_set_buffer_funcs_status(adev, true1); | |||
1510 | } | |||
1511 | ||||
1512 | if (adev->mman.buffer_funcs_ring == ring) | |||
1513 | amdgpu_ttm_set_buffer_funcs_status(adev, true1); | |||
1514 | } | |||
1515 | ||||
1516 | return r; | |||
1517 | } | |||
1518 | ||||
1519 | /** | |||
1520 | * sdma_v4_0_ring_test_ring - simple async dma engine test | |||
1521 | * | |||
1522 | * @ring: amdgpu_ring structure holding ring information | |||
1523 | * | |||
1524 | * Test the DMA engine by writing using it to write an | |||
1525 | * value to memory. (VEGA10). | |||
1526 | * Returns 0 for success, error for failure. | |||
1527 | */ | |||
1528 | static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring) | |||
1529 | { | |||
1530 | struct amdgpu_device *adev = ring->adev; | |||
1531 | unsigned i; | |||
1532 | unsigned index; | |||
1533 | int r; | |||
1534 | u32 tmp; | |||
1535 | u64 gpu_addr; | |||
1536 | ||||
1537 | r = amdgpu_device_wb_get(adev, &index); | |||
1538 | if (r) | |||
1539 | return r; | |||
1540 | ||||
1541 | gpu_addr = adev->wb.gpu_addr + (index * 4); | |||
1542 | tmp = 0xCAFEDEAD; | |||
1543 | adev->wb.wb[index] = cpu_to_le32(tmp)((__uint32_t)(tmp)); | |||
1544 | ||||
1545 | r = amdgpu_ring_alloc(ring, 5); | |||
1546 | if (r) | |||
1547 | goto error_free_wb; | |||
1548 | ||||
1549 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE)(((2) & 0x000000FF) << 0) | | |||
1550 | SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)(((0) & 0x000000FF) << 8)); | |||
1551 | amdgpu_ring_write(ring, lower_32_bits(gpu_addr)((u32)(gpu_addr))); | |||
1552 | amdgpu_ring_write(ring, upper_32_bits(gpu_addr)((u32)(((gpu_addr) >> 16) >> 16))); | |||
1553 | amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0)(((0) & 0x000FFFFF) << 0)); | |||
1554 | amdgpu_ring_write(ring, 0xDEADBEEF); | |||
1555 | amdgpu_ring_commit(ring); | |||
1556 | ||||
1557 | for (i = 0; i < adev->usec_timeout; i++) { | |||
1558 | tmp = le32_to_cpu(adev->wb.wb[index])((__uint32_t)(adev->wb.wb[index])); | |||
1559 | if (tmp == 0xDEADBEEF) | |||
1560 | break; | |||
1561 | udelay(1); | |||
1562 | } | |||
1563 | ||||
1564 | if (i >= adev->usec_timeout) | |||
1565 | r = -ETIMEDOUT60; | |||
1566 | ||||
1567 | error_free_wb: | |||
1568 | amdgpu_device_wb_free(adev, index); | |||
1569 | return r; | |||
1570 | } | |||
1571 | ||||
1572 | /** | |||
1573 | * sdma_v4_0_ring_test_ib - test an IB on the DMA engine | |||
1574 | * | |||
1575 | * @ring: amdgpu_ring structure holding ring information | |||
1576 | * | |||
1577 | * Test a simple IB in the DMA ring (VEGA10). | |||
1578 | * Returns 0 on success, error on failure. | |||
1579 | */ | |||
1580 | static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) | |||
1581 | { | |||
1582 | struct amdgpu_device *adev = ring->adev; | |||
1583 | struct amdgpu_ib ib; | |||
1584 | struct dma_fence *f = NULL((void *)0); | |||
1585 | unsigned index; | |||
1586 | long r; | |||
1587 | u32 tmp = 0; | |||
1588 | u64 gpu_addr; | |||
1589 | ||||
1590 | r = amdgpu_device_wb_get(adev, &index); | |||
1591 | if (r) | |||
1592 | return r; | |||
1593 | ||||
1594 | gpu_addr = adev->wb.gpu_addr + (index * 4); | |||
1595 | tmp = 0xCAFEDEAD; | |||
1596 | adev->wb.wb[index] = cpu_to_le32(tmp)((__uint32_t)(tmp)); | |||
1597 | memset(&ib, 0, sizeof(ib))__builtin_memset((&ib), (0), (sizeof(ib))); | |||
1598 | r = amdgpu_ib_get(adev, NULL((void *)0), 256, | |||
1599 | AMDGPU_IB_POOL_DIRECT, &ib); | |||
1600 | if (r) | |||
1601 | goto err0; | |||
1602 | ||||
1603 | ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE)(((2) & 0x000000FF) << 0) | | |||
1604 | SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)(((0) & 0x000000FF) << 8); | |||
1605 | ib.ptr[1] = lower_32_bits(gpu_addr)((u32)(gpu_addr)); | |||
1606 | ib.ptr[2] = upper_32_bits(gpu_addr)((u32)(((gpu_addr) >> 16) >> 16)); | |||
1607 | ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0)(((0) & 0x000FFFFF) << 0); | |||
1608 | ib.ptr[4] = 0xDEADBEEF; | |||
1609 | ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0); | |||
1610 | ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0); | |||
1611 | ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0); | |||
1612 | ib.length_dw = 8; | |||
1613 | ||||
1614 | r = amdgpu_ib_schedule(ring, 1, &ib, NULL((void *)0), &f); | |||
1615 | if (r) | |||
1616 | goto err1; | |||
1617 | ||||
1618 | r = dma_fence_wait_timeout(f, false0, timeout); | |||
1619 | if (r == 0) { | |||
1620 | r = -ETIMEDOUT60; | |||
1621 | goto err1; | |||
1622 | } else if (r < 0) { | |||
1623 | goto err1; | |||
1624 | } | |||
1625 | tmp = le32_to_cpu(adev->wb.wb[index])((__uint32_t)(adev->wb.wb[index])); | |||
1626 | if (tmp == 0xDEADBEEF) | |||
1627 | r = 0; | |||
1628 | else | |||
1629 | r = -EINVAL22; | |||
1630 | ||||
1631 | err1: | |||
1632 | amdgpu_ib_free(adev, &ib, NULL((void *)0)); | |||
1633 | dma_fence_put(f); | |||
1634 | err0: | |||
1635 | amdgpu_device_wb_free(adev, index); | |||
1636 | return r; | |||
1637 | } | |||
1638 | ||||
1639 | ||||
1640 | /** | |||
1641 | * sdma_v4_0_vm_copy_pte - update PTEs by copying them from the GART | |||
1642 | * | |||
1643 | * @ib: indirect buffer to fill with commands | |||
1644 | * @pe: addr of the page entry | |||
1645 | * @src: src addr to copy from | |||
1646 | * @count: number of page entries to update | |||
1647 | * | |||
1648 | * Update PTEs by copying them from the GART using sDMA (VEGA10). | |||
1649 | */ | |||
1650 | static void sdma_v4_0_vm_copy_pte(struct amdgpu_ib *ib, | |||
1651 | uint64_t pe, uint64_t src, | |||
1652 | unsigned count) | |||
1653 | { | |||
1654 | unsigned bytes = count * 8; | |||
1655 | ||||
1656 | ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY)(((1) & 0x000000FF) << 0) | | |||
1657 | SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR)(((0) & 0x000000FF) << 8); | |||
1658 | ib->ptr[ib->length_dw++] = bytes - 1; | |||
1659 | ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ | |||
1660 | ib->ptr[ib->length_dw++] = lower_32_bits(src)((u32)(src)); | |||
1661 | ib->ptr[ib->length_dw++] = upper_32_bits(src)((u32)(((src) >> 16) >> 16)); | |||
1662 | ib->ptr[ib->length_dw++] = lower_32_bits(pe)((u32)(pe)); | |||
1663 | ib->ptr[ib->length_dw++] = upper_32_bits(pe)((u32)(((pe) >> 16) >> 16)); | |||
1664 | ||||
1665 | } | |||
1666 | ||||
1667 | /** | |||
1668 | * sdma_v4_0_vm_write_pte - update PTEs by writing them manually | |||
1669 | * | |||
1670 | * @ib: indirect buffer to fill with commands | |||
1671 | * @pe: addr of the page entry | |||
1672 | * @addr: dst addr to write into pe | |||
1673 | * @count: number of page entries to update | |||
1674 | * @incr: increase next addr by incr bytes | |||
1675 | * @flags: access flags | |||
1676 | * | |||
1677 | * Update PTEs by writing them manually using sDMA (VEGA10). | |||
1678 | */ | |||
1679 | static void sdma_v4_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe, | |||
1680 | uint64_t value, unsigned count, | |||
1681 | uint32_t incr) | |||
1682 | { | |||
1683 | unsigned ndw = count * 2; | |||
1684 | ||||
1685 | ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE)(((2) & 0x000000FF) << 0) | | |||
1686 | SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)(((0) & 0x000000FF) << 8); | |||
1687 | ib->ptr[ib->length_dw++] = lower_32_bits(pe)((u32)(pe)); | |||
1688 | ib->ptr[ib->length_dw++] = upper_32_bits(pe)((u32)(((pe) >> 16) >> 16)); | |||
1689 | ib->ptr[ib->length_dw++] = ndw - 1; | |||
1690 | for (; ndw > 0; ndw -= 2) { | |||
1691 | ib->ptr[ib->length_dw++] = lower_32_bits(value)((u32)(value)); | |||
1692 | ib->ptr[ib->length_dw++] = upper_32_bits(value)((u32)(((value) >> 16) >> 16)); | |||
1693 | value += incr; | |||
1694 | } | |||
1695 | } | |||
1696 | ||||
1697 | /** | |||
1698 | * sdma_v4_0_vm_set_pte_pde - update the page tables using sDMA | |||
1699 | * | |||
1700 | * @ib: indirect buffer to fill with commands | |||
1701 | * @pe: addr of the page entry | |||
1702 | * @addr: dst addr to write into pe | |||
1703 | * @count: number of page entries to update | |||
1704 | * @incr: increase next addr by incr bytes | |||
1705 | * @flags: access flags | |||
1706 | * | |||
1707 | * Update the page tables using sDMA (VEGA10). | |||
1708 | */ | |||
1709 | static void sdma_v4_0_vm_set_pte_pde(struct amdgpu_ib *ib, | |||
1710 | uint64_t pe, | |||
1711 | uint64_t addr, unsigned count, | |||
1712 | uint32_t incr, uint64_t flags) | |||
1713 | { | |||
1714 | /* for physically contiguous pages (vram) */ | |||
1715 | ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE)(((12) & 0x000000FF) << 0); | |||
1716 | ib->ptr[ib->length_dw++] = lower_32_bits(pe)((u32)(pe)); /* dst addr */ | |||
1717 | ib->ptr[ib->length_dw++] = upper_32_bits(pe)((u32)(((pe) >> 16) >> 16)); | |||
1718 | ib->ptr[ib->length_dw++] = lower_32_bits(flags)((u32)(flags)); /* mask */ | |||
1719 | ib->ptr[ib->length_dw++] = upper_32_bits(flags)((u32)(((flags) >> 16) >> 16)); | |||
1720 | ib->ptr[ib->length_dw++] = lower_32_bits(addr)((u32)(addr)); /* value */ | |||
1721 | ib->ptr[ib->length_dw++] = upper_32_bits(addr)((u32)(((addr) >> 16) >> 16)); | |||
1722 | ib->ptr[ib->length_dw++] = incr; /* increment size */ | |||
1723 | ib->ptr[ib->length_dw++] = 0; | |||
1724 | ib->ptr[ib->length_dw++] = count - 1; /* number of entries */ | |||
1725 | } | |||
1726 | ||||
1727 | /** | |||
1728 | * sdma_v4_0_ring_pad_ib - pad the IB to the required number of dw | |||
1729 | * | |||
1730 | * @ib: indirect buffer to fill with padding | |||
1731 | * | |||
1732 | */ | |||
1733 | static void sdma_v4_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) | |||
1734 | { | |||
1735 | struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring); | |||
1736 | u32 pad_count; | |||
1737 | int i; | |||
1738 | ||||
1739 | pad_count = (-ib->length_dw) & 7; | |||
1740 | for (i = 0; i < pad_count; i++) | |||
1741 | if (sdma && sdma->burst_nop && (i == 0)) | |||
1742 | ib->ptr[ib->length_dw++] = | |||
1743 | SDMA_PKT_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0) | | |||
1744 | SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1)(((pad_count - 1) & 0x00003FFF) << 16); | |||
1745 | else | |||
1746 | ib->ptr[ib->length_dw++] = | |||
1747 | SDMA_PKT_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0); | |||
1748 | } | |||
1749 | ||||
1750 | ||||
1751 | /** | |||
1752 | * sdma_v4_0_ring_emit_pipeline_sync - sync the pipeline | |||
1753 | * | |||
1754 | * @ring: amdgpu_ring pointer | |||
1755 | * | |||
1756 | * Make sure all previous operations are completed (CIK). | |||
1757 | */ | |||
1758 | static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) | |||
1759 | { | |||
1760 | uint32_t seq = ring->fence_drv.sync_seq; | |||
1761 | uint64_t addr = ring->fence_drv.gpu_addr; | |||
1762 | ||||
1763 | /* wait for idle */ | |||
1764 | sdma_v4_0_wait_reg_mem(ring, 1, 0, | |||
1765 | addr & 0xfffffffc, | |||
1766 | upper_32_bits(addr)((u32)(((addr) >> 16) >> 16)) & 0xffffffff, | |||
1767 | seq, 0xffffffff, 4); | |||
1768 | } | |||
1769 | ||||
1770 | ||||
1771 | /** | |||
1772 | * sdma_v4_0_ring_emit_vm_flush - vm flush using sDMA | |||
1773 | * | |||
1774 | * @ring: amdgpu_ring pointer | |||
1775 | * @vm: amdgpu_vm pointer | |||
1776 | * | |||
1777 | * Update the page table base and flush the VM TLB | |||
1778 | * using sDMA (VEGA10). | |||
1779 | */ | |||
1780 | static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring, | |||
1781 | unsigned vmid, uint64_t pd_addr) | |||
1782 | { | |||
1783 | amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr)(ring)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((ring ), (vmid), (pd_addr)); | |||
1784 | } | |||
1785 | ||||
1786 | static void sdma_v4_0_ring_emit_wreg(struct amdgpu_ring *ring, | |||
1787 | uint32_t reg, uint32_t val) | |||
1788 | { | |||
1789 | amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE)(((14) & 0x000000FF) << 0) | | |||
1790 | SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)(((0xf) & 0x0000000F) << 28)); | |||
1791 | amdgpu_ring_write(ring, reg); | |||
1792 | amdgpu_ring_write(ring, val); | |||
1793 | } | |||
1794 | ||||
1795 | static void sdma_v4_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, | |||
1796 | uint32_t val, uint32_t mask) | |||
1797 | { | |||
1798 | sdma_v4_0_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10); | |||
1799 | } | |||
1800 | ||||
1801 | static bool_Bool sdma_v4_0_fw_support_paging_queue(struct amdgpu_device *adev) | |||
1802 | { | |||
1803 | uint fw_version = adev->sdma.instance[0].fw_version; | |||
1804 | ||||
1805 | switch (adev->asic_type) { | |||
1806 | case CHIP_VEGA10: | |||
1807 | return fw_version >= 430; | |||
1808 | case CHIP_VEGA12: | |||
1809 | /*return fw_version >= 31;*/ | |||
1810 | return false0; | |||
1811 | case CHIP_VEGA20: | |||
1812 | return fw_version >= 123; | |||
1813 | default: | |||
1814 | return false0; | |||
1815 | } | |||
1816 | } | |||
1817 | ||||
1818 | static int sdma_v4_0_early_init(void *handle) | |||
1819 | { | |||
1820 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1821 | int r; | |||
1822 | ||||
1823 | if (adev->flags & AMD_IS_APU) | |||
| ||||
1824 | adev->sdma.num_instances = 1; | |||
1825 | else if (adev->asic_type == CHIP_ARCTURUS) | |||
1826 | adev->sdma.num_instances = 8; | |||
1827 | else | |||
1828 | adev->sdma.num_instances = 2; | |||
1829 | ||||
1830 | r = sdma_v4_0_init_microcode(adev); | |||
1831 | if (r) { | |||
1832 | DRM_ERROR("Failed to load sdma firmware!\n")__drm_err("Failed to load sdma firmware!\n"); | |||
1833 | return r; | |||
1834 | } | |||
1835 | ||||
1836 | /* TODO: Page queue breaks driver reload under SRIOV */ | |||
1837 | if ((adev->asic_type == CHIP_VEGA10) && amdgpu_sriov_vf((adev))(((adev))->virt.caps & (1 << 2))) | |||
1838 | adev->sdma.has_page_queue = false0; | |||
1839 | else if (sdma_v4_0_fw_support_paging_queue(adev)) | |||
1840 | adev->sdma.has_page_queue = true1; | |||
1841 | ||||
1842 | sdma_v4_0_set_ring_funcs(adev); | |||
1843 | sdma_v4_0_set_buffer_funcs(adev); | |||
1844 | sdma_v4_0_set_vm_pte_funcs(adev); | |||
1845 | sdma_v4_0_set_irq_funcs(adev); | |||
1846 | sdma_v4_0_set_ras_funcs(adev); | |||
1847 | ||||
1848 | return 0; | |||
1849 | } | |||
1850 | ||||
1851 | static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev, | |||
1852 | void *err_data, | |||
1853 | struct amdgpu_iv_entry *entry); | |||
1854 | ||||
1855 | static int sdma_v4_0_late_init(void *handle) | |||
1856 | { | |||
1857 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1858 | struct ras_ih_if ih_info = { | |||
1859 | .cb = sdma_v4_0_process_ras_data_cb, | |||
1860 | }; | |||
1861 | ||||
1862 | sdma_v4_0_setup_ulv(adev); | |||
1863 | ||||
1864 | if (adev->sdma.funcs && adev->sdma.funcs->reset_ras_error_count) | |||
1865 | adev->sdma.funcs->reset_ras_error_count(adev); | |||
1866 | ||||
1867 | if (adev->sdma.funcs && adev->sdma.funcs->ras_late_init) | |||
1868 | return adev->sdma.funcs->ras_late_init(adev, &ih_info); | |||
1869 | else | |||
1870 | return 0; | |||
1871 | } | |||
1872 | ||||
1873 | static int sdma_v4_0_sw_init(void *handle) | |||
1874 | { | |||
1875 | struct amdgpu_ring *ring; | |||
1876 | int r, i; | |||
1877 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1878 | ||||
1879 | /* SDMA trap event */ | |||
1880 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1881 | r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i), | |||
1882 | SDMA0_4_0__SRCID__SDMA_TRAP224, | |||
1883 | &adev->sdma.trap_irq); | |||
1884 | if (r) | |||
1885 | return r; | |||
1886 | } | |||
1887 | ||||
1888 | /* SDMA SRAM ECC event */ | |||
1889 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1890 | r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i), | |||
1891 | SDMA0_4_0__SRCID__SDMA_SRAM_ECC228, | |||
1892 | &adev->sdma.ecc_irq); | |||
1893 | if (r) | |||
1894 | return r; | |||
1895 | } | |||
1896 | ||||
1897 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1898 | ring = &adev->sdma.instance[i].ring; | |||
1899 | ring->ring_obj = NULL((void *)0); | |||
1900 | ring->use_doorbell = true1; | |||
1901 | ||||
1902 | DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,__drm_dbg(DRM_UT_CORE, "SDMA %d use_doorbell being set to: [%s]\n" , i, ring->use_doorbell?"true":"false") | |||
1903 | ring->use_doorbell?"true":"false")__drm_dbg(DRM_UT_CORE, "SDMA %d use_doorbell being set to: [%s]\n" , i, ring->use_doorbell?"true":"false"); | |||
1904 | ||||
1905 | /* doorbell size is 2 dwords, get DWORD offset */ | |||
1906 | ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1; | |||
1907 | ||||
1908 | snprintf(ring->name, sizeof(ring->name), "sdma%d", i); | |||
1909 | r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq, | |||
1910 | AMDGPU_SDMA_IRQ_INSTANCE0 + i, | |||
1911 | AMDGPU_RING_PRIO_DEFAULT1); | |||
1912 | if (r) | |||
1913 | return r; | |||
1914 | ||||
1915 | if (adev->sdma.has_page_queue) { | |||
1916 | ring = &adev->sdma.instance[i].page; | |||
1917 | ring->ring_obj = NULL((void *)0); | |||
1918 | ring->use_doorbell = true1; | |||
1919 | ||||
1920 | /* paging queue use same doorbell index/routing as gfx queue | |||
1921 | * with 0x400 (4096 dwords) offset on second doorbell page | |||
1922 | */ | |||
1923 | ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1; | |||
1924 | ring->doorbell_index += 0x400; | |||
1925 | ||||
1926 | snprintf(ring->name, sizeof(ring->name), "page%d", i); | |||
1927 | r = amdgpu_ring_init(adev, ring, 1024, | |||
1928 | &adev->sdma.trap_irq, | |||
1929 | AMDGPU_SDMA_IRQ_INSTANCE0 + i, | |||
1930 | AMDGPU_RING_PRIO_DEFAULT1); | |||
1931 | if (r) | |||
1932 | return r; | |||
1933 | } | |||
1934 | } | |||
1935 | ||||
1936 | return r; | |||
1937 | } | |||
1938 | ||||
1939 | static int sdma_v4_0_sw_fini(void *handle) | |||
1940 | { | |||
1941 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1942 | int i; | |||
1943 | ||||
1944 | if (adev->sdma.funcs && adev->sdma.funcs->ras_fini) | |||
1945 | adev->sdma.funcs->ras_fini(adev); | |||
1946 | ||||
1947 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1948 | amdgpu_ring_fini(&adev->sdma.instance[i].ring); | |||
1949 | if (adev->sdma.has_page_queue) | |||
1950 | amdgpu_ring_fini(&adev->sdma.instance[i].page); | |||
1951 | } | |||
1952 | ||||
1953 | sdma_v4_0_destroy_inst_ctx(adev); | |||
1954 | ||||
1955 | return 0; | |||
1956 | } | |||
1957 | ||||
1958 | static int sdma_v4_0_hw_init(void *handle) | |||
1959 | { | |||
1960 | int r; | |||
1961 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1962 | ||||
1963 | if (adev->flags & AMD_IS_APU) | |||
1964 | amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, false0); | |||
1965 | ||||
1966 | if (!amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) | |||
1967 | sdma_v4_0_init_golden_registers(adev); | |||
1968 | ||||
1969 | r = sdma_v4_0_start(adev); | |||
1970 | ||||
1971 | return r; | |||
1972 | } | |||
1973 | ||||
1974 | static int sdma_v4_0_hw_fini(void *handle) | |||
1975 | { | |||
1976 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1977 | int i; | |||
1978 | ||||
1979 | if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) | |||
1980 | return 0; | |||
1981 | ||||
1982 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
1983 | amdgpu_irq_put(adev, &adev->sdma.ecc_irq, | |||
1984 | AMDGPU_SDMA_IRQ_INSTANCE0 + i); | |||
1985 | } | |||
1986 | ||||
1987 | sdma_v4_0_ctx_switch_enable(adev, false0); | |||
1988 | sdma_v4_0_enable(adev, false0); | |||
1989 | ||||
1990 | if (adev->flags & AMD_IS_APU) | |||
1991 | amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, true1); | |||
1992 | ||||
1993 | return 0; | |||
1994 | } | |||
1995 | ||||
1996 | static int sdma_v4_0_suspend(void *handle) | |||
1997 | { | |||
1998 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
1999 | ||||
2000 | return sdma_v4_0_hw_fini(adev); | |||
2001 | } | |||
2002 | ||||
2003 | static int sdma_v4_0_resume(void *handle) | |||
2004 | { | |||
2005 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
2006 | ||||
2007 | return sdma_v4_0_hw_init(adev); | |||
2008 | } | |||
2009 | ||||
2010 | static bool_Bool sdma_v4_0_is_idle(void *handle) | |||
2011 | { | |||
2012 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
2013 | u32 i; | |||
2014 | ||||
2015 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2016 | u32 tmp = RREG32_SDMA(i, mmSDMA0_STATUS_REG)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0025))), 0); | |||
2017 | ||||
2018 | if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK0x00000001L)) | |||
2019 | return false0; | |||
2020 | } | |||
2021 | ||||
2022 | return true1; | |||
2023 | } | |||
2024 | ||||
2025 | static int sdma_v4_0_wait_for_idle(void *handle) | |||
2026 | { | |||
2027 | unsigned i, j; | |||
2028 | u32 sdma[AMDGPU_MAX_SDMA_INSTANCES8]; | |||
2029 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
2030 | ||||
2031 | for (i = 0; i < adev->usec_timeout; i++) { | |||
2032 | for (j = 0; j < adev->sdma.num_instances; j++) { | |||
2033 | sdma[j] = RREG32_SDMA(j, mmSDMA0_STATUS_REG)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (j), (0x0025))), 0); | |||
2034 | if (!(sdma[j] & SDMA0_STATUS_REG__IDLE_MASK0x00000001L)) | |||
2035 | break; | |||
2036 | } | |||
2037 | if (j == adev->sdma.num_instances) | |||
2038 | return 0; | |||
2039 | udelay(1); | |||
2040 | } | |||
2041 | return -ETIMEDOUT60; | |||
2042 | } | |||
2043 | ||||
2044 | static int sdma_v4_0_soft_reset(void *handle) | |||
2045 | { | |||
2046 | /* todo */ | |||
2047 | ||||
2048 | return 0; | |||
2049 | } | |||
2050 | ||||
2051 | static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev, | |||
2052 | struct amdgpu_irq_src *source, | |||
2053 | unsigned type, | |||
2054 | enum amdgpu_interrupt_state state) | |||
2055 | { | |||
2056 | u32 sdma_cntl; | |||
2057 | ||||
2058 | sdma_cntl = RREG32_SDMA(type, mmSDMA0_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (type ), (0x001c))), 0); | |||
2059 | sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,(((sdma_cntl) & ~0x00000001L) | (0x00000001L & ((state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0) << 0x0))) | |||
2060 | state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0)(((sdma_cntl) & ~0x00000001L) | (0x00000001L & ((state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0) << 0x0))); | |||
2061 | WREG32_SDMA(type, mmSDMA0_CNTL, sdma_cntl)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (type ), (0x001c))), (sdma_cntl), 0); | |||
2062 | ||||
2063 | return 0; | |||
2064 | } | |||
2065 | ||||
2066 | static int sdma_v4_0_process_trap_irq(struct amdgpu_device *adev, | |||
2067 | struct amdgpu_irq_src *source, | |||
2068 | struct amdgpu_iv_entry *entry) | |||
2069 | { | |||
2070 | uint32_t instance; | |||
2071 | ||||
2072 | DRM_DEBUG("IH: SDMA trap\n")__drm_dbg(DRM_UT_CORE, "IH: SDMA trap\n"); | |||
2073 | instance = sdma_v4_0_irq_id_to_seq(entry->client_id); | |||
2074 | switch (entry->ring_id) { | |||
2075 | case 0: | |||
2076 | amdgpu_fence_process(&adev->sdma.instance[instance].ring); | |||
2077 | break; | |||
2078 | case 1: | |||
2079 | if (adev->asic_type == CHIP_VEGA20) | |||
2080 | amdgpu_fence_process(&adev->sdma.instance[instance].page); | |||
2081 | break; | |||
2082 | case 2: | |||
2083 | /* XXX compute */ | |||
2084 | break; | |||
2085 | case 3: | |||
2086 | if (adev->asic_type != CHIP_VEGA20) | |||
2087 | amdgpu_fence_process(&adev->sdma.instance[instance].page); | |||
2088 | break; | |||
2089 | } | |||
2090 | return 0; | |||
2091 | } | |||
2092 | ||||
2093 | static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev, | |||
2094 | void *err_data, | |||
2095 | struct amdgpu_iv_entry *entry) | |||
2096 | { | |||
2097 | int instance; | |||
2098 | ||||
2099 | /* When “Full RAS” is enabled, the per-IP interrupt sources should | |||
2100 | * be disabled and the driver should only look for the aggregated | |||
2101 | * interrupt via sync flood | |||
2102 | */ | |||
2103 | if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) | |||
2104 | goto out; | |||
2105 | ||||
2106 | instance = sdma_v4_0_irq_id_to_seq(entry->client_id); | |||
2107 | if (instance < 0) | |||
2108 | goto out; | |||
2109 | ||||
2110 | amdgpu_sdma_process_ras_data_cb(adev, err_data, entry); | |||
2111 | ||||
2112 | out: | |||
2113 | return AMDGPU_RAS_SUCCESS; | |||
2114 | } | |||
2115 | ||||
2116 | static int sdma_v4_0_process_illegal_inst_irq(struct amdgpu_device *adev, | |||
2117 | struct amdgpu_irq_src *source, | |||
2118 | struct amdgpu_iv_entry *entry) | |||
2119 | { | |||
2120 | int instance; | |||
2121 | ||||
2122 | DRM_ERROR("Illegal instruction in SDMA command stream\n")__drm_err("Illegal instruction in SDMA command stream\n"); | |||
2123 | ||||
2124 | instance = sdma_v4_0_irq_id_to_seq(entry->client_id); | |||
2125 | if (instance < 0) | |||
2126 | return 0; | |||
2127 | ||||
2128 | switch (entry->ring_id) { | |||
2129 | case 0: | |||
2130 | drm_sched_fault(&adev->sdma.instance[instance].ring.sched); | |||
2131 | break; | |||
2132 | } | |||
2133 | return 0; | |||
2134 | } | |||
2135 | ||||
2136 | static int sdma_v4_0_set_ecc_irq_state(struct amdgpu_device *adev, | |||
2137 | struct amdgpu_irq_src *source, | |||
2138 | unsigned type, | |||
2139 | enum amdgpu_interrupt_state state) | |||
2140 | { | |||
2141 | u32 sdma_edc_config; | |||
2142 | ||||
2143 | sdma_edc_config = RREG32_SDMA(type, mmSDMA0_EDC_CONFIG)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (type ), (0x0032))), 0); | |||
2144 | sdma_edc_config = REG_SET_FIELD(sdma_edc_config, SDMA0_EDC_CONFIG, ECC_INT_ENABLE,(((sdma_edc_config) & ~0x00000004L) | (0x00000004L & ( (state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0) << 0x2))) | |||
2145 | state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0)(((sdma_edc_config) & ~0x00000004L) | (0x00000004L & ( (state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0) << 0x2))); | |||
2146 | WREG32_SDMA(type, mmSDMA0_EDC_CONFIG, sdma_edc_config)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (type ), (0x0032))), (sdma_edc_config), 0); | |||
2147 | ||||
2148 | return 0; | |||
2149 | } | |||
2150 | ||||
2151 | static void sdma_v4_0_update_medium_grain_clock_gating( | |||
2152 | struct amdgpu_device *adev, | |||
2153 | bool_Bool enable) | |||
2154 | { | |||
2155 | uint32_t data, def; | |||
2156 | int i; | |||
2157 | ||||
2158 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG(1 << 11))) { | |||
2159 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2160 | def = data = RREG32_SDMA(i, mmSDMA0_CLK_CTRL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001b))), 0); | |||
2161 | data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK0x01000000L | | |||
2162 | SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK0x02000000L | | |||
2163 | SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK0x04000000L | | |||
2164 | SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK0x08000000L | | |||
2165 | SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK0x10000000L | | |||
2166 | SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK0x20000000L | | |||
2167 | SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK0x40000000L | | |||
2168 | SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK0x80000000L); | |||
2169 | if (def != data) | |||
2170 | WREG32_SDMA(i, mmSDMA0_CLK_CTRL, data)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001b))), (data), 0); | |||
2171 | } | |||
2172 | } else { | |||
2173 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2174 | def = data = RREG32_SDMA(i, mmSDMA0_CLK_CTRL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001b))), 0); | |||
2175 | data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK0x01000000L | | |||
2176 | SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK0x02000000L | | |||
2177 | SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK0x04000000L | | |||
2178 | SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK0x08000000L | | |||
2179 | SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK0x10000000L | | |||
2180 | SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK0x20000000L | | |||
2181 | SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK0x40000000L | | |||
2182 | SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK0x80000000L); | |||
2183 | if (def != data) | |||
2184 | WREG32_SDMA(i, mmSDMA0_CLK_CTRL, data)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x001b))), (data), 0); | |||
2185 | } | |||
2186 | } | |||
2187 | } | |||
2188 | ||||
2189 | ||||
2190 | static void sdma_v4_0_update_medium_grain_light_sleep( | |||
2191 | struct amdgpu_device *adev, | |||
2192 | bool_Bool enable) | |||
2193 | { | |||
2194 | uint32_t data, def; | |||
2195 | int i; | |||
2196 | ||||
2197 | if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS(1 << 10))) { | |||
2198 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2199 | /* 1-not override: enable sdma mem light sleep */ | |||
2200 | def = data = RREG32_SDMA(0, mmSDMA0_POWER_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (0), (0x001a))), 0); | |||
2201 | data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK0x00000100L; | |||
2202 | if (def != data) | |||
2203 | WREG32_SDMA(0, mmSDMA0_POWER_CNTL, data)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (0), (0x001a))), (data), 0); | |||
2204 | } | |||
2205 | } else { | |||
2206 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2207 | /* 0-override:disable sdma mem light sleep */ | |||
2208 | def = data = RREG32_SDMA(0, mmSDMA0_POWER_CNTL)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (0), (0x001a))), 0); | |||
2209 | data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK0x00000100L; | |||
2210 | if (def != data) | |||
2211 | WREG32_SDMA(0, mmSDMA0_POWER_CNTL, data)amdgpu_device_wreg(adev, (sdma_v4_0_get_reg_offset(adev, (0), (0x001a))), (data), 0); | |||
2212 | } | |||
2213 | } | |||
2214 | } | |||
2215 | ||||
2216 | static int sdma_v4_0_set_clockgating_state(void *handle, | |||
2217 | enum amd_clockgating_state state) | |||
2218 | { | |||
2219 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
2220 | ||||
2221 | if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) | |||
2222 | return 0; | |||
2223 | ||||
2224 | switch (adev->asic_type) { | |||
2225 | case CHIP_VEGA10: | |||
2226 | case CHIP_VEGA12: | |||
2227 | case CHIP_VEGA20: | |||
2228 | case CHIP_RAVEN: | |||
2229 | case CHIP_ARCTURUS: | |||
2230 | case CHIP_RENOIR: | |||
2231 | sdma_v4_0_update_medium_grain_clock_gating(adev, | |||
2232 | state == AMD_CG_STATE_GATE); | |||
2233 | sdma_v4_0_update_medium_grain_light_sleep(adev, | |||
2234 | state == AMD_CG_STATE_GATE); | |||
2235 | break; | |||
2236 | default: | |||
2237 | break; | |||
2238 | } | |||
2239 | return 0; | |||
2240 | } | |||
2241 | ||||
2242 | static int sdma_v4_0_set_powergating_state(void *handle, | |||
2243 | enum amd_powergating_state state) | |||
2244 | { | |||
2245 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
2246 | ||||
2247 | switch (adev->asic_type) { | |||
2248 | case CHIP_RAVEN: | |||
2249 | case CHIP_RENOIR: | |||
2250 | sdma_v4_1_update_power_gating(adev, | |||
2251 | state == AMD_PG_STATE_GATE ? true1 : false0); | |||
2252 | break; | |||
2253 | default: | |||
2254 | break; | |||
2255 | } | |||
2256 | ||||
2257 | return 0; | |||
2258 | } | |||
2259 | ||||
2260 | static void sdma_v4_0_get_clockgating_state(void *handle, u32 *flags) | |||
2261 | { | |||
2262 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |||
2263 | int data; | |||
2264 | ||||
2265 | if (amdgpu_sriov_vf(adev)((adev)->virt.caps & (1 << 2))) | |||
2266 | *flags = 0; | |||
2267 | ||||
2268 | /* AMD_CG_SUPPORT_SDMA_MGCG */ | |||
2269 | data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001b)), 0); | |||
2270 | if (!(data & SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK0x01000000L)) | |||
2271 | *flags |= AMD_CG_SUPPORT_SDMA_MGCG(1 << 11); | |||
2272 | ||||
2273 | /* AMD_CG_SUPPORT_SDMA_LS */ | |||
2274 | data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL))amdgpu_device_rreg(adev, ((adev->reg_offset[SDMA0_HWIP][0] [0] + 0x001a)), 0); | |||
2275 | if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK0x00000100L) | |||
2276 | *flags |= AMD_CG_SUPPORT_SDMA_LS(1 << 10); | |||
2277 | } | |||
2278 | ||||
2279 | const struct amd_ip_funcs sdma_v4_0_ip_funcs = { | |||
2280 | .name = "sdma_v4_0", | |||
2281 | .early_init = sdma_v4_0_early_init, | |||
2282 | .late_init = sdma_v4_0_late_init, | |||
2283 | .sw_init = sdma_v4_0_sw_init, | |||
2284 | .sw_fini = sdma_v4_0_sw_fini, | |||
2285 | .hw_init = sdma_v4_0_hw_init, | |||
2286 | .hw_fini = sdma_v4_0_hw_fini, | |||
2287 | .suspend = sdma_v4_0_suspend, | |||
2288 | .resume = sdma_v4_0_resume, | |||
2289 | .is_idle = sdma_v4_0_is_idle, | |||
2290 | .wait_for_idle = sdma_v4_0_wait_for_idle, | |||
2291 | .soft_reset = sdma_v4_0_soft_reset, | |||
2292 | .set_clockgating_state = sdma_v4_0_set_clockgating_state, | |||
2293 | .set_powergating_state = sdma_v4_0_set_powergating_state, | |||
2294 | .get_clockgating_state = sdma_v4_0_get_clockgating_state, | |||
2295 | }; | |||
2296 | ||||
2297 | static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = { | |||
2298 | .type = AMDGPU_RING_TYPE_SDMA, | |||
2299 | .align_mask = 0xf, | |||
2300 | .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0), | |||
2301 | .support_64bit_ptrs = true1, | |||
2302 | .vmhub = AMDGPU_MMHUB_01, | |||
2303 | .get_rptr = sdma_v4_0_ring_get_rptr, | |||
2304 | .get_wptr = sdma_v4_0_ring_get_wptr, | |||
2305 | .set_wptr = sdma_v4_0_ring_set_wptr, | |||
2306 | .emit_frame_size = | |||
2307 | 6 + /* sdma_v4_0_ring_emit_hdp_flush */ | |||
2308 | 3 + /* hdp invalidate */ | |||
2309 | 6 + /* sdma_v4_0_ring_emit_pipeline_sync */ | |||
2310 | /* sdma_v4_0_ring_emit_vm_flush */ | |||
2311 | SOC15_FLUSH_GPU_TLB_NUM_WREG6 * 3 + | |||
2312 | SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT3 * 6 + | |||
2313 | 10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */ | |||
2314 | .emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */ | |||
2315 | .emit_ib = sdma_v4_0_ring_emit_ib, | |||
2316 | .emit_fence = sdma_v4_0_ring_emit_fence, | |||
2317 | .emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync, | |||
2318 | .emit_vm_flush = sdma_v4_0_ring_emit_vm_flush, | |||
2319 | .emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush, | |||
2320 | .test_ring = sdma_v4_0_ring_test_ring, | |||
2321 | .test_ib = sdma_v4_0_ring_test_ib, | |||
2322 | .insert_nop = sdma_v4_0_ring_insert_nop, | |||
2323 | .pad_ib = sdma_v4_0_ring_pad_ib, | |||
2324 | .emit_wreg = sdma_v4_0_ring_emit_wreg, | |||
2325 | .emit_reg_wait = sdma_v4_0_ring_emit_reg_wait, | |||
2326 | .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, | |||
2327 | }; | |||
2328 | ||||
2329 | /* | |||
2330 | * On Arcturus, SDMA instance 5~7 has a different vmhub type(AMDGPU_MMHUB_1). | |||
2331 | * So create a individual constant ring_funcs for those instances. | |||
2332 | */ | |||
2333 | static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs_2nd_mmhub = { | |||
2334 | .type = AMDGPU_RING_TYPE_SDMA, | |||
2335 | .align_mask = 0xf, | |||
2336 | .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0), | |||
2337 | .support_64bit_ptrs = true1, | |||
2338 | .vmhub = AMDGPU_MMHUB_12, | |||
2339 | .get_rptr = sdma_v4_0_ring_get_rptr, | |||
2340 | .get_wptr = sdma_v4_0_ring_get_wptr, | |||
2341 | .set_wptr = sdma_v4_0_ring_set_wptr, | |||
2342 | .emit_frame_size = | |||
2343 | 6 + /* sdma_v4_0_ring_emit_hdp_flush */ | |||
2344 | 3 + /* hdp invalidate */ | |||
2345 | 6 + /* sdma_v4_0_ring_emit_pipeline_sync */ | |||
2346 | /* sdma_v4_0_ring_emit_vm_flush */ | |||
2347 | SOC15_FLUSH_GPU_TLB_NUM_WREG6 * 3 + | |||
2348 | SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT3 * 6 + | |||
2349 | 10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */ | |||
2350 | .emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */ | |||
2351 | .emit_ib = sdma_v4_0_ring_emit_ib, | |||
2352 | .emit_fence = sdma_v4_0_ring_emit_fence, | |||
2353 | .emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync, | |||
2354 | .emit_vm_flush = sdma_v4_0_ring_emit_vm_flush, | |||
2355 | .emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush, | |||
2356 | .test_ring = sdma_v4_0_ring_test_ring, | |||
2357 | .test_ib = sdma_v4_0_ring_test_ib, | |||
2358 | .insert_nop = sdma_v4_0_ring_insert_nop, | |||
2359 | .pad_ib = sdma_v4_0_ring_pad_ib, | |||
2360 | .emit_wreg = sdma_v4_0_ring_emit_wreg, | |||
2361 | .emit_reg_wait = sdma_v4_0_ring_emit_reg_wait, | |||
2362 | .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, | |||
2363 | }; | |||
2364 | ||||
2365 | static const struct amdgpu_ring_funcs sdma_v4_0_page_ring_funcs = { | |||
2366 | .type = AMDGPU_RING_TYPE_SDMA, | |||
2367 | .align_mask = 0xf, | |||
2368 | .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0), | |||
2369 | .support_64bit_ptrs = true1, | |||
2370 | .vmhub = AMDGPU_MMHUB_01, | |||
2371 | .get_rptr = sdma_v4_0_ring_get_rptr, | |||
2372 | .get_wptr = sdma_v4_0_page_ring_get_wptr, | |||
2373 | .set_wptr = sdma_v4_0_page_ring_set_wptr, | |||
2374 | .emit_frame_size = | |||
2375 | 6 + /* sdma_v4_0_ring_emit_hdp_flush */ | |||
2376 | 3 + /* hdp invalidate */ | |||
2377 | 6 + /* sdma_v4_0_ring_emit_pipeline_sync */ | |||
2378 | /* sdma_v4_0_ring_emit_vm_flush */ | |||
2379 | SOC15_FLUSH_GPU_TLB_NUM_WREG6 * 3 + | |||
2380 | SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT3 * 6 + | |||
2381 | 10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */ | |||
2382 | .emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */ | |||
2383 | .emit_ib = sdma_v4_0_ring_emit_ib, | |||
2384 | .emit_fence = sdma_v4_0_ring_emit_fence, | |||
2385 | .emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync, | |||
2386 | .emit_vm_flush = sdma_v4_0_ring_emit_vm_flush, | |||
2387 | .emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush, | |||
2388 | .test_ring = sdma_v4_0_ring_test_ring, | |||
2389 | .test_ib = sdma_v4_0_ring_test_ib, | |||
2390 | .insert_nop = sdma_v4_0_ring_insert_nop, | |||
2391 | .pad_ib = sdma_v4_0_ring_pad_ib, | |||
2392 | .emit_wreg = sdma_v4_0_ring_emit_wreg, | |||
2393 | .emit_reg_wait = sdma_v4_0_ring_emit_reg_wait, | |||
2394 | .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, | |||
2395 | }; | |||
2396 | ||||
2397 | static const struct amdgpu_ring_funcs sdma_v4_0_page_ring_funcs_2nd_mmhub = { | |||
2398 | .type = AMDGPU_RING_TYPE_SDMA, | |||
2399 | .align_mask = 0xf, | |||
2400 | .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP)(((0) & 0x000000FF) << 0), | |||
2401 | .support_64bit_ptrs = true1, | |||
2402 | .vmhub = AMDGPU_MMHUB_12, | |||
2403 | .get_rptr = sdma_v4_0_ring_get_rptr, | |||
2404 | .get_wptr = sdma_v4_0_page_ring_get_wptr, | |||
2405 | .set_wptr = sdma_v4_0_page_ring_set_wptr, | |||
2406 | .emit_frame_size = | |||
2407 | 6 + /* sdma_v4_0_ring_emit_hdp_flush */ | |||
2408 | 3 + /* hdp invalidate */ | |||
2409 | 6 + /* sdma_v4_0_ring_emit_pipeline_sync */ | |||
2410 | /* sdma_v4_0_ring_emit_vm_flush */ | |||
2411 | SOC15_FLUSH_GPU_TLB_NUM_WREG6 * 3 + | |||
2412 | SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT3 * 6 + | |||
2413 | 10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */ | |||
2414 | .emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */ | |||
2415 | .emit_ib = sdma_v4_0_ring_emit_ib, | |||
2416 | .emit_fence = sdma_v4_0_ring_emit_fence, | |||
2417 | .emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync, | |||
2418 | .emit_vm_flush = sdma_v4_0_ring_emit_vm_flush, | |||
2419 | .emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush, | |||
2420 | .test_ring = sdma_v4_0_ring_test_ring, | |||
2421 | .test_ib = sdma_v4_0_ring_test_ib, | |||
2422 | .insert_nop = sdma_v4_0_ring_insert_nop, | |||
2423 | .pad_ib = sdma_v4_0_ring_pad_ib, | |||
2424 | .emit_wreg = sdma_v4_0_ring_emit_wreg, | |||
2425 | .emit_reg_wait = sdma_v4_0_ring_emit_reg_wait, | |||
2426 | .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, | |||
2427 | }; | |||
2428 | ||||
2429 | static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev) | |||
2430 | { | |||
2431 | int i; | |||
2432 | ||||
2433 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2434 | if (adev->asic_type == CHIP_ARCTURUS && i >= 5) | |||
2435 | adev->sdma.instance[i].ring.funcs = | |||
2436 | &sdma_v4_0_ring_funcs_2nd_mmhub; | |||
2437 | else | |||
2438 | adev->sdma.instance[i].ring.funcs = | |||
2439 | &sdma_v4_0_ring_funcs; | |||
2440 | adev->sdma.instance[i].ring.me = i; | |||
2441 | if (adev->sdma.has_page_queue) { | |||
2442 | if (adev->asic_type == CHIP_ARCTURUS && i >= 5) | |||
2443 | adev->sdma.instance[i].page.funcs = | |||
2444 | &sdma_v4_0_page_ring_funcs_2nd_mmhub; | |||
2445 | else | |||
2446 | adev->sdma.instance[i].page.funcs = | |||
2447 | &sdma_v4_0_page_ring_funcs; | |||
2448 | adev->sdma.instance[i].page.me = i; | |||
2449 | } | |||
2450 | } | |||
2451 | } | |||
2452 | ||||
2453 | static const struct amdgpu_irq_src_funcs sdma_v4_0_trap_irq_funcs = { | |||
2454 | .set = sdma_v4_0_set_trap_irq_state, | |||
2455 | .process = sdma_v4_0_process_trap_irq, | |||
2456 | }; | |||
2457 | ||||
2458 | static const struct amdgpu_irq_src_funcs sdma_v4_0_illegal_inst_irq_funcs = { | |||
2459 | .process = sdma_v4_0_process_illegal_inst_irq, | |||
2460 | }; | |||
2461 | ||||
2462 | static const struct amdgpu_irq_src_funcs sdma_v4_0_ecc_irq_funcs = { | |||
2463 | .set = sdma_v4_0_set_ecc_irq_state, | |||
2464 | .process = amdgpu_sdma_process_ecc_irq, | |||
2465 | }; | |||
2466 | ||||
2467 | ||||
2468 | ||||
2469 | static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev) | |||
2470 | { | |||
2471 | switch (adev->sdma.num_instances) { | |||
2472 | case 1: | |||
2473 | adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE1; | |||
2474 | adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE1; | |||
2475 | break; | |||
2476 | case 8: | |||
2477 | adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST; | |||
2478 | adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_LAST; | |||
2479 | break; | |||
2480 | case 2: | |||
2481 | default: | |||
2482 | adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE2; | |||
2483 | adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE2; | |||
2484 | break; | |||
2485 | } | |||
2486 | adev->sdma.trap_irq.funcs = &sdma_v4_0_trap_irq_funcs; | |||
2487 | adev->sdma.illegal_inst_irq.funcs = &sdma_v4_0_illegal_inst_irq_funcs; | |||
2488 | adev->sdma.ecc_irq.funcs = &sdma_v4_0_ecc_irq_funcs; | |||
2489 | } | |||
2490 | ||||
2491 | /** | |||
2492 | * sdma_v4_0_emit_copy_buffer - copy buffer using the sDMA engine | |||
2493 | * | |||
2494 | * @ring: amdgpu_ring structure holding ring information | |||
2495 | * @src_offset: src GPU address | |||
2496 | * @dst_offset: dst GPU address | |||
2497 | * @byte_count: number of bytes to xfer | |||
2498 | * | |||
2499 | * Copy GPU buffers using the DMA engine (VEGA10/12). | |||
2500 | * Used by the amdgpu ttm implementation to move pages if | |||
2501 | * registered as the asic copy callback. | |||
2502 | */ | |||
2503 | static void sdma_v4_0_emit_copy_buffer(struct amdgpu_ib *ib, | |||
2504 | uint64_t src_offset, | |||
2505 | uint64_t dst_offset, | |||
2506 | uint32_t byte_count, | |||
2507 | bool_Bool tmz) | |||
2508 | { | |||
2509 | ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY)(((1) & 0x000000FF) << 0) | | |||
2510 | SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR)(((0) & 0x000000FF) << 8) | | |||
2511 | SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0)(((tmz ? 1 : 0) & 0x00000001) << 18); | |||
2512 | ib->ptr[ib->length_dw++] = byte_count - 1; | |||
2513 | ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ | |||
2514 | ib->ptr[ib->length_dw++] = lower_32_bits(src_offset)((u32)(src_offset)); | |||
2515 | ib->ptr[ib->length_dw++] = upper_32_bits(src_offset)((u32)(((src_offset) >> 16) >> 16)); | |||
2516 | ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset)((u32)(dst_offset)); | |||
2517 | ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset)((u32)(((dst_offset) >> 16) >> 16)); | |||
2518 | } | |||
2519 | ||||
2520 | /** | |||
2521 | * sdma_v4_0_emit_fill_buffer - fill buffer using the sDMA engine | |||
2522 | * | |||
2523 | * @ring: amdgpu_ring structure holding ring information | |||
2524 | * @src_data: value to write to buffer | |||
2525 | * @dst_offset: dst GPU address | |||
2526 | * @byte_count: number of bytes to xfer | |||
2527 | * | |||
2528 | * Fill GPU buffers using the DMA engine (VEGA10/12). | |||
2529 | */ | |||
2530 | static void sdma_v4_0_emit_fill_buffer(struct amdgpu_ib *ib, | |||
2531 | uint32_t src_data, | |||
2532 | uint64_t dst_offset, | |||
2533 | uint32_t byte_count) | |||
2534 | { | |||
2535 | ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL)(((11) & 0x000000FF) << 0); | |||
2536 | ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset)((u32)(dst_offset)); | |||
2537 | ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset)((u32)(((dst_offset) >> 16) >> 16)); | |||
2538 | ib->ptr[ib->length_dw++] = src_data; | |||
2539 | ib->ptr[ib->length_dw++] = byte_count - 1; | |||
2540 | } | |||
2541 | ||||
2542 | static const struct amdgpu_buffer_funcs sdma_v4_0_buffer_funcs = { | |||
2543 | .copy_max_bytes = 0x400000, | |||
2544 | .copy_num_dw = 7, | |||
2545 | .emit_copy_buffer = sdma_v4_0_emit_copy_buffer, | |||
2546 | ||||
2547 | .fill_max_bytes = 0x400000, | |||
2548 | .fill_num_dw = 5, | |||
2549 | .emit_fill_buffer = sdma_v4_0_emit_fill_buffer, | |||
2550 | }; | |||
2551 | ||||
2552 | static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev) | |||
2553 | { | |||
2554 | adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs; | |||
2555 | if (adev->sdma.has_page_queue) | |||
2556 | adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page; | |||
2557 | else | |||
2558 | adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; | |||
2559 | } | |||
2560 | ||||
2561 | static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = { | |||
2562 | .copy_pte_num_dw = 7, | |||
2563 | .copy_pte = sdma_v4_0_vm_copy_pte, | |||
2564 | ||||
2565 | .write_pte = sdma_v4_0_vm_write_pte, | |||
2566 | .set_pte_pde = sdma_v4_0_vm_set_pte_pde, | |||
2567 | }; | |||
2568 | ||||
2569 | static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev) | |||
2570 | { | |||
2571 | struct drm_gpu_scheduler *sched; | |||
2572 | unsigned i; | |||
2573 | ||||
2574 | adev->vm_manager.vm_pte_funcs = &sdma_v4_0_vm_pte_funcs; | |||
2575 | for (i = 0; i < adev->sdma.num_instances; i++) { | |||
2576 | if (adev->sdma.has_page_queue) | |||
2577 | sched = &adev->sdma.instance[i].page.sched; | |||
2578 | else | |||
2579 | sched = &adev->sdma.instance[i].ring.sched; | |||
2580 | adev->vm_manager.vm_pte_scheds[i] = sched; | |||
2581 | } | |||
2582 | adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances; | |||
2583 | } | |||
2584 | ||||
2585 | static void sdma_v4_0_get_ras_error_count(uint32_t value, | |||
2586 | uint32_t instance, | |||
2587 | uint32_t *sec_count) | |||
2588 | { | |||
2589 | uint32_t i; | |||
2590 | uint32_t sec_cnt; | |||
2591 | ||||
2592 | /* double bits error (multiple bits) error detection is not supported */ | |||
2593 | for (i = 0; i < ARRAY_SIZE(sdma_v4_0_ras_fields)(sizeof((sdma_v4_0_ras_fields)) / sizeof((sdma_v4_0_ras_fields )[0])); i++) { | |||
2594 | /* the SDMA_EDC_COUNTER register in each sdma instance | |||
2595 | * shares the same sed shift_mask | |||
2596 | * */ | |||
2597 | sec_cnt = (value & | |||
2598 | sdma_v4_0_ras_fields[i].sec_count_mask) >> | |||
2599 | sdma_v4_0_ras_fields[i].sec_count_shift; | |||
2600 | if (sec_cnt) { | |||
2601 | DRM_INFO("Detected %s in SDMA%d, SED %d\n",printk("\0016" "[" "drm" "] " "Detected %s in SDMA%d, SED %d\n" , sdma_v4_0_ras_fields[i].name, instance, sec_cnt) | |||
2602 | sdma_v4_0_ras_fields[i].name,printk("\0016" "[" "drm" "] " "Detected %s in SDMA%d, SED %d\n" , sdma_v4_0_ras_fields[i].name, instance, sec_cnt) | |||
2603 | instance, sec_cnt)printk("\0016" "[" "drm" "] " "Detected %s in SDMA%d, SED %d\n" , sdma_v4_0_ras_fields[i].name, instance, sec_cnt); | |||
2604 | *sec_count += sec_cnt; | |||
2605 | } | |||
2606 | } | |||
2607 | } | |||
2608 | ||||
2609 | static int sdma_v4_0_query_ras_error_count(struct amdgpu_device *adev, | |||
2610 | uint32_t instance, void *ras_error_status) | |||
2611 | { | |||
2612 | struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; | |||
2613 | uint32_t sec_count = 0; | |||
2614 | uint32_t reg_value = 0; | |||
2615 | ||||
2616 | reg_value = RREG32_SDMA(instance, mmSDMA0_EDC_COUNTER)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (instance ), (0x0036))), 0); | |||
2617 | /* double bit error is not supported */ | |||
2618 | if (reg_value) | |||
2619 | sdma_v4_0_get_ras_error_count(reg_value, | |||
2620 | instance, &sec_count); | |||
2621 | /* err_data->ce_count should be initialized to 0 | |||
2622 | * before calling into this function */ | |||
2623 | err_data->ce_count += sec_count; | |||
2624 | /* double bit error is not supported | |||
2625 | * set ue count to 0 */ | |||
2626 | err_data->ue_count = 0; | |||
2627 | ||||
2628 | return 0; | |||
2629 | }; | |||
2630 | ||||
2631 | static void sdma_v4_0_reset_ras_error_count(struct amdgpu_device *adev) | |||
2632 | { | |||
2633 | int i; | |||
2634 | ||||
2635 | /* read back edc counter registers to clear the counters */ | |||
2636 | if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) { | |||
2637 | for (i = 0; i < adev->sdma.num_instances; i++) | |||
2638 | RREG32_SDMA(i, mmSDMA0_EDC_COUNTER)amdgpu_device_rreg(adev, (sdma_v4_0_get_reg_offset(adev, (i), (0x0036))), 0); | |||
2639 | } | |||
2640 | } | |||
2641 | ||||
2642 | static const struct amdgpu_sdma_ras_funcs sdma_v4_0_ras_funcs = { | |||
2643 | .ras_late_init = amdgpu_sdma_ras_late_init, | |||
2644 | .ras_fini = amdgpu_sdma_ras_fini, | |||
2645 | .query_ras_error_count = sdma_v4_0_query_ras_error_count, | |||
2646 | .reset_ras_error_count = sdma_v4_0_reset_ras_error_count, | |||
2647 | }; | |||
2648 | ||||
2649 | static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev) | |||
2650 | { | |||
2651 | switch (adev->asic_type) { | |||
2652 | case CHIP_VEGA20: | |||
2653 | case CHIP_ARCTURUS: | |||
2654 | adev->sdma.funcs = &sdma_v4_0_ras_funcs; | |||
2655 | break; | |||
2656 | default: | |||
2657 | break; | |||
2658 | } | |||
2659 | } | |||
2660 | ||||
2661 | const struct amdgpu_ip_block_version sdma_v4_0_ip_block = { | |||
2662 | .type = AMD_IP_BLOCK_TYPE_SDMA, | |||
2663 | .major = 4, | |||
2664 | .minor = 0, | |||
2665 | .rev = 0, | |||
2666 | .funcs = &sdma_v4_0_ip_funcs, | |||
2667 | }; |
1 | /* Public domain. */ |
2 | |
3 | #ifndef _LINUX_FIRMWARE_H |
4 | #define _LINUX_FIRMWARE_H |
5 | |
6 | #include <sys/types.h> |
7 | #include <sys/malloc.h> |
8 | #include <sys/device.h> |
9 | #include <linux/types.h> |
10 | #include <linux/gfp.h> |
11 | |
12 | #ifndef __DECONST |
13 | #define __DECONST(type, var)((type)(__uintptr_t)(const void *)(var)) ((type)(__uintptr_t)(const void *)(var)) |
14 | #endif |
15 | |
16 | struct firmware { |
17 | size_t size; |
18 | const u8 *data; |
19 | }; |
20 | |
21 | static inline int |
22 | request_firmware(const struct firmware **fw, const char *name, |
23 | struct device *device) |
24 | { |
25 | int r; |
26 | struct firmware *f = malloc(sizeof(struct firmware), M_DRM145, |
27 | M_WAITOK0x0001 | M_ZERO0x0008); |
28 | r = loadfirmware(name, __DECONST(u_char **, &f->data)((u_char **)(__uintptr_t)(const void *)(&f->data)), &f->size); |
29 | if (r != 0) { |
30 | free(f, M_DRM145, sizeof(struct firmware)); |
31 | *fw = NULL((void *)0); |
32 | return -r; |
33 | } else { |
34 | *fw = f; |
35 | return 0; |
36 | } |
37 | } |
38 | |
39 | static inline int |
40 | request_firmware_direct(const struct firmware **fw, const char *name, |
41 | struct device *device) |
42 | { |
43 | return request_firmware(fw, name, device); |
44 | } |
45 | |
46 | #define request_firmware_nowait(a, b, c, d, e, f, g)-22 -EINVAL22 |
47 | |
48 | static inline void |
49 | release_firmware(const struct firmware *fw) |
50 | { |
51 | if (fw) |
52 | free(__DECONST(u_char *, fw->data)((u_char *)(__uintptr_t)(const void *)(fw->data)), M_DEVBUF2, fw->size); |
53 | free(__DECONST(struct firmware *, fw)((struct firmware *)(__uintptr_t)(const void *)(fw)), M_DRM145, sizeof(*fw)); |
54 | } |
55 | |
56 | #endif |