File: | dev/pci/drm/i915/intel_uncore.c |
Warning: | line 1755, column 2 Value stored to 'mmio_bar' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* |
2 | * Copyright © 2013 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | */ |
23 | |
24 | #include <linux/pm_runtime.h> |
25 | #include <asm/iosf_mbi.h> |
26 | |
27 | #include "i915_drv.h" |
28 | #include "i915_trace.h" |
29 | #include "i915_vgpu.h" |
30 | #include "intel_pm.h" |
31 | |
32 | #define FORCEWAKE_ACK_TIMEOUT_MS50 50 |
33 | #define GT_FIFO_TIMEOUT_MS10 10 |
34 | |
35 | #define __raw_posting_read(...)((void)__raw_uncore_read32(...)) ((void)__raw_uncore_read32(__VA_ARGS__)) |
36 | |
37 | void |
38 | intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug) |
39 | { |
40 | mtx_init(&mmio_debug->lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&mmio_debug ->lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9) ? 0x9 : ((0x9)))); } while (0); |
41 | mmio_debug->unclaimed_mmio_check = 1; |
42 | } |
43 | |
44 | static void mmio_debug_suspend(struct intel_uncore_mmio_debug *mmio_debug) |
45 | { |
46 | lockdep_assert_held(&mmio_debug->lock)do { (void)(&mmio_debug->lock); } while(0); |
47 | |
48 | /* Save and disable mmio debugging for the user bypass */ |
49 | if (!mmio_debug->suspend_count++) { |
50 | mmio_debug->saved_mmio_check = mmio_debug->unclaimed_mmio_check; |
51 | mmio_debug->unclaimed_mmio_check = 0; |
52 | } |
53 | } |
54 | |
55 | static void mmio_debug_resume(struct intel_uncore_mmio_debug *mmio_debug) |
56 | { |
57 | lockdep_assert_held(&mmio_debug->lock)do { (void)(&mmio_debug->lock); } while(0); |
58 | |
59 | if (!--mmio_debug->suspend_count) |
60 | mmio_debug->unclaimed_mmio_check = mmio_debug->saved_mmio_check; |
61 | } |
62 | |
63 | static const char * const forcewake_domain_names[] = { |
64 | "render", |
65 | "blitter", |
66 | "media", |
67 | "vdbox0", |
68 | "vdbox1", |
69 | "vdbox2", |
70 | "vdbox3", |
71 | "vebox0", |
72 | "vebox1", |
73 | }; |
74 | |
75 | const char * |
76 | intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id) |
77 | { |
78 | BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT)extern char _ctassert[(!((sizeof((forcewake_domain_names)) / sizeof ((forcewake_domain_names)[0])) != FW_DOMAIN_ID_COUNT)) ? 1 : - 1 ] __attribute__((__unused__)); |
79 | |
80 | if (id >= 0 && id < FW_DOMAIN_ID_COUNT) |
81 | return forcewake_domain_names[id]; |
82 | |
83 | WARN_ON(id)({ int __ret = !!((id)); if (__ret) printf("%s", "WARN_ON(" "id" ")"); __builtin_expect(!!(__ret), 0); }); |
84 | |
85 | return "unknown"; |
86 | } |
87 | |
88 | #define fw_ack(d)ioread32((d)->reg_ack) readl((d)->reg_ack)ioread32((d)->reg_ack) |
89 | #define fw_set(d, val)iowrite32(({ typeof((val)) _a = ((val)); ({ if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }), (d )->reg_set) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)iowrite32(({ typeof((val)) _a = ((val)); ({ if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }), (d )->reg_set) |
90 | #define fw_clear(d, val)iowrite32((({ if (__builtin_constant_p(((val)))) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((val))) && __builtin_constant_p(0)) do { } while (0 ); ((((val))) << 16 | (0)); })), (d)->reg_set) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)iowrite32((({ if (__builtin_constant_p(((val)))) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((val))) && __builtin_constant_p(0)) do { } while (0 ); ((((val))) << 16 | (0)); })), (d)->reg_set) |
91 | |
92 | static inline void |
93 | fw_domain_reset(const struct intel_uncore_forcewake_domain *d) |
94 | { |
95 | /* |
96 | * We don't really know if the powerwell for the forcewake domain we are |
97 | * trying to reset here does exist at this point (engines could be fused |
98 | * off in ICL+), so no waiting for acks |
99 | */ |
100 | /* WaRsClearFWBitsAtReset:bdw,skl */ |
101 | fw_clear(d, 0xffff)iowrite32((({ if (__builtin_constant_p(((0xffff)))) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((0xffff))) && __builtin_constant_p(0)) do { } while (0); ((((0xffff))) << 16 | (0)); })), (d)->reg_set); |
102 | } |
103 | |
104 | static inline void |
105 | fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d) |
106 | { |
107 | GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask)((void)0); |
108 | d->uncore->fw_domains_timer |= d->mask; |
109 | d->wake_count++; |
110 | #ifdef __linux__ |
111 | hrtimer_start_range_ns(&d->timer, |
112 | NSEC_PER_MSEC1000000L, |
113 | NSEC_PER_MSEC1000000L, |
114 | HRTIMER_MODE_REL1); |
115 | #else |
116 | timeout_add_msec(&d->timer, 1); |
117 | #endif |
118 | } |
119 | |
120 | static inline int |
121 | __wait_for_ack(const struct intel_uncore_forcewake_domain *d, |
122 | const u32 ack, |
123 | const u32 value) |
124 | { |
125 | return wait_for_atomic((fw_ack(d) & ack) == value,({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((((ioread32((d)-> reg_ack) & ack) == value))) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (! (1)) { ; if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile( "movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cpuid); base = local_clock (); } } } ret; }); }) |
126 | FORCEWAKE_ACK_TIMEOUT_MS)({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((((ioread32((d)-> reg_ack) & ack) == value))) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (! (1)) { ; if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile( "movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cpuid); base = local_clock (); } } } ret; }); }); |
127 | } |
128 | |
129 | static inline int |
130 | wait_ack_clear(const struct intel_uncore_forcewake_domain *d, |
131 | const u32 ack) |
132 | { |
133 | return __wait_for_ack(d, ack, 0); |
134 | } |
135 | |
136 | static inline int |
137 | wait_ack_set(const struct intel_uncore_forcewake_domain *d, |
138 | const u32 ack) |
139 | { |
140 | return __wait_for_ack(d, ack, ack); |
141 | } |
142 | |
143 | static inline void |
144 | fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d) |
145 | { |
146 | if (wait_ack_clear(d, FORCEWAKE_KERNEL(1UL << (0)))) { |
147 | DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",__drm_err("%s: timed out waiting for forcewake ack to clear.\n" , intel_uncore_forcewake_domain_to_str(d->id)) |
148 | intel_uncore_forcewake_domain_to_str(d->id))__drm_err("%s: timed out waiting for forcewake ack to clear.\n" , intel_uncore_forcewake_domain_to_str(d->id)); |
149 | add_taint_for_CI(d->uncore->i915, TAINT_WARN1); /* CI now unreliable */ |
150 | } |
151 | } |
152 | |
153 | enum ack_type { |
154 | ACK_CLEAR = 0, |
155 | ACK_SET |
156 | }; |
157 | |
158 | static int |
159 | fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d, |
160 | const enum ack_type type) |
161 | { |
162 | const u32 ack_bit = FORCEWAKE_KERNEL(1UL << (0)); |
163 | const u32 value = type == ACK_SET ? ack_bit : 0; |
164 | unsigned int pass; |
165 | bool_Bool ack_detected; |
166 | |
167 | /* |
168 | * There is a possibility of driver's wake request colliding |
169 | * with hardware's own wake requests and that can cause |
170 | * hardware to not deliver the driver's ack message. |
171 | * |
172 | * Use a fallback bit toggle to kick the gpu state machine |
173 | * in the hope that the original ack will be delivered along with |
174 | * the fallback ack. |
175 | * |
176 | * This workaround is described in HSDES #1604254524 and it's known as: |
177 | * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl |
178 | * although the name is a bit misleading. |
179 | */ |
180 | |
181 | pass = 1; |
182 | do { |
183 | wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK(1UL << (15))); |
184 | |
185 | fw_set(d, FORCEWAKE_KERNEL_FALLBACK)iowrite32(({ typeof(((1UL << (15)))) _a = (((1UL << (15)))); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a) && __builtin_constant_p(_a)) do { } while (0); ( (_a) << 16 | (_a)); }); }), (d)->reg_set); |
186 | /* Give gt some time to relax before the polling frenzy */ |
187 | udelay(10 * pass); |
188 | wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK(1UL << (15))); |
189 | |
190 | ack_detected = (fw_ack(d)ioread32((d)->reg_ack) & ack_bit) == value; |
191 | |
192 | fw_clear(d, FORCEWAKE_KERNEL_FALLBACK)iowrite32((({ if (__builtin_constant_p((((1UL << (15))) ))) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p((((1UL << (15))))) && __builtin_constant_p(0)) do { } while (0); (((((1UL << (15))))) << 16 | (0)); })), (d)->reg_set); |
193 | } while (!ack_detected && pass++ < 10); |
194 | |
195 | DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",__drm_dbg(DRM_UT_DRIVER, "%s had to use fallback to %s ack, 0x%x (passes %u)\n" , intel_uncore_forcewake_domain_to_str(d->id), type == ACK_SET ? "set" : "clear", ioread32((d)->reg_ack), pass) |
196 | intel_uncore_forcewake_domain_to_str(d->id),__drm_dbg(DRM_UT_DRIVER, "%s had to use fallback to %s ack, 0x%x (passes %u)\n" , intel_uncore_forcewake_domain_to_str(d->id), type == ACK_SET ? "set" : "clear", ioread32((d)->reg_ack), pass) |
197 | type == ACK_SET ? "set" : "clear",__drm_dbg(DRM_UT_DRIVER, "%s had to use fallback to %s ack, 0x%x (passes %u)\n" , intel_uncore_forcewake_domain_to_str(d->id), type == ACK_SET ? "set" : "clear", ioread32((d)->reg_ack), pass) |
198 | fw_ack(d),__drm_dbg(DRM_UT_DRIVER, "%s had to use fallback to %s ack, 0x%x (passes %u)\n" , intel_uncore_forcewake_domain_to_str(d->id), type == ACK_SET ? "set" : "clear", ioread32((d)->reg_ack), pass) |
199 | pass)__drm_dbg(DRM_UT_DRIVER, "%s had to use fallback to %s ack, 0x%x (passes %u)\n" , intel_uncore_forcewake_domain_to_str(d->id), type == ACK_SET ? "set" : "clear", ioread32((d)->reg_ack), pass); |
200 | |
201 | return ack_detected ? 0 : -ETIMEDOUT60; |
202 | } |
203 | |
204 | static inline void |
205 | fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d) |
206 | { |
207 | if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL))__builtin_expect(!!(!wait_ack_clear(d, (1UL << (0)))), 1 )) |
208 | return; |
209 | |
210 | if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR)) |
211 | fw_domain_wait_ack_clear(d); |
212 | } |
213 | |
214 | static inline void |
215 | fw_domain_get(const struct intel_uncore_forcewake_domain *d) |
216 | { |
217 | fw_set(d, FORCEWAKE_KERNEL)iowrite32(({ typeof(((1UL << (0)))) _a = (((1UL << (0)))); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a) && __builtin_constant_p(_a)) do { } while (0); ( (_a) << 16 | (_a)); }); }), (d)->reg_set); |
218 | } |
219 | |
220 | static inline void |
221 | fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d) |
222 | { |
223 | if (wait_ack_set(d, FORCEWAKE_KERNEL(1UL << (0)))) { |
224 | DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",__drm_err("%s: timed out waiting for forcewake ack request.\n" , intel_uncore_forcewake_domain_to_str(d->id)) |
225 | intel_uncore_forcewake_domain_to_str(d->id))__drm_err("%s: timed out waiting for forcewake ack request.\n" , intel_uncore_forcewake_domain_to_str(d->id)); |
226 | add_taint_for_CI(d->uncore->i915, TAINT_WARN1); /* CI now unreliable */ |
227 | } |
228 | } |
229 | |
230 | static inline void |
231 | fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d) |
232 | { |
233 | if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL))__builtin_expect(!!(!wait_ack_set(d, (1UL << (0)))), 1)) |
234 | return; |
235 | |
236 | if (fw_domain_wait_ack_with_fallback(d, ACK_SET)) |
237 | fw_domain_wait_ack_set(d); |
238 | } |
239 | |
240 | static inline void |
241 | fw_domain_put(const struct intel_uncore_forcewake_domain *d) |
242 | { |
243 | fw_clear(d, FORCEWAKE_KERNEL)iowrite32((({ if (__builtin_constant_p((((1UL << (0)))) )) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p((((1UL << (0))))) && __builtin_constant_p(0)) do { } while (0); (((((1UL << (0))))) << 16 | (0)); })), (d)->reg_set); |
244 | } |
245 | |
246 | static void |
247 | fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains) |
248 | { |
249 | struct intel_uncore_forcewake_domain *d; |
250 | unsigned int tmp; |
251 | |
252 | GEM_BUG_ON(fw_domains & ~uncore->fw_domains)((void)0); |
253 | |
254 | for_each_fw_domain_masked(d, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(d = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else { |
255 | fw_domain_wait_ack_clear(d); |
256 | fw_domain_get(d); |
257 | } |
258 | |
259 | for_each_fw_domain_masked(d, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(d = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else |
260 | fw_domain_wait_ack_set(d); |
261 | |
262 | uncore->fw_domains_active |= fw_domains; |
263 | } |
264 | |
265 | static void |
266 | fw_domains_get_with_fallback(struct intel_uncore *uncore, |
267 | enum forcewake_domains fw_domains) |
268 | { |
269 | struct intel_uncore_forcewake_domain *d; |
270 | unsigned int tmp; |
271 | |
272 | GEM_BUG_ON(fw_domains & ~uncore->fw_domains)((void)0); |
273 | |
274 | for_each_fw_domain_masked(d, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(d = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else { |
275 | fw_domain_wait_ack_clear_fallback(d); |
276 | fw_domain_get(d); |
277 | } |
278 | |
279 | for_each_fw_domain_masked(d, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(d = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else |
280 | fw_domain_wait_ack_set_fallback(d); |
281 | |
282 | uncore->fw_domains_active |= fw_domains; |
283 | } |
284 | |
285 | static void |
286 | fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains) |
287 | { |
288 | struct intel_uncore_forcewake_domain *d; |
289 | unsigned int tmp; |
290 | |
291 | GEM_BUG_ON(fw_domains & ~uncore->fw_domains)((void)0); |
292 | |
293 | for_each_fw_domain_masked(d, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(d = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else |
294 | fw_domain_put(d); |
295 | |
296 | uncore->fw_domains_active &= ~fw_domains; |
297 | } |
298 | |
299 | static void |
300 | fw_domains_reset(struct intel_uncore *uncore, |
301 | enum forcewake_domains fw_domains) |
302 | { |
303 | struct intel_uncore_forcewake_domain *d; |
304 | unsigned int tmp; |
305 | |
306 | if (!fw_domains) |
307 | return; |
308 | |
309 | GEM_BUG_ON(fw_domains & ~uncore->fw_domains)((void)0); |
310 | |
311 | for_each_fw_domain_masked(d, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(d = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else |
312 | fw_domain_reset(d); |
313 | } |
314 | |
315 | static inline u32 gt_thread_status(struct intel_uncore *uncore) |
316 | { |
317 | u32 val; |
318 | |
319 | val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG((const i915_reg_t){ .reg = (0x13805c) })); |
320 | val &= GEN6_GT_THREAD_STATUS_CORE_MASK0x7; |
321 | |
322 | return val; |
323 | } |
324 | |
325 | static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore) |
326 | { |
327 | /* |
328 | * w/a for a sporadic read returning 0 by waiting for the GT |
329 | * thread to wake up. |
330 | */ |
331 | drm_WARN_ONCE(&uncore->i915->drm,({ static int __warned; int __ret = !!(({ extern char _ctassert [(!(!__builtin_constant_p(5000))) ? 1 : -1 ] __attribute__((__unused__ )); extern char _ctassert[(!((5000) > 50000)) ? 1 : -1 ] __attribute__ ((__unused__)); ({ int cpu, ret, timeout = ((5000)) * 1000; u64 base; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((gt_thread_status(uncore ) == 0)) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { ; if (__builtin_expect (!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); })); if (__ret && !__warned) { printf("%s %s: " "GT thread status wait timed out\n" , dev_driver_string((&uncore->i915->drm)->dev), "" ); __warned = 1; } __builtin_expect(!!(__ret), 0); }) |
332 | wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),({ static int __warned; int __ret = !!(({ extern char _ctassert [(!(!__builtin_constant_p(5000))) ? 1 : -1 ] __attribute__((__unused__ )); extern char _ctassert[(!((5000) > 50000)) ? 1 : -1 ] __attribute__ ((__unused__)); ({ int cpu, ret, timeout = ((5000)) * 1000; u64 base; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((gt_thread_status(uncore ) == 0)) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { ; if (__builtin_expect (!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); })); if (__ret && !__warned) { printf("%s %s: " "GT thread status wait timed out\n" , dev_driver_string((&uncore->i915->drm)->dev), "" ); __warned = 1; } __builtin_expect(!!(__ret), 0); }) |
333 | "GT thread status wait timed out\n")({ static int __warned; int __ret = !!(({ extern char _ctassert [(!(!__builtin_constant_p(5000))) ? 1 : -1 ] __attribute__((__unused__ )); extern char _ctassert[(!((5000) > 50000)) ? 1 : -1 ] __attribute__ ((__unused__)); ({ int cpu, ret, timeout = ((5000)) * 1000; u64 base; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((gt_thread_status(uncore ) == 0)) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { ; if (__builtin_expect (!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); })); if (__ret && !__warned) { printf("%s %s: " "GT thread status wait timed out\n" , dev_driver_string((&uncore->i915->drm)->dev), "" ); __warned = 1; } __builtin_expect(!!(__ret), 0); }); |
334 | } |
335 | |
336 | static void fw_domains_get_with_thread_status(struct intel_uncore *uncore, |
337 | enum forcewake_domains fw_domains) |
338 | { |
339 | fw_domains_get(uncore, fw_domains); |
340 | |
341 | /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */ |
342 | __gen6_gt_wait_for_thread_c0(uncore); |
343 | } |
344 | |
345 | static inline u32 fifo_free_entries(struct intel_uncore *uncore) |
346 | { |
347 | u32 count = __raw_uncore_read32(uncore, GTFIFOCTL((const i915_reg_t){ .reg = (0x120008) })); |
348 | |
349 | return count & GT_FIFO_FREE_ENTRIES_MASK0x7f; |
350 | } |
351 | |
352 | static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore) |
353 | { |
354 | u32 n; |
355 | |
356 | /* On VLV, FIFO will be shared by both SW and HW. |
357 | * So, we need to read the FREE_ENTRIES everytime */ |
358 | if (IS_VALLEYVIEW(uncore->i915)IS_PLATFORM(uncore->i915, INTEL_VALLEYVIEW)) |
359 | n = fifo_free_entries(uncore); |
360 | else |
361 | n = uncore->fifo_count; |
362 | |
363 | if (n <= GT_FIFO_NUM_RESERVED_ENTRIES20) { |
364 | if (wait_for_atomic((n = fifo_free_entries(uncore)) >({ extern char _ctassert[(!(!__builtin_constant_p((10) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((10) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((10) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((((n = fifo_free_entries (uncore)) > 20))) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { ; if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci; asm volatile ("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }) |
365 | GT_FIFO_NUM_RESERVED_ENTRIES,({ extern char _ctassert[(!(!__builtin_constant_p((10) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((10) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((10) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((((n = fifo_free_entries (uncore)) > 20))) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { ; if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci; asm volatile ("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }) |
366 | GT_FIFO_TIMEOUT_MS)({ extern char _ctassert[(!(!__builtin_constant_p((10) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((10) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((10) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { ; cpu = (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) ; __asm volatile("" : : : "memory"); if ((((n = fifo_free_entries (uncore)) > 20))) { ret = 0; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { ; if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci; asm volatile ("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); })) { |
367 | drm_dbg(&uncore->i915->drm,drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "GT_FIFO timeout, entries: %u\n", n) |
368 | "GT_FIFO timeout, entries: %u\n", n)drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "GT_FIFO timeout, entries: %u\n", n); |
369 | return; |
370 | } |
371 | } |
372 | |
373 | uncore->fifo_count = n - 1; |
374 | } |
375 | |
376 | #ifdef __linux__ |
377 | |
378 | static enum hrtimer_restart |
379 | intel_uncore_fw_release_timer(struct hrtimer *timer) |
380 | { |
381 | struct intel_uncore_forcewake_domain *domain = |
382 | container_of(timer, struct intel_uncore_forcewake_domain, timer)({ const __typeof( ((struct intel_uncore_forcewake_domain *)0 )->timer ) *__mptr = (timer); (struct intel_uncore_forcewake_domain *)( (char *)__mptr - __builtin_offsetof(struct intel_uncore_forcewake_domain , timer) );}); |
383 | struct intel_uncore *uncore = domain->uncore; |
384 | unsigned long irqflags; |
385 | |
386 | assert_rpm_device_not_suspended(uncore->rpm); |
387 | |
388 | if (xchg(&domain->active, false)__sync_lock_test_and_set(&domain->active, 0)) |
389 | return HRTIMER_RESTART; |
390 | |
391 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); |
392 | |
393 | uncore->fw_domains_timer &= ~domain->mask; |
394 | |
395 | GEM_BUG_ON(!domain->wake_count)((void)0); |
396 | if (--domain->wake_count == 0) |
397 | uncore->funcs.force_wake_put(uncore, domain->mask); |
398 | |
399 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); |
400 | |
401 | return HRTIMER_NORESTART; |
402 | } |
403 | |
404 | #else |
405 | |
406 | void |
407 | intel_uncore_fw_release_timer(void *arg) |
408 | { |
409 | struct intel_uncore_forcewake_domain *domain = arg; |
410 | struct intel_uncore *uncore = domain->uncore; |
411 | unsigned long irqflags; |
412 | |
413 | assert_rpm_device_not_suspended(uncore->rpm); |
414 | |
415 | if (xchg(&domain->active, false)__sync_lock_test_and_set(&domain->active, 0)) |
416 | return; |
417 | |
418 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); |
419 | |
420 | uncore->fw_domains_timer &= ~domain->mask; |
421 | |
422 | GEM_BUG_ON(!domain->wake_count)((void)0); |
423 | if (--domain->wake_count == 0) |
424 | uncore->funcs.force_wake_put(uncore, domain->mask); |
425 | |
426 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); |
427 | } |
428 | |
429 | #endif |
430 | |
431 | /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */ |
432 | static unsigned int |
433 | intel_uncore_forcewake_reset(struct intel_uncore *uncore) |
434 | { |
435 | unsigned long irqflags; |
436 | struct intel_uncore_forcewake_domain *domain; |
437 | int retry_count = 100; |
438 | enum forcewake_domains fw, active_domains; |
439 | |
440 | iosf_mbi_assert_punit_acquired(); |
441 | |
442 | /* Hold uncore.lock across reset to prevent any register access |
443 | * with forcewake not set correctly. Wait until all pending |
444 | * timers are run before holding. |
445 | */ |
446 | while (1) { |
447 | unsigned int tmp; |
448 | |
449 | active_domains = 0; |
450 | |
451 | for_each_fw_domain(domain, uncore, tmp)for (tmp = ((uncore)->fw_domains); tmp ;) if (!(domain = ( uncore)->fw_domain[({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx; })])) {} else { |
452 | smp_store_mb(domain->active, false)do { domain->active = 0; do { __asm volatile("mfence" ::: "memory" ); } while (0); } while (0); |
453 | if (hrtimer_cancel(&domain->timer)timeout_del(&domain->timer) == 0) |
454 | continue; |
455 | |
456 | intel_uncore_fw_release_timer(&domain->timer); |
457 | } |
458 | |
459 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); |
460 | |
461 | for_each_fw_domain(domain, uncore, tmp)for (tmp = ((uncore)->fw_domains); tmp ;) if (!(domain = ( uncore)->fw_domain[({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx; })])) {} else { |
462 | if (hrtimer_active(&domain->timer)((&domain->timer)->to_flags & 0x02)) |
463 | active_domains |= domain->mask; |
464 | } |
465 | |
466 | if (active_domains == 0) |
467 | break; |
468 | |
469 | if (--retry_count == 0) { |
470 | drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Timed out waiting for forcewake timers to finish\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__); |
471 | break; |
472 | } |
473 | |
474 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); |
475 | cond_resched()do { if (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_schedstate.spc_schedflags & 0x0002) yield (); } while (0); |
476 | } |
477 | |
478 | drm_WARN_ON(&uncore->i915->drm, active_domains)({ int __ret = !!((active_domains)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&uncore->i915->drm))-> dev), "", "drm_WARN_ON(" "active_domains" ")"); __builtin_expect (!!(__ret), 0); }); |
479 | |
480 | fw = uncore->fw_domains_active; |
481 | if (fw) |
482 | uncore->funcs.force_wake_put(uncore, fw); |
483 | |
484 | fw_domains_reset(uncore, uncore->fw_domains); |
485 | assert_forcewakes_inactive(uncore); |
486 | |
487 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); |
488 | |
489 | return fw; /* track the lost user forcewake domains */ |
490 | } |
491 | |
492 | static bool_Bool |
493 | fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore) |
494 | { |
495 | u32 dbg; |
496 | |
497 | dbg = __raw_uncore_read32(uncore, FPGA_DBG((const i915_reg_t){ .reg = (0x42300) })); |
498 | if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM))__builtin_expect(!!(!(dbg & (1 << 31))), 1)) |
499 | return false0; |
500 | |
501 | __raw_uncore_write32(uncore, FPGA_DBG((const i915_reg_t){ .reg = (0x42300) }), FPGA_DBG_RM_NOCLAIM(1 << 31)); |
502 | |
503 | return true1; |
504 | } |
505 | |
506 | static bool_Bool |
507 | vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore) |
508 | { |
509 | u32 cer; |
510 | |
511 | cer = __raw_uncore_read32(uncore, CLAIM_ER((const i915_reg_t){ .reg = (0x180000 + 0x2028) })); |
512 | if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK)))__builtin_expect(!!(!(cer & ((1 << 16) | 0xffff))), 1)) |
513 | return false0; |
514 | |
515 | __raw_uncore_write32(uncore, CLAIM_ER((const i915_reg_t){ .reg = (0x180000 + 0x2028) }), CLAIM_ER_CLR(1 << 31)); |
516 | |
517 | return true1; |
518 | } |
519 | |
520 | static bool_Bool |
521 | gen6_check_for_fifo_debug(struct intel_uncore *uncore) |
522 | { |
523 | u32 fifodbg; |
524 | |
525 | fifodbg = __raw_uncore_read32(uncore, GTFIFODBG((const i915_reg_t){ .reg = (0x120000) })); |
526 | |
527 | if (unlikely(fifodbg)__builtin_expect(!!(fifodbg), 0)) { |
528 | drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg)drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "GTFIFODBG = 0x08%x\n", fifodbg); |
529 | __raw_uncore_write32(uncore, GTFIFODBG((const i915_reg_t){ .reg = (0x120000) }), fifodbg); |
530 | } |
531 | |
532 | return fifodbg; |
533 | } |
534 | |
535 | static bool_Bool |
536 | check_for_unclaimed_mmio(struct intel_uncore *uncore) |
537 | { |
538 | bool_Bool ret = false0; |
539 | |
540 | lockdep_assert_held(&uncore->debug->lock)do { (void)(&uncore->debug->lock); } while(0); |
541 | |
542 | if (uncore->debug->suspend_count) |
543 | return false0; |
544 | |
545 | if (intel_uncore_has_fpga_dbg_unclaimed(uncore)) |
546 | ret |= fpga_check_for_unclaimed_mmio(uncore); |
547 | |
548 | if (intel_uncore_has_dbg_unclaimed(uncore)) |
549 | ret |= vlv_check_for_unclaimed_mmio(uncore); |
550 | |
551 | if (intel_uncore_has_fifo(uncore)) |
552 | ret |= gen6_check_for_fifo_debug(uncore); |
553 | |
554 | return ret; |
555 | } |
556 | |
557 | static void forcewake_early_sanitize(struct intel_uncore *uncore, |
558 | unsigned int restore_forcewake) |
559 | { |
560 | GEM_BUG_ON(!intel_uncore_has_forcewake(uncore))((void)0); |
561 | |
562 | /* WaDisableShadowRegForCpd:chv */ |
563 | if (IS_CHERRYVIEW(uncore->i915)IS_PLATFORM(uncore->i915, INTEL_CHERRYVIEW)) { |
564 | __raw_uncore_write32(uncore, GTFIFOCTL((const i915_reg_t){ .reg = (0x120008) }), |
565 | __raw_uncore_read32(uncore, GTFIFOCTL((const i915_reg_t){ .reg = (0x120008) })) | |
566 | GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL(1 << 12) | |
567 | GT_FIFO_CTL_RC6_POLICY_STALL(1 << 11)); |
568 | } |
569 | |
570 | iosf_mbi_punit_acquire(); |
571 | intel_uncore_forcewake_reset(uncore); |
572 | if (restore_forcewake) { |
573 | spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock); |
574 | uncore->funcs.force_wake_get(uncore, restore_forcewake); |
575 | |
576 | if (intel_uncore_has_fifo(uncore)) |
577 | uncore->fifo_count = fifo_free_entries(uncore); |
578 | spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock); |
579 | } |
580 | iosf_mbi_punit_release(); |
581 | } |
582 | |
583 | void intel_uncore_suspend(struct intel_uncore *uncore) |
584 | { |
585 | if (!intel_uncore_has_forcewake(uncore)) |
586 | return; |
587 | |
588 | iosf_mbi_punit_acquire(); |
589 | iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(0 |
590 | &uncore->pmic_bus_access_nb)0; |
591 | uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore); |
592 | iosf_mbi_punit_release(); |
593 | } |
594 | |
595 | void intel_uncore_resume_early(struct intel_uncore *uncore) |
596 | { |
597 | unsigned int restore_forcewake; |
598 | |
599 | if (intel_uncore_unclaimed_mmio(uncore)) |
600 | drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n")drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "unclaimed mmio detected on resume, clearing\n"); |
601 | |
602 | if (!intel_uncore_has_forcewake(uncore)) |
603 | return; |
604 | |
605 | restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved)({ typeof(*&uncore->fw_domains_saved) __T = *(&uncore ->fw_domains_saved); *(&uncore->fw_domains_saved) = (typeof(*&uncore->fw_domains_saved))0; __T; }); |
606 | forcewake_early_sanitize(uncore, restore_forcewake); |
607 | |
608 | iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb)0; |
609 | } |
610 | |
611 | void intel_uncore_runtime_resume(struct intel_uncore *uncore) |
612 | { |
613 | if (!intel_uncore_has_forcewake(uncore)) |
614 | return; |
615 | |
616 | iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb)0; |
617 | } |
618 | |
619 | static void __intel_uncore_forcewake_get(struct intel_uncore *uncore, |
620 | enum forcewake_domains fw_domains) |
621 | { |
622 | struct intel_uncore_forcewake_domain *domain; |
623 | unsigned int tmp; |
624 | |
625 | fw_domains &= uncore->fw_domains; |
626 | |
627 | for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(domain = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else { |
628 | if (domain->wake_count++) { |
629 | fw_domains &= ~domain->mask; |
630 | domain->active = true1; |
631 | } |
632 | } |
633 | |
634 | if (fw_domains) |
635 | uncore->funcs.force_wake_get(uncore, fw_domains); |
636 | } |
637 | |
638 | /** |
639 | * intel_uncore_forcewake_get - grab forcewake domain references |
640 | * @uncore: the intel_uncore structure |
641 | * @fw_domains: forcewake domains to get reference on |
642 | * |
643 | * This function can be used get GT's forcewake domain references. |
644 | * Normal register access will handle the forcewake domains automatically. |
645 | * However if some sequence requires the GT to not power down a particular |
646 | * forcewake domains this function should be called at the beginning of the |
647 | * sequence. And subsequently the reference should be dropped by symmetric |
648 | * call to intel_unforce_forcewake_put(). Usually caller wants all the domains |
649 | * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL. |
650 | */ |
651 | void intel_uncore_forcewake_get(struct intel_uncore *uncore, |
652 | enum forcewake_domains fw_domains) |
653 | { |
654 | unsigned long irqflags; |
655 | |
656 | if (!uncore->funcs.force_wake_get) |
657 | return; |
658 | |
659 | assert_rpm_wakelock_held(uncore->rpm); |
660 | |
661 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); |
662 | __intel_uncore_forcewake_get(uncore, fw_domains); |
663 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); |
664 | } |
665 | |
666 | /** |
667 | * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace |
668 | * @uncore: the intel_uncore structure |
669 | * |
670 | * This function is a wrapper around intel_uncore_forcewake_get() to acquire |
671 | * the GT powerwell and in the process disable our debugging for the |
672 | * duration of userspace's bypass. |
673 | */ |
674 | void intel_uncore_forcewake_user_get(struct intel_uncore *uncore) |
675 | { |
676 | spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock); |
677 | if (!uncore->user_forcewake_count++) { |
678 | intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL); |
679 | spin_lock(&uncore->debug->lock)mtx_enter(&uncore->debug->lock); |
680 | mmio_debug_suspend(uncore->debug); |
681 | spin_unlock(&uncore->debug->lock)mtx_leave(&uncore->debug->lock); |
682 | } |
683 | spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock); |
684 | } |
685 | |
686 | /** |
687 | * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace |
688 | * @uncore: the intel_uncore structure |
689 | * |
690 | * This function complements intel_uncore_forcewake_user_get() and releases |
691 | * the GT powerwell taken on behalf of the userspace bypass. |
692 | */ |
693 | void intel_uncore_forcewake_user_put(struct intel_uncore *uncore) |
694 | { |
695 | spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock); |
696 | if (!--uncore->user_forcewake_count) { |
697 | spin_lock(&uncore->debug->lock)mtx_enter(&uncore->debug->lock); |
698 | mmio_debug_resume(uncore->debug); |
699 | |
700 | if (check_for_unclaimed_mmio(uncore)) |
701 | drm_info(&uncore->i915->drm,do { } while(0) |
702 | "Invalid mmio detected during user access\n")do { } while(0); |
703 | spin_unlock(&uncore->debug->lock)mtx_leave(&uncore->debug->lock); |
704 | |
705 | intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL); |
706 | } |
707 | spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock); |
708 | } |
709 | |
710 | /** |
711 | * intel_uncore_forcewake_get__locked - grab forcewake domain references |
712 | * @uncore: the intel_uncore structure |
713 | * @fw_domains: forcewake domains to get reference on |
714 | * |
715 | * See intel_uncore_forcewake_get(). This variant places the onus |
716 | * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. |
717 | */ |
718 | void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore, |
719 | enum forcewake_domains fw_domains) |
720 | { |
721 | lockdep_assert_held(&uncore->lock)do { (void)(&uncore->lock); } while(0); |
722 | |
723 | if (!uncore->funcs.force_wake_get) |
724 | return; |
725 | |
726 | __intel_uncore_forcewake_get(uncore, fw_domains); |
727 | } |
728 | |
729 | static void __intel_uncore_forcewake_put(struct intel_uncore *uncore, |
730 | enum forcewake_domains fw_domains) |
731 | { |
732 | struct intel_uncore_forcewake_domain *domain; |
733 | unsigned int tmp; |
734 | |
735 | fw_domains &= uncore->fw_domains; |
736 | |
737 | for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(domain = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else { |
738 | GEM_BUG_ON(!domain->wake_count)((void)0); |
739 | |
740 | if (--domain->wake_count) { |
741 | domain->active = true1; |
742 | continue; |
743 | } |
744 | |
745 | uncore->funcs.force_wake_put(uncore, domain->mask); |
746 | } |
747 | } |
748 | |
749 | /** |
750 | * intel_uncore_forcewake_put - release a forcewake domain reference |
751 | * @uncore: the intel_uncore structure |
752 | * @fw_domains: forcewake domains to put references |
753 | * |
754 | * This function drops the device-level forcewakes for specified |
755 | * domains obtained by intel_uncore_forcewake_get(). |
756 | */ |
757 | void intel_uncore_forcewake_put(struct intel_uncore *uncore, |
758 | enum forcewake_domains fw_domains) |
759 | { |
760 | unsigned long irqflags; |
761 | |
762 | if (!uncore->funcs.force_wake_put) |
763 | return; |
764 | |
765 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); |
766 | __intel_uncore_forcewake_put(uncore, fw_domains); |
767 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); |
768 | } |
769 | |
770 | /** |
771 | * intel_uncore_forcewake_flush - flush the delayed release |
772 | * @uncore: the intel_uncore structure |
773 | * @fw_domains: forcewake domains to flush |
774 | */ |
775 | void intel_uncore_forcewake_flush(struct intel_uncore *uncore, |
776 | enum forcewake_domains fw_domains) |
777 | { |
778 | struct intel_uncore_forcewake_domain *domain; |
779 | unsigned int tmp; |
780 | |
781 | if (!uncore->funcs.force_wake_put) |
782 | return; |
783 | |
784 | fw_domains &= uncore->fw_domains; |
785 | for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(domain = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else { |
786 | WRITE_ONCE(domain->active, false)({ typeof(domain->active) __tmp = (0); *(volatile typeof(domain ->active) *)&(domain->active) = __tmp; __tmp; }); |
787 | if (hrtimer_cancel(&domain->timer)timeout_del(&domain->timer)) |
788 | intel_uncore_fw_release_timer(&domain->timer); |
789 | } |
790 | } |
791 | |
792 | /** |
793 | * intel_uncore_forcewake_put__locked - grab forcewake domain references |
794 | * @uncore: the intel_uncore structure |
795 | * @fw_domains: forcewake domains to get reference on |
796 | * |
797 | * See intel_uncore_forcewake_put(). This variant places the onus |
798 | * on the caller to explicitly handle the dev_priv->uncore.lock spinlock. |
799 | */ |
800 | void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore, |
801 | enum forcewake_domains fw_domains) |
802 | { |
803 | lockdep_assert_held(&uncore->lock)do { (void)(&uncore->lock); } while(0); |
804 | |
805 | if (!uncore->funcs.force_wake_put) |
806 | return; |
807 | |
808 | __intel_uncore_forcewake_put(uncore, fw_domains); |
809 | } |
810 | |
811 | void assert_forcewakes_inactive(struct intel_uncore *uncore) |
812 | { |
813 | if (!uncore->funcs.force_wake_get) |
814 | return; |
815 | |
816 | drm_WARN(&uncore->i915->drm, uncore->fw_domains_active,({ int __ret = !!(uncore->fw_domains_active); if (__ret) printf ("%s %s: " "Expected all fw_domains to be inactive, but %08x are still on\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , uncore->fw_domains_active); __builtin_expect(!!(__ret), 0 ); }) |
817 | "Expected all fw_domains to be inactive, but %08x are still on\n",({ int __ret = !!(uncore->fw_domains_active); if (__ret) printf ("%s %s: " "Expected all fw_domains to be inactive, but %08x are still on\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , uncore->fw_domains_active); __builtin_expect(!!(__ret), 0 ); }) |
818 | uncore->fw_domains_active)({ int __ret = !!(uncore->fw_domains_active); if (__ret) printf ("%s %s: " "Expected all fw_domains to be inactive, but %08x are still on\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , uncore->fw_domains_active); __builtin_expect(!!(__ret), 0 ); }); |
819 | } |
820 | |
821 | void assert_forcewakes_active(struct intel_uncore *uncore, |
822 | enum forcewake_domains fw_domains) |
823 | { |
824 | struct intel_uncore_forcewake_domain *domain; |
825 | unsigned int tmp; |
826 | |
827 | if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)0) |
828 | return; |
829 | |
830 | if (!uncore->funcs.force_wake_get) |
831 | return; |
832 | |
833 | spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock); |
834 | |
835 | assert_rpm_wakelock_held(uncore->rpm); |
836 | |
837 | fw_domains &= uncore->fw_domains; |
838 | drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active,({ int __ret = !!(fw_domains & ~uncore->fw_domains_active ); if (__ret) printf("%s %s: " "Expected %08x fw_domains to be active, but %08x are off\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , fw_domains, fw_domains & ~uncore->fw_domains_active) ; __builtin_expect(!!(__ret), 0); }) |
839 | "Expected %08x fw_domains to be active, but %08x are off\n",({ int __ret = !!(fw_domains & ~uncore->fw_domains_active ); if (__ret) printf("%s %s: " "Expected %08x fw_domains to be active, but %08x are off\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , fw_domains, fw_domains & ~uncore->fw_domains_active) ; __builtin_expect(!!(__ret), 0); }) |
840 | fw_domains, fw_domains & ~uncore->fw_domains_active)({ int __ret = !!(fw_domains & ~uncore->fw_domains_active ); if (__ret) printf("%s %s: " "Expected %08x fw_domains to be active, but %08x are off\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , fw_domains, fw_domains & ~uncore->fw_domains_active) ; __builtin_expect(!!(__ret), 0); }); |
841 | |
842 | /* |
843 | * Check that the caller has an explicit wakeref and we don't mistake |
844 | * it for the auto wakeref. |
845 | */ |
846 | for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(domain = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else { |
847 | unsigned int actual = READ_ONCE(domain->wake_count)({ typeof(domain->wake_count) __tmp = *(volatile typeof(domain ->wake_count) *)&(domain->wake_count); membar_datadep_consumer (); __tmp; }); |
848 | unsigned int expect = 1; |
849 | |
850 | if (uncore->fw_domains_timer & domain->mask) |
851 | expect++; /* pending automatic release */ |
852 | |
853 | if (drm_WARN(&uncore->i915->drm, actual < expect,({ int __ret = !!(actual < expect); if (__ret) printf("%s %s: " "Expected domain %d to be held awake by caller, count=%d\n", dev_driver_string((&uncore->i915->drm)->dev), "" , domain->id, actual); __builtin_expect(!!(__ret), 0); }) |
854 | "Expected domain %d to be held awake by caller, count=%d\n",({ int __ret = !!(actual < expect); if (__ret) printf("%s %s: " "Expected domain %d to be held awake by caller, count=%d\n", dev_driver_string((&uncore->i915->drm)->dev), "" , domain->id, actual); __builtin_expect(!!(__ret), 0); }) |
855 | domain->id, actual)({ int __ret = !!(actual < expect); if (__ret) printf("%s %s: " "Expected domain %d to be held awake by caller, count=%d\n", dev_driver_string((&uncore->i915->drm)->dev), "" , domain->id, actual); __builtin_expect(!!(__ret), 0); })) |
856 | break; |
857 | } |
858 | |
859 | spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock); |
860 | } |
861 | |
862 | /* We give fast paths for the really cool registers */ |
863 | #define NEEDS_FORCE_WAKE(reg)((reg) < 0x40000) ((reg) < 0x40000) |
864 | |
865 | #define __gen6_reg_read_fw_domains(uncore, offset)({ enum forcewake_domains __fwd; if (((offset) < 0x40000)) __fwd = FORCEWAKE_RENDER; else __fwd = 0; __fwd; }) \ |
866 | ({ \ |
867 | enum forcewake_domains __fwd; \ |
868 | if (NEEDS_FORCE_WAKE(offset)((offset) < 0x40000)) \ |
869 | __fwd = FORCEWAKE_RENDER; \ |
870 | else \ |
871 | __fwd = 0; \ |
872 | __fwd; \ |
873 | }) |
874 | |
875 | static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry) |
876 | { |
877 | if (offset < entry->start) |
878 | return -1; |
879 | else if (offset > entry->end) |
880 | return 1; |
881 | else |
882 | return 0; |
883 | } |
884 | |
885 | /* Copied and "macroized" from lib/bsearch.c */ |
886 | #define BSEARCH(key, base, num, cmp)({ unsigned int start__ = 0, end__ = (num); typeof(base) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (cmp)((key), ( base) + mid__); if (ret__ < 0) { end__ = mid__; } else if ( ret__ > 0) { start__ = mid__ + 1; } else { result__ = (base ) + mid__; break; } } result__; }) ({ \ |
887 | unsigned int start__ = 0, end__ = (num); \ |
888 | typeof(base) result__ = NULL((void *)0); \ |
889 | while (start__ < end__) { \ |
890 | unsigned int mid__ = start__ + (end__ - start__) / 2; \ |
891 | int ret__ = (cmp)((key), (base) + mid__); \ |
892 | if (ret__ < 0) { \ |
893 | end__ = mid__; \ |
894 | } else if (ret__ > 0) { \ |
895 | start__ = mid__ + 1; \ |
896 | } else { \ |
897 | result__ = (base) + mid__; \ |
898 | break; \ |
899 | } \ |
900 | } \ |
901 | result__; \ |
902 | }) |
903 | |
904 | static enum forcewake_domains |
905 | find_fw_domain(struct intel_uncore *uncore, u32 offset) |
906 | { |
907 | const struct intel_forcewake_range *entry; |
908 | |
909 | entry = BSEARCH(offset,({ unsigned int start__ = 0, end__ = (uncore->fw_domains_table_entries ); typeof(uncore->fw_domains_table) result__ = ((void *)0) ; while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (fw_range_cmp)((offset), ( uncore->fw_domains_table) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (uncore->fw_domains_table) + mid__; break; } } result__; }) |
910 | uncore->fw_domains_table,({ unsigned int start__ = 0, end__ = (uncore->fw_domains_table_entries ); typeof(uncore->fw_domains_table) result__ = ((void *)0) ; while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (fw_range_cmp)((offset), ( uncore->fw_domains_table) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (uncore->fw_domains_table) + mid__; break; } } result__; }) |
911 | uncore->fw_domains_table_entries,({ unsigned int start__ = 0, end__ = (uncore->fw_domains_table_entries ); typeof(uncore->fw_domains_table) result__ = ((void *)0) ; while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (fw_range_cmp)((offset), ( uncore->fw_domains_table) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (uncore->fw_domains_table) + mid__; break; } } result__; }) |
912 | fw_range_cmp)({ unsigned int start__ = 0, end__ = (uncore->fw_domains_table_entries ); typeof(uncore->fw_domains_table) result__ = ((void *)0) ; while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (fw_range_cmp)((offset), ( uncore->fw_domains_table) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (uncore->fw_domains_table) + mid__; break; } } result__; }); |
913 | |
914 | if (!entry) |
915 | return 0; |
916 | |
917 | /* |
918 | * The list of FW domains depends on the SKU in gen11+ so we |
919 | * can't determine it statically. We use FORCEWAKE_ALL and |
920 | * translate it here to the list of available domains. |
921 | */ |
922 | if (entry->domains == FORCEWAKE_ALL) |
923 | return uncore->fw_domains; |
924 | |
925 | drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains,({ int __ret = !!(entry->domains & ~uncore->fw_domains ); if (__ret) printf("%s %s: " "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , entry->domains & ~uncore->fw_domains, offset); __builtin_expect (!!(__ret), 0); }) |
926 | "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",({ int __ret = !!(entry->domains & ~uncore->fw_domains ); if (__ret) printf("%s %s: " "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , entry->domains & ~uncore->fw_domains, offset); __builtin_expect (!!(__ret), 0); }) |
927 | entry->domains & ~uncore->fw_domains, offset)({ int __ret = !!(entry->domains & ~uncore->fw_domains ); if (__ret) printf("%s %s: " "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , entry->domains & ~uncore->fw_domains, offset); __builtin_expect (!!(__ret), 0); }); |
928 | |
929 | return entry->domains; |
930 | } |
931 | |
932 | #define GEN_FW_RANGE(s, e, d){ .start = (s), .end = (e), .domains = (d) } \ |
933 | { .start = (s), .end = (e), .domains = (d) } |
934 | |
935 | /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ |
936 | static const struct intel_forcewake_range __vlv_fw_ranges[] = { |
937 | GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER){ .start = (0x2000), .end = (0x3fff), .domains = (FORCEWAKE_RENDER ) }, |
938 | GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER){ .start = (0x5000), .end = (0x7fff), .domains = (FORCEWAKE_RENDER ) }, |
939 | GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER){ .start = (0xb000), .end = (0x11fff), .domains = (FORCEWAKE_RENDER ) }, |
940 | GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA){ .start = (0x12000), .end = (0x13fff), .domains = (FORCEWAKE_MEDIA ) }, |
941 | GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA){ .start = (0x22000), .end = (0x23fff), .domains = (FORCEWAKE_MEDIA ) }, |
942 | GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER){ .start = (0x2e000), .end = (0x2ffff), .domains = (FORCEWAKE_RENDER ) }, |
943 | GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA){ .start = (0x30000), .end = (0x3ffff), .domains = (FORCEWAKE_MEDIA ) }, |
944 | }; |
945 | |
946 | #define __fwtable_reg_read_fw_domains(uncore, offset)({ enum forcewake_domains __fwd = 0; if ((((offset)) < 0x40000 )) __fwd = find_fw_domain(uncore, offset); __fwd; }) \ |
947 | ({ \ |
948 | enum forcewake_domains __fwd = 0; \ |
949 | if (NEEDS_FORCE_WAKE((offset))(((offset)) < 0x40000)) \ |
950 | __fwd = find_fw_domain(uncore, offset); \ |
951 | __fwd; \ |
952 | }) |
953 | |
954 | #define __gen11_fwtable_reg_read_fw_domains(uncore, offset)find_fw_domain(uncore, offset) \ |
955 | find_fw_domain(uncore, offset) |
956 | |
957 | #define __gen12_fwtable_reg_read_fw_domains(uncore, offset)find_fw_domain(uncore, offset) \ |
958 | find_fw_domain(uncore, offset) |
959 | |
960 | /* *Must* be sorted by offset! See intel_shadow_table_check(). */ |
961 | static const i915_reg_t gen8_shadowed_regs[] = { |
962 | RING_TAIL(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x30) }), /* 0x2000 (base) */ |
963 | GEN6_RPNSWREQ((const i915_reg_t){ .reg = (0xA008) }), /* 0xA008 */ |
964 | GEN6_RC_VIDEO_FREQ((const i915_reg_t){ .reg = (0xA00C) }), /* 0xA00C */ |
965 | RING_TAIL(GEN6_BSD_RING_BASE)((const i915_reg_t){ .reg = ((0x12000) + 0x30) }), /* 0x12000 (base) */ |
966 | RING_TAIL(VEBOX_RING_BASE)((const i915_reg_t){ .reg = ((0x1a000) + 0x30) }), /* 0x1a000 (base) */ |
967 | RING_TAIL(BLT_RING_BASE)((const i915_reg_t){ .reg = ((0x22000) + 0x30) }), /* 0x22000 (base) */ |
968 | /* TODO: Other registers are not yet used */ |
969 | }; |
970 | |
971 | static const i915_reg_t gen11_shadowed_regs[] = { |
972 | RING_TAIL(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x30) }), /* 0x2000 (base) */ |
973 | GEN6_RPNSWREQ((const i915_reg_t){ .reg = (0xA008) }), /* 0xA008 */ |
974 | GEN6_RC_VIDEO_FREQ((const i915_reg_t){ .reg = (0xA00C) }), /* 0xA00C */ |
975 | RING_TAIL(BLT_RING_BASE)((const i915_reg_t){ .reg = ((0x22000) + 0x30) }), /* 0x22000 (base) */ |
976 | RING_TAIL(GEN11_BSD_RING_BASE)((const i915_reg_t){ .reg = ((0x1c0000) + 0x30) }), /* 0x1C0000 (base) */ |
977 | RING_TAIL(GEN11_BSD2_RING_BASE)((const i915_reg_t){ .reg = ((0x1c4000) + 0x30) }), /* 0x1C4000 (base) */ |
978 | RING_TAIL(GEN11_VEBOX_RING_BASE)((const i915_reg_t){ .reg = ((0x1c8000) + 0x30) }), /* 0x1C8000 (base) */ |
979 | RING_TAIL(GEN11_BSD3_RING_BASE)((const i915_reg_t){ .reg = ((0x1d0000) + 0x30) }), /* 0x1D0000 (base) */ |
980 | RING_TAIL(GEN11_BSD4_RING_BASE)((const i915_reg_t){ .reg = ((0x1d4000) + 0x30) }), /* 0x1D4000 (base) */ |
981 | RING_TAIL(GEN11_VEBOX2_RING_BASE)((const i915_reg_t){ .reg = ((0x1d8000) + 0x30) }), /* 0x1D8000 (base) */ |
982 | /* TODO: Other registers are not yet used */ |
983 | }; |
984 | |
985 | static const i915_reg_t gen12_shadowed_regs[] = { |
986 | RING_TAIL(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x30) }), /* 0x2000 (base) */ |
987 | GEN6_RPNSWREQ((const i915_reg_t){ .reg = (0xA008) }), /* 0xA008 */ |
988 | GEN6_RC_VIDEO_FREQ((const i915_reg_t){ .reg = (0xA00C) }), /* 0xA00C */ |
989 | RING_TAIL(BLT_RING_BASE)((const i915_reg_t){ .reg = ((0x22000) + 0x30) }), /* 0x22000 (base) */ |
990 | RING_TAIL(GEN11_BSD_RING_BASE)((const i915_reg_t){ .reg = ((0x1c0000) + 0x30) }), /* 0x1C0000 (base) */ |
991 | RING_TAIL(GEN11_BSD2_RING_BASE)((const i915_reg_t){ .reg = ((0x1c4000) + 0x30) }), /* 0x1C4000 (base) */ |
992 | RING_TAIL(GEN11_VEBOX_RING_BASE)((const i915_reg_t){ .reg = ((0x1c8000) + 0x30) }), /* 0x1C8000 (base) */ |
993 | RING_TAIL(GEN11_BSD3_RING_BASE)((const i915_reg_t){ .reg = ((0x1d0000) + 0x30) }), /* 0x1D0000 (base) */ |
994 | RING_TAIL(GEN11_BSD4_RING_BASE)((const i915_reg_t){ .reg = ((0x1d4000) + 0x30) }), /* 0x1D4000 (base) */ |
995 | RING_TAIL(GEN11_VEBOX2_RING_BASE)((const i915_reg_t){ .reg = ((0x1d8000) + 0x30) }), /* 0x1D8000 (base) */ |
996 | /* TODO: Other registers are not yet used */ |
997 | }; |
998 | |
999 | static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) |
1000 | { |
1001 | u32 offset = i915_mmio_reg_offset(*reg); |
1002 | |
1003 | if (key < offset) |
1004 | return -1; |
1005 | else if (key > offset) |
1006 | return 1; |
1007 | else |
1008 | return 0; |
1009 | } |
1010 | |
1011 | #define __is_genX_shadowed(x)static _Bool is_genx_shadowed(u32 offset) { const i915_reg_t * regs = genx_shadowed_regs; return ({ unsigned int start__ = 0 , end__ = ((sizeof((genx_shadowed_regs)) / sizeof((genx_shadowed_regs )[0]))); typeof(regs) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2 ; int ret__ = (mmio_reg_cmp)((offset), (regs) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (regs) + mid__; break; } } result__ ; }); } \ |
1012 | static bool_Bool is_gen##x##_shadowed(u32 offset) \ |
1013 | { \ |
1014 | const i915_reg_t *regs = gen##x##_shadowed_regs; \ |
1015 | return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \({ unsigned int start__ = 0, end__ = ((sizeof((gen##x##_shadowed_regs )) / sizeof((gen##x##_shadowed_regs)[0]))); typeof(regs) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (mmio_reg_cmp )((offset), (regs) + mid__); if (ret__ < 0) { end__ = mid__ ; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (regs) + mid__; break; } } result__; }) |
1016 | mmio_reg_cmp)({ unsigned int start__ = 0, end__ = ((sizeof((gen##x##_shadowed_regs )) / sizeof((gen##x##_shadowed_regs)[0]))); typeof(regs) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2; int ret__ = (mmio_reg_cmp )((offset), (regs) + mid__); if (ret__ < 0) { end__ = mid__ ; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (regs) + mid__; break; } } result__; }); \ |
1017 | } |
1018 | |
1019 | __is_genX_shadowed(8)static _Bool is_gen8_shadowed(u32 offset) { const i915_reg_t * regs = gen8_shadowed_regs; return ({ unsigned int start__ = 0 , end__ = ((sizeof((gen8_shadowed_regs)) / sizeof((gen8_shadowed_regs )[0]))); typeof(regs) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2 ; int ret__ = (mmio_reg_cmp)((offset), (regs) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (regs) + mid__; break; } } result__ ; }); } |
1020 | __is_genX_shadowed(11)static _Bool is_gen11_shadowed(u32 offset) { const i915_reg_t *regs = gen11_shadowed_regs; return ({ unsigned int start__ = 0, end__ = ((sizeof((gen11_shadowed_regs)) / sizeof((gen11_shadowed_regs )[0]))); typeof(regs) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2 ; int ret__ = (mmio_reg_cmp)((offset), (regs) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (regs) + mid__; break; } } result__ ; }); } |
1021 | __is_genX_shadowed(12)static _Bool is_gen12_shadowed(u32 offset) { const i915_reg_t *regs = gen12_shadowed_regs; return ({ unsigned int start__ = 0, end__ = ((sizeof((gen12_shadowed_regs)) / sizeof((gen12_shadowed_regs )[0]))); typeof(regs) result__ = ((void *)0); while (start__ < end__) { unsigned int mid__ = start__ + (end__ - start__) / 2 ; int ret__ = (mmio_reg_cmp)((offset), (regs) + mid__); if (ret__ < 0) { end__ = mid__; } else if (ret__ > 0) { start__ = mid__ + 1; } else { result__ = (regs) + mid__; break; } } result__ ; }); } |
1022 | |
1023 | static enum forcewake_domains |
1024 | gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) |
1025 | { |
1026 | return FORCEWAKE_RENDER; |
1027 | } |
1028 | |
1029 | #define __gen8_reg_write_fw_domains(uncore, offset)({ enum forcewake_domains __fwd; if (((offset) < 0x40000) && !is_gen8_shadowed(offset)) __fwd = FORCEWAKE_RENDER; else __fwd = 0; __fwd; }) \ |
1030 | ({ \ |
1031 | enum forcewake_domains __fwd; \ |
1032 | if (NEEDS_FORCE_WAKE(offset)((offset) < 0x40000) && !is_gen8_shadowed(offset)) \ |
1033 | __fwd = FORCEWAKE_RENDER; \ |
1034 | else \ |
1035 | __fwd = 0; \ |
1036 | __fwd; \ |
1037 | }) |
1038 | |
1039 | /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ |
1040 | static const struct intel_forcewake_range __chv_fw_ranges[] = { |
1041 | GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER){ .start = (0x2000), .end = (0x3fff), .domains = (FORCEWAKE_RENDER ) }, |
1042 | GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA){ .start = (0x4000), .end = (0x4fff), .domains = (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) }, |
1043 | GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER){ .start = (0x5200), .end = (0x7fff), .domains = (FORCEWAKE_RENDER ) }, |
1044 | GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA){ .start = (0x8000), .end = (0x82ff), .domains = (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) }, |
1045 | GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER){ .start = (0x8300), .end = (0x84ff), .domains = (FORCEWAKE_RENDER ) }, |
1046 | GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA){ .start = (0x8500), .end = (0x85ff), .domains = (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) }, |
1047 | GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA){ .start = (0x8800), .end = (0x88ff), .domains = (FORCEWAKE_MEDIA ) }, |
1048 | GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA){ .start = (0x9000), .end = (0xafff), .domains = (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) }, |
1049 | GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER){ .start = (0xb000), .end = (0xb47f), .domains = (FORCEWAKE_RENDER ) }, |
1050 | GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA){ .start = (0xd000), .end = (0xd7ff), .domains = (FORCEWAKE_MEDIA ) }, |
1051 | GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER){ .start = (0xe000), .end = (0xe7ff), .domains = (FORCEWAKE_RENDER ) }, |
1052 | GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA){ .start = (0xf000), .end = (0xffff), .domains = (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) }, |
1053 | GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA){ .start = (0x12000), .end = (0x13fff), .domains = (FORCEWAKE_MEDIA ) }, |
1054 | GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA){ .start = (0x1a000), .end = (0x1bfff), .domains = (FORCEWAKE_MEDIA ) }, |
1055 | GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA){ .start = (0x1e800), .end = (0x1e9ff), .domains = (FORCEWAKE_MEDIA ) }, |
1056 | GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA){ .start = (0x30000), .end = (0x37fff), .domains = (FORCEWAKE_MEDIA ) }, |
1057 | }; |
1058 | |
1059 | #define __fwtable_reg_write_fw_domains(uncore, offset)({ enum forcewake_domains __fwd = 0; if ((((offset)) < 0x40000 ) && !is_gen8_shadowed(offset)) __fwd = find_fw_domain (uncore, offset); __fwd; }) \ |
1060 | ({ \ |
1061 | enum forcewake_domains __fwd = 0; \ |
1062 | if (NEEDS_FORCE_WAKE((offset))(((offset)) < 0x40000) && !is_gen8_shadowed(offset)) \ |
1063 | __fwd = find_fw_domain(uncore, offset); \ |
1064 | __fwd; \ |
1065 | }) |
1066 | |
1067 | #define __gen11_fwtable_reg_write_fw_domains(uncore, offset)({ enum forcewake_domains __fwd = 0; const u32 __offset = (offset ); if (!is_gen11_shadowed(__offset)) __fwd = find_fw_domain(uncore , __offset); __fwd; }) \ |
1068 | ({ \ |
1069 | enum forcewake_domains __fwd = 0; \ |
1070 | const u32 __offset = (offset); \ |
1071 | if (!is_gen11_shadowed(__offset)) \ |
1072 | __fwd = find_fw_domain(uncore, __offset); \ |
1073 | __fwd; \ |
1074 | }) |
1075 | |
1076 | #define __gen12_fwtable_reg_write_fw_domains(uncore, offset)({ enum forcewake_domains __fwd = 0; const u32 __offset = (offset ); if (!is_gen12_shadowed(__offset)) __fwd = find_fw_domain(uncore , __offset); __fwd; }) \ |
1077 | ({ \ |
1078 | enum forcewake_domains __fwd = 0; \ |
1079 | const u32 __offset = (offset); \ |
1080 | if (!is_gen12_shadowed(__offset)) \ |
1081 | __fwd = find_fw_domain(uncore, __offset); \ |
1082 | __fwd; \ |
1083 | }) |
1084 | |
1085 | /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ |
1086 | static const struct intel_forcewake_range __gen9_fw_ranges[] = { |
1087 | GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER){ .start = (0x0), .end = (0xaff), .domains = (FORCEWAKE_BLITTER ) }, |
1088 | GEN_FW_RANGE(0xb00, 0x1fff, 0){ .start = (0xb00), .end = (0x1fff), .domains = (0) }, /* uncore range */ |
1089 | GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER){ .start = (0x2000), .end = (0x26ff), .domains = (FORCEWAKE_RENDER ) }, |
1090 | GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER){ .start = (0x2700), .end = (0x2fff), .domains = (FORCEWAKE_BLITTER ) }, |
1091 | GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER){ .start = (0x3000), .end = (0x3fff), .domains = (FORCEWAKE_RENDER ) }, |
1092 | GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER){ .start = (0x4000), .end = (0x51ff), .domains = (FORCEWAKE_BLITTER ) }, |
1093 | GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER){ .start = (0x5200), .end = (0x7fff), .domains = (FORCEWAKE_RENDER ) }, |
1094 | GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER){ .start = (0x8000), .end = (0x812f), .domains = (FORCEWAKE_BLITTER ) }, |
1095 | GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA){ .start = (0x8130), .end = (0x813f), .domains = (FORCEWAKE_MEDIA ) }, |
1096 | GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER){ .start = (0x8140), .end = (0x815f), .domains = (FORCEWAKE_RENDER ) }, |
1097 | GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER){ .start = (0x8160), .end = (0x82ff), .domains = (FORCEWAKE_BLITTER ) }, |
1098 | GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER){ .start = (0x8300), .end = (0x84ff), .domains = (FORCEWAKE_RENDER ) }, |
1099 | GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER){ .start = (0x8500), .end = (0x87ff), .domains = (FORCEWAKE_BLITTER ) }, |
1100 | GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA){ .start = (0x8800), .end = (0x89ff), .domains = (FORCEWAKE_MEDIA ) }, |
1101 | GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER){ .start = (0x8a00), .end = (0x8bff), .domains = (FORCEWAKE_BLITTER ) }, |
1102 | GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER){ .start = (0x8c00), .end = (0x8cff), .domains = (FORCEWAKE_RENDER ) }, |
1103 | GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER){ .start = (0x8d00), .end = (0x93ff), .domains = (FORCEWAKE_BLITTER ) }, |
1104 | GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA){ .start = (0x9400), .end = (0x97ff), .domains = (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) }, |
1105 | GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER){ .start = (0x9800), .end = (0xafff), .domains = (FORCEWAKE_BLITTER ) }, |
1106 | GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER){ .start = (0xb000), .end = (0xb47f), .domains = (FORCEWAKE_RENDER ) }, |
1107 | GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER){ .start = (0xb480), .end = (0xcfff), .domains = (FORCEWAKE_BLITTER ) }, |
1108 | GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA){ .start = (0xd000), .end = (0xd7ff), .domains = (FORCEWAKE_MEDIA ) }, |
1109 | GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER){ .start = (0xd800), .end = (0xdfff), .domains = (FORCEWAKE_BLITTER ) }, |
1110 | GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER){ .start = (0xe000), .end = (0xe8ff), .domains = (FORCEWAKE_RENDER ) }, |
1111 | GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER){ .start = (0xe900), .end = (0x11fff), .domains = (FORCEWAKE_BLITTER ) }, |
1112 | GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA){ .start = (0x12000), .end = (0x13fff), .domains = (FORCEWAKE_MEDIA ) }, |
1113 | GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER){ .start = (0x14000), .end = (0x19fff), .domains = (FORCEWAKE_BLITTER ) }, |
1114 | GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA){ .start = (0x1a000), .end = (0x1e9ff), .domains = (FORCEWAKE_MEDIA ) }, |
1115 | GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER){ .start = (0x1ea00), .end = (0x243ff), .domains = (FORCEWAKE_BLITTER ) }, |
1116 | GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER){ .start = (0x24400), .end = (0x247ff), .domains = (FORCEWAKE_RENDER ) }, |
1117 | GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER){ .start = (0x24800), .end = (0x2ffff), .domains = (FORCEWAKE_BLITTER ) }, |
1118 | GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA){ .start = (0x30000), .end = (0x3ffff), .domains = (FORCEWAKE_MEDIA ) }, |
1119 | }; |
1120 | |
1121 | /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ |
1122 | static const struct intel_forcewake_range __gen11_fw_ranges[] = { |
1123 | GEN_FW_RANGE(0x0, 0x1fff, 0){ .start = (0x0), .end = (0x1fff), .domains = (0) }, /* uncore range */ |
1124 | GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER){ .start = (0x2000), .end = (0x26ff), .domains = (FORCEWAKE_RENDER ) }, |
1125 | GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER){ .start = (0x2700), .end = (0x2fff), .domains = (FORCEWAKE_BLITTER ) }, |
1126 | GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER){ .start = (0x3000), .end = (0x3fff), .domains = (FORCEWAKE_RENDER ) }, |
1127 | GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER){ .start = (0x4000), .end = (0x51ff), .domains = (FORCEWAKE_BLITTER ) }, |
1128 | GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER){ .start = (0x5200), .end = (0x7fff), .domains = (FORCEWAKE_RENDER ) }, |
1129 | GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER){ .start = (0x8000), .end = (0x813f), .domains = (FORCEWAKE_BLITTER ) }, |
1130 | GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER){ .start = (0x8140), .end = (0x815f), .domains = (FORCEWAKE_RENDER ) }, |
1131 | GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER){ .start = (0x8160), .end = (0x82ff), .domains = (FORCEWAKE_BLITTER ) }, |
1132 | GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER){ .start = (0x8300), .end = (0x84ff), .domains = (FORCEWAKE_RENDER ) }, |
1133 | GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER){ .start = (0x8500), .end = (0x87ff), .domains = (FORCEWAKE_BLITTER ) }, |
1134 | GEN_FW_RANGE(0x8800, 0x8bff, 0){ .start = (0x8800), .end = (0x8bff), .domains = (0) }, |
1135 | GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER){ .start = (0x8c00), .end = (0x8cff), .domains = (FORCEWAKE_RENDER ) }, |
1136 | GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_BLITTER){ .start = (0x8d00), .end = (0x94cf), .domains = (FORCEWAKE_BLITTER ) }, |
1137 | GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER){ .start = (0x94d0), .end = (0x955f), .domains = (FORCEWAKE_RENDER ) }, |
1138 | GEN_FW_RANGE(0x9560, 0x95ff, 0){ .start = (0x9560), .end = (0x95ff), .domains = (0) }, |
1139 | GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_BLITTER){ .start = (0x9600), .end = (0xafff), .domains = (FORCEWAKE_BLITTER ) }, |
1140 | GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER){ .start = (0xb000), .end = (0xb47f), .domains = (FORCEWAKE_RENDER ) }, |
1141 | GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_BLITTER){ .start = (0xb480), .end = (0xdeff), .domains = (FORCEWAKE_BLITTER ) }, |
1142 | GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER){ .start = (0xdf00), .end = (0xe8ff), .domains = (FORCEWAKE_RENDER ) }, |
1143 | GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_BLITTER){ .start = (0xe900), .end = (0x16dff), .domains = (FORCEWAKE_BLITTER ) }, |
1144 | GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER){ .start = (0x16e00), .end = (0x19fff), .domains = (FORCEWAKE_RENDER ) }, |
1145 | GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_BLITTER){ .start = (0x1a000), .end = (0x23fff), .domains = (FORCEWAKE_BLITTER ) }, |
1146 | GEN_FW_RANGE(0x24000, 0x2407f, 0){ .start = (0x24000), .end = (0x2407f), .domains = (0) }, |
1147 | GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_BLITTER){ .start = (0x24080), .end = (0x2417f), .domains = (FORCEWAKE_BLITTER ) }, |
1148 | GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER){ .start = (0x24180), .end = (0x242ff), .domains = (FORCEWAKE_RENDER ) }, |
1149 | GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_BLITTER){ .start = (0x24300), .end = (0x243ff), .domains = (FORCEWAKE_BLITTER ) }, |
1150 | GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER){ .start = (0x24400), .end = (0x24fff), .domains = (FORCEWAKE_RENDER ) }, |
1151 | GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_BLITTER){ .start = (0x25000), .end = (0x3ffff), .domains = (FORCEWAKE_BLITTER ) }, |
1152 | GEN_FW_RANGE(0x40000, 0x1bffff, 0){ .start = (0x40000), .end = (0x1bffff), .domains = (0) }, |
1153 | GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0){ .start = (0x1c0000), .end = (0x1c3fff), .domains = (FORCEWAKE_MEDIA_VDBOX0 ) }, |
1154 | GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0){ .start = (0x1c4000), .end = (0x1c7fff), .domains = (0) }, |
1155 | GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0){ .start = (0x1c8000), .end = (0x1cffff), .domains = (FORCEWAKE_MEDIA_VEBOX0 ) }, |
1156 | GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2){ .start = (0x1d0000), .end = (0x1d3fff), .domains = (FORCEWAKE_MEDIA_VDBOX2 ) }, |
1157 | GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0){ .start = (0x1d4000), .end = (0x1dbfff), .domains = (0) } |
1158 | }; |
1159 | |
1160 | /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ |
1161 | static const struct intel_forcewake_range __gen12_fw_ranges[] = { |
1162 | GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER){ .start = (0x0), .end = (0xaff), .domains = (FORCEWAKE_BLITTER ) }, |
1163 | GEN_FW_RANGE(0xb00, 0x1fff, 0){ .start = (0xb00), .end = (0x1fff), .domains = (0) }, /* uncore range */ |
1164 | GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER){ .start = (0x2000), .end = (0x26ff), .domains = (FORCEWAKE_RENDER ) }, |
1165 | GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER){ .start = (0x2700), .end = (0x2fff), .domains = (FORCEWAKE_BLITTER ) }, |
1166 | GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER){ .start = (0x3000), .end = (0x3fff), .domains = (FORCEWAKE_RENDER ) }, |
1167 | GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER){ .start = (0x4000), .end = (0x51ff), .domains = (FORCEWAKE_BLITTER ) }, |
1168 | GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER){ .start = (0x5200), .end = (0x7fff), .domains = (FORCEWAKE_RENDER ) }, |
1169 | GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER){ .start = (0x8000), .end = (0x813f), .domains = (FORCEWAKE_BLITTER ) }, |
1170 | GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER){ .start = (0x8140), .end = (0x815f), .domains = (FORCEWAKE_RENDER ) }, |
1171 | GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER){ .start = (0x8160), .end = (0x82ff), .domains = (FORCEWAKE_BLITTER ) }, |
1172 | GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER){ .start = (0x8300), .end = (0x84ff), .domains = (FORCEWAKE_RENDER ) }, |
1173 | GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER){ .start = (0x8500), .end = (0x8bff), .domains = (FORCEWAKE_BLITTER ) }, |
1174 | GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER){ .start = (0x8c00), .end = (0x8cff), .domains = (FORCEWAKE_RENDER ) }, |
1175 | GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER){ .start = (0x8d00), .end = (0x93ff), .domains = (FORCEWAKE_BLITTER ) }, |
1176 | GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL){ .start = (0x9400), .end = (0x97ff), .domains = (FORCEWAKE_ALL ) }, |
1177 | GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER){ .start = (0x9800), .end = (0xafff), .domains = (FORCEWAKE_BLITTER ) }, |
1178 | GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER){ .start = (0xb000), .end = (0xb47f), .domains = (FORCEWAKE_RENDER ) }, |
1179 | GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER){ .start = (0xb480), .end = (0xdfff), .domains = (FORCEWAKE_BLITTER ) }, |
1180 | GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER){ .start = (0xe000), .end = (0xe8ff), .domains = (FORCEWAKE_RENDER ) }, |
1181 | GEN_FW_RANGE(0xe900, 0x147ff, FORCEWAKE_BLITTER){ .start = (0xe900), .end = (0x147ff), .domains = (FORCEWAKE_BLITTER ) }, |
1182 | GEN_FW_RANGE(0x14800, 0x148ff, FORCEWAKE_RENDER){ .start = (0x14800), .end = (0x148ff), .domains = (FORCEWAKE_RENDER ) }, |
1183 | GEN_FW_RANGE(0x14900, 0x19fff, FORCEWAKE_BLITTER){ .start = (0x14900), .end = (0x19fff), .domains = (FORCEWAKE_BLITTER ) }, |
1184 | GEN_FW_RANGE(0x1a000, 0x1a7ff, FORCEWAKE_RENDER){ .start = (0x1a000), .end = (0x1a7ff), .domains = (FORCEWAKE_RENDER ) }, |
1185 | GEN_FW_RANGE(0x1a800, 0x1afff, FORCEWAKE_BLITTER){ .start = (0x1a800), .end = (0x1afff), .domains = (FORCEWAKE_BLITTER ) }, |
1186 | GEN_FW_RANGE(0x1b000, 0x1bfff, FORCEWAKE_RENDER){ .start = (0x1b000), .end = (0x1bfff), .domains = (FORCEWAKE_RENDER ) }, |
1187 | GEN_FW_RANGE(0x1c000, 0x243ff, FORCEWAKE_BLITTER){ .start = (0x1c000), .end = (0x243ff), .domains = (FORCEWAKE_BLITTER ) }, |
1188 | GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER){ .start = (0x24400), .end = (0x247ff), .domains = (FORCEWAKE_RENDER ) }, |
1189 | GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER){ .start = (0x24800), .end = (0x3ffff), .domains = (FORCEWAKE_BLITTER ) }, |
1190 | GEN_FW_RANGE(0x40000, 0x1bffff, 0){ .start = (0x40000), .end = (0x1bffff), .domains = (0) }, |
1191 | GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0){ .start = (0x1c0000), .end = (0x1c3fff), .domains = (FORCEWAKE_MEDIA_VDBOX0 ) }, |
1192 | GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1){ .start = (0x1c4000), .end = (0x1c7fff), .domains = (FORCEWAKE_MEDIA_VDBOX1 ) }, |
1193 | GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0){ .start = (0x1c8000), .end = (0x1cbfff), .domains = (FORCEWAKE_MEDIA_VEBOX0 ) }, |
1194 | GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER){ .start = (0x1cc000), .end = (0x1cffff), .domains = (FORCEWAKE_BLITTER ) }, |
1195 | GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2){ .start = (0x1d0000), .end = (0x1d3fff), .domains = (FORCEWAKE_MEDIA_VDBOX2 ) }, |
1196 | GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3){ .start = (0x1d4000), .end = (0x1d7fff), .domains = (FORCEWAKE_MEDIA_VDBOX3 ) }, |
1197 | GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1){ .start = (0x1d8000), .end = (0x1dbfff), .domains = (FORCEWAKE_MEDIA_VEBOX1 ) } |
1198 | }; |
1199 | |
1200 | static void |
1201 | ilk_dummy_write(struct intel_uncore *uncore) |
1202 | { |
1203 | /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up |
1204 | * the chip from rc6 before touching it for real. MI_MODE is masked, |
1205 | * hence harmless to write 0 into. */ |
1206 | __raw_uncore_write32(uncore, MI_MODE((const i915_reg_t){ .reg = (0x209c) }), 0); |
1207 | } |
1208 | |
1209 | static void |
1210 | __unclaimed_reg_debug(struct intel_uncore *uncore, |
1211 | const i915_reg_t reg, |
1212 | const bool_Bool read, |
1213 | const bool_Bool before) |
1214 | { |
1215 | if (drm_WARN(&uncore->i915->drm,({ int __ret = !!(check_for_unclaimed_mmio(uncore) && !before); if (__ret) printf("%s %s: " "Unclaimed %s register 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , read ? "read from" : "write to", i915_mmio_reg_offset(reg)) ; __builtin_expect(!!(__ret), 0); }) |
1216 | check_for_unclaimed_mmio(uncore) && !before,({ int __ret = !!(check_for_unclaimed_mmio(uncore) && !before); if (__ret) printf("%s %s: " "Unclaimed %s register 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , read ? "read from" : "write to", i915_mmio_reg_offset(reg)) ; __builtin_expect(!!(__ret), 0); }) |
1217 | "Unclaimed %s register 0x%x\n",({ int __ret = !!(check_for_unclaimed_mmio(uncore) && !before); if (__ret) printf("%s %s: " "Unclaimed %s register 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , read ? "read from" : "write to", i915_mmio_reg_offset(reg)) ; __builtin_expect(!!(__ret), 0); }) |
1218 | read ? "read from" : "write to",({ int __ret = !!(check_for_unclaimed_mmio(uncore) && !before); if (__ret) printf("%s %s: " "Unclaimed %s register 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , read ? "read from" : "write to", i915_mmio_reg_offset(reg)) ; __builtin_expect(!!(__ret), 0); }) |
1219 | i915_mmio_reg_offset(reg))({ int __ret = !!(check_for_unclaimed_mmio(uncore) && !before); if (__ret) printf("%s %s: " "Unclaimed %s register 0x%x\n" , dev_driver_string((&uncore->i915->drm)->dev), "" , read ? "read from" : "write to", i915_mmio_reg_offset(reg)) ; __builtin_expect(!!(__ret), 0); })) |
1220 | /* Only report the first N failures */ |
1221 | uncore->i915->params.mmio_debug--; |
1222 | } |
1223 | |
1224 | static inline void |
1225 | unclaimed_reg_debug(struct intel_uncore *uncore, |
1226 | const i915_reg_t reg, |
1227 | const bool_Bool read, |
1228 | const bool_Bool before) |
1229 | { |
1230 | if (likely(!uncore->i915->params.mmio_debug)__builtin_expect(!!(!uncore->i915->params.mmio_debug), 1 )) |
1231 | return; |
1232 | |
1233 | /* interrupts are disabled and re-enabled around uncore->lock usage */ |
1234 | lockdep_assert_held(&uncore->lock)do { (void)(&uncore->lock); } while(0); |
1235 | |
1236 | if (before) |
1237 | spin_lock(&uncore->debug->lock)mtx_enter(&uncore->debug->lock); |
1238 | |
1239 | __unclaimed_reg_debug(uncore, reg, read, before); |
1240 | |
1241 | if (!before) |
1242 | spin_unlock(&uncore->debug->lock)mtx_leave(&uncore->debug->lock); |
1243 | } |
1244 | |
1245 | #define __vgpu_read(x)static ux vgpu_readx(struct intel_uncore *uncore, i915_reg_t reg , _Bool trace) { ux val = __raw_uncore_readx(uncore, reg); ; return val; } \ |
1246 | static u##x \ |
1247 | vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool_Bool trace) { \ |
1248 | u##x val = __raw_uncore_read##x(uncore, reg); \ |
1249 | trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ |
1250 | return val; \ |
1251 | } |
1252 | __vgpu_read(8)static u8 vgpu_read8(struct intel_uncore *uncore, i915_reg_t reg , _Bool trace) { u8 val = __raw_uncore_read8(uncore, reg); ; return val; } |
1253 | __vgpu_read(16)static u16 vgpu_read16(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { u16 val = __raw_uncore_read16(uncore, reg ); ; return val; } |
1254 | __vgpu_read(32)static u32 vgpu_read32(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { u32 val = __raw_uncore_read32(uncore, reg ); ; return val; } |
1255 | __vgpu_read(64)static u64 vgpu_read64(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { u64 val = __raw_uncore_read64(uncore, reg ); ; return val; } |
1256 | |
1257 | #define GEN2_READ_HEADER(x) \ |
1258 | u##x val = 0; \ |
1259 | assert_rpm_wakelock_held(uncore->rpm); |
1260 | |
1261 | #define GEN2_READ_FOOTER \ |
1262 | trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ |
1263 | return val |
1264 | |
1265 | #define __gen2_read(x) \ |
1266 | static u##x \ |
1267 | gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool_Bool trace) { \ |
1268 | GEN2_READ_HEADER(x); \ |
1269 | val = __raw_uncore_read##x(uncore, reg); \ |
1270 | GEN2_READ_FOOTER; \ |
1271 | } |
1272 | |
1273 | #define __gen5_read(x) \ |
1274 | static u##x \ |
1275 | gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool_Bool trace) { \ |
1276 | GEN2_READ_HEADER(x); \ |
1277 | ilk_dummy_write(uncore); \ |
1278 | val = __raw_uncore_read##x(uncore, reg); \ |
1279 | GEN2_READ_FOOTER; \ |
1280 | } |
1281 | |
1282 | __gen5_read(8) |
1283 | __gen5_read(16) |
1284 | __gen5_read(32) |
1285 | __gen5_read(64) |
1286 | __gen2_read(8) |
1287 | __gen2_read(16) |
1288 | __gen2_read(32) |
1289 | __gen2_read(64) |
1290 | |
1291 | #undef __gen5_read |
1292 | #undef __gen2_read |
1293 | |
1294 | #undef GEN2_READ_FOOTER |
1295 | #undef GEN2_READ_HEADER |
1296 | |
1297 | #define GEN6_READ_HEADER(x) \ |
1298 | u32 offset = i915_mmio_reg_offset(reg); \ |
1299 | unsigned long irqflags; \ |
1300 | u##x val = 0; \ |
1301 | assert_rpm_wakelock_held(uncore->rpm); \ |
1302 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); \ |
1303 | unclaimed_reg_debug(uncore, reg, true1, true1) |
1304 | |
1305 | #define GEN6_READ_FOOTER \ |
1306 | unclaimed_reg_debug(uncore, reg, true1, false0); \ |
1307 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0); \ |
1308 | trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ |
1309 | return val |
1310 | |
1311 | static noinline__attribute__((__noinline__)) void ___force_wake_auto(struct intel_uncore *uncore, |
1312 | enum forcewake_domains fw_domains) |
1313 | { |
1314 | struct intel_uncore_forcewake_domain *domain; |
1315 | unsigned int tmp; |
1316 | |
1317 | GEM_BUG_ON(fw_domains & ~uncore->fw_domains)((void)0); |
1318 | |
1319 | for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)for (tmp = (fw_domains); tmp ;) if (!(domain = (uncore)->fw_domain [({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx )); __idx; })])) {} else |
1320 | fw_domain_arm_timer(domain); |
1321 | |
1322 | uncore->funcs.force_wake_get(uncore, fw_domains); |
1323 | } |
1324 | |
1325 | static inline void __force_wake_auto(struct intel_uncore *uncore, |
1326 | enum forcewake_domains fw_domains) |
1327 | { |
1328 | GEM_BUG_ON(!fw_domains)((void)0); |
1329 | |
1330 | /* Turn on all requested but inactive supported forcewake domains. */ |
1331 | fw_domains &= uncore->fw_domains; |
1332 | fw_domains &= ~uncore->fw_domains_active; |
1333 | |
1334 | if (fw_domains) |
1335 | ___force_wake_auto(uncore, fw_domains); |
1336 | } |
1337 | |
1338 | #define __gen_read(func, x)static ux func_readx(struct intel_uncore *uncore, i915_reg_t reg , _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (x); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_readx (uncore, reg); GEN6_READ_FOOTER; } \ |
1339 | static u##x \ |
1340 | func##_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool_Bool trace) { \ |
1341 | enum forcewake_domains fw_engine; \ |
1342 | GEN6_READ_HEADER(x); \ |
1343 | fw_engine = __##func##_reg_read_fw_domains(uncore, offset); \ |
1344 | if (fw_engine) \ |
1345 | __force_wake_auto(uncore, fw_engine); \ |
1346 | val = __raw_uncore_read##x(uncore, reg); \ |
1347 | GEN6_READ_FOOTER; \ |
1348 | } |
1349 | |
1350 | #define __gen_reg_read_funcs(func) \ |
1351 | static enum forcewake_domains \ |
1352 | func##_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) { \ |
1353 | return __##func##_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg)); \ |
1354 | } \static u8 func_read8(struct intel_uncore *uncore, i915_reg_t reg , _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (8); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read8 (uncore, reg); GEN6_READ_FOOTER; } |
1355 | \static u8 func_read8(struct intel_uncore *uncore, i915_reg_t reg , _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (8); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read8 (uncore, reg); GEN6_READ_FOOTER; } |
1356 | __gen_read(func, 8)static u8 func_read8(struct intel_uncore *uncore, i915_reg_t reg , _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (8); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read8 (uncore, reg); GEN6_READ_FOOTER; } \static u16 func_read16(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (16); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read16 (uncore, reg); GEN6_READ_FOOTER; } |
1357 | __gen_read(func, 16)static u16 func_read16(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (16); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read16 (uncore, reg); GEN6_READ_FOOTER; } \static u32 func_read32(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (32); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read32 (uncore, reg); GEN6_READ_FOOTER; } |
1358 | __gen_read(func, 32)static u32 func_read32(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (32); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read32 (uncore, reg); GEN6_READ_FOOTER; } \static u64 func_read64(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (64); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read64 (uncore, reg); GEN6_READ_FOOTER; } |
1359 | __gen_read(func, 64)static u64 func_read64(struct intel_uncore *uncore, i915_reg_t reg, _Bool trace) { enum forcewake_domains fw_engine; GEN6_READ_HEADER (64); fw_engine = __func_reg_read_fw_domains(uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine); val = __raw_uncore_read64 (uncore, reg); GEN6_READ_FOOTER; } |
1360 | |
1361 | __gen_reg_read_funcs(gen12_fwtable); |
1362 | __gen_reg_read_funcs(gen11_fwtable); |
1363 | __gen_reg_read_funcs(fwtable); |
1364 | __gen_reg_read_funcs(gen6); |
1365 | |
1366 | #undef __gen_reg_read_funcs |
1367 | #undef GEN6_READ_FOOTER |
1368 | #undef GEN6_READ_HEADER |
1369 | |
1370 | #define GEN2_WRITE_HEADER \ |
1371 | trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ |
1372 | assert_rpm_wakelock_held(uncore->rpm); \ |
1373 | |
1374 | #define GEN2_WRITE_FOOTER |
1375 | |
1376 | #define __gen2_write(x) \ |
1377 | static void \ |
1378 | gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool_Bool trace) { \ |
1379 | GEN2_WRITE_HEADER; \ |
1380 | __raw_uncore_write##x(uncore, reg, val); \ |
1381 | GEN2_WRITE_FOOTER; \ |
1382 | } |
1383 | |
1384 | #define __gen5_write(x) \ |
1385 | static void \ |
1386 | gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool_Bool trace) { \ |
1387 | GEN2_WRITE_HEADER; \ |
1388 | ilk_dummy_write(uncore); \ |
1389 | __raw_uncore_write##x(uncore, reg, val); \ |
1390 | GEN2_WRITE_FOOTER; \ |
1391 | } |
1392 | |
1393 | __gen5_write(8) |
1394 | __gen5_write(16) |
1395 | __gen5_write(32) |
1396 | __gen2_write(8) |
1397 | __gen2_write(16) |
1398 | __gen2_write(32) |
1399 | |
1400 | #undef __gen5_write |
1401 | #undef __gen2_write |
1402 | |
1403 | #undef GEN2_WRITE_FOOTER |
1404 | #undef GEN2_WRITE_HEADER |
1405 | |
1406 | #define GEN6_WRITE_HEADER \ |
1407 | u32 offset = i915_mmio_reg_offset(reg); \ |
1408 | unsigned long irqflags; \ |
1409 | trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ |
1410 | assert_rpm_wakelock_held(uncore->rpm); \ |
1411 | spin_lock_irqsave(&uncore->lock, irqflags)do { irqflags = 0; mtx_enter(&uncore->lock); } while ( 0); \ |
1412 | unclaimed_reg_debug(uncore, reg, false0, true1) |
1413 | |
1414 | #define GEN6_WRITE_FOOTER \ |
1415 | unclaimed_reg_debug(uncore, reg, false0, false0); \ |
1416 | spin_unlock_irqrestore(&uncore->lock, irqflags)do { (void)(irqflags); mtx_leave(&uncore->lock); } while (0) |
1417 | |
1418 | #define __gen6_write(x)static void gen6_writex(struct intel_uncore *uncore, i915_reg_t reg, ux val, _Bool trace) { GEN6_WRITE_HEADER; if (((offset) < 0x40000)) __gen6_gt_wait_for_fifo(uncore); __raw_uncore_writex (uncore, reg, val); GEN6_WRITE_FOOTER; } \ |
1419 | static void \ |
1420 | gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool_Bool trace) { \ |
1421 | GEN6_WRITE_HEADER; \ |
1422 | if (NEEDS_FORCE_WAKE(offset)((offset) < 0x40000)) \ |
1423 | __gen6_gt_wait_for_fifo(uncore); \ |
1424 | __raw_uncore_write##x(uncore, reg, val); \ |
1425 | GEN6_WRITE_FOOTER; \ |
1426 | } |
1427 | __gen6_write(8)static void gen6_write8(struct intel_uncore *uncore, i915_reg_t reg, u8 val, _Bool trace) { GEN6_WRITE_HEADER; if (((offset) < 0x40000)) __gen6_gt_wait_for_fifo(uncore); __raw_uncore_write8 (uncore, reg, val); GEN6_WRITE_FOOTER; } |
1428 | __gen6_write(16)static void gen6_write16(struct intel_uncore *uncore, i915_reg_t reg, u16 val, _Bool trace) { GEN6_WRITE_HEADER; if (((offset ) < 0x40000)) __gen6_gt_wait_for_fifo(uncore); __raw_uncore_write16 (uncore, reg, val); GEN6_WRITE_FOOTER; } |
1429 | __gen6_write(32)static void gen6_write32(struct intel_uncore *uncore, i915_reg_t reg, u32 val, _Bool trace) { GEN6_WRITE_HEADER; if (((offset ) < 0x40000)) __gen6_gt_wait_for_fifo(uncore); __raw_uncore_write32 (uncore, reg, val); GEN6_WRITE_FOOTER; } |
1430 | |
1431 | #define __gen_write(func, x)static void func_writex(struct intel_uncore *uncore, i915_reg_t reg, ux val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_writex(uncore, reg, val); GEN6_WRITE_FOOTER; } \ |
1432 | static void \ |
1433 | func##_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool_Bool trace) { \ |
1434 | enum forcewake_domains fw_engine; \ |
1435 | GEN6_WRITE_HEADER; \ |
1436 | fw_engine = __##func##_reg_write_fw_domains(uncore, offset); \ |
1437 | if (fw_engine) \ |
1438 | __force_wake_auto(uncore, fw_engine); \ |
1439 | __raw_uncore_write##x(uncore, reg, val); \ |
1440 | GEN6_WRITE_FOOTER; \ |
1441 | } |
1442 | |
1443 | #define __gen_reg_write_funcs(func) \ |
1444 | static enum forcewake_domains \ |
1445 | func##_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) { \ |
1446 | return __##func##_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg)); \ |
1447 | } \static void func_write8(struct intel_uncore *uncore, i915_reg_t reg, u8 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write8(uncore, reg, val); GEN6_WRITE_FOOTER; } |
1448 | \static void func_write8(struct intel_uncore *uncore, i915_reg_t reg, u8 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write8(uncore, reg, val); GEN6_WRITE_FOOTER; } |
1449 | __gen_write(func, 8)static void func_write8(struct intel_uncore *uncore, i915_reg_t reg, u8 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write8(uncore, reg, val); GEN6_WRITE_FOOTER; } \static void func_write16(struct intel_uncore *uncore, i915_reg_t reg, u16 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write16(uncore, reg, val); GEN6_WRITE_FOOTER; } |
1450 | __gen_write(func, 16)static void func_write16(struct intel_uncore *uncore, i915_reg_t reg, u16 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write16(uncore, reg, val); GEN6_WRITE_FOOTER; } \static void func_write32(struct intel_uncore *uncore, i915_reg_t reg, u32 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write32(uncore, reg, val); GEN6_WRITE_FOOTER; } |
1451 | __gen_write(func, 32)static void func_write32(struct intel_uncore *uncore, i915_reg_t reg, u32 val, _Bool trace) { enum forcewake_domains fw_engine ; GEN6_WRITE_HEADER; fw_engine = __func_reg_write_fw_domains( uncore, offset); if (fw_engine) __force_wake_auto(uncore, fw_engine ); __raw_uncore_write32(uncore, reg, val); GEN6_WRITE_FOOTER; } |
1452 | |
1453 | __gen_reg_write_funcs(gen12_fwtable); |
1454 | __gen_reg_write_funcs(gen11_fwtable); |
1455 | __gen_reg_write_funcs(fwtable); |
1456 | __gen_reg_write_funcs(gen8); |
1457 | |
1458 | #undef __gen_reg_write_funcs |
1459 | #undef GEN6_WRITE_FOOTER |
1460 | #undef GEN6_WRITE_HEADER |
1461 | |
1462 | #define __vgpu_write(x)static void vgpu_writex(struct intel_uncore *uncore, i915_reg_t reg, ux val, _Bool trace) { ; __raw_uncore_writex(uncore, reg , val); } \ |
1463 | static void \ |
1464 | vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool_Bool trace) { \ |
1465 | trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ |
1466 | __raw_uncore_write##x(uncore, reg, val); \ |
1467 | } |
1468 | __vgpu_write(8)static void vgpu_write8(struct intel_uncore *uncore, i915_reg_t reg, u8 val, _Bool trace) { ; __raw_uncore_write8(uncore, reg , val); } |
1469 | __vgpu_write(16)static void vgpu_write16(struct intel_uncore *uncore, i915_reg_t reg, u16 val, _Bool trace) { ; __raw_uncore_write16(uncore, reg , val); } |
1470 | __vgpu_write(32)static void vgpu_write32(struct intel_uncore *uncore, i915_reg_t reg, u32 val, _Bool trace) { ; __raw_uncore_write32(uncore, reg , val); } |
1471 | |
1472 | #define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x)do { (uncore)->funcs.mmio_writeb = x_write8; (uncore)-> funcs.mmio_writew = x_write16; (uncore)->funcs.mmio_writel = x_write32; } while (0) \ |
1473 | do { \ |
1474 | (uncore)->funcs.mmio_writeb = x##_write8; \ |
1475 | (uncore)->funcs.mmio_writew = x##_write16; \ |
1476 | (uncore)->funcs.mmio_writel = x##_write32; \ |
1477 | } while (0) |
1478 | |
1479 | #define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x)do { (uncore)->funcs.mmio_readb = x_read8; (uncore)->funcs .mmio_readw = x_read16; (uncore)->funcs.mmio_readl = x_read32 ; (uncore)->funcs.mmio_readq = x_read64; } while (0) \ |
1480 | do { \ |
1481 | (uncore)->funcs.mmio_readb = x##_read8; \ |
1482 | (uncore)->funcs.mmio_readw = x##_read16; \ |
1483 | (uncore)->funcs.mmio_readl = x##_read32; \ |
1484 | (uncore)->funcs.mmio_readq = x##_read64; \ |
1485 | } while (0) |
1486 | |
1487 | #define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x)do { do { ((uncore))->funcs.mmio_writeb = x_write8; ((uncore ))->funcs.mmio_writew = x_write16; ((uncore))->funcs.mmio_writel = x_write32; } while (0); (uncore)->funcs.write_fw_domains = x_reg_write_fw_domains; } while (0) \ |
1488 | do { \ |
1489 | ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x)do { ((uncore))->funcs.mmio_writeb = x_write8; ((uncore))-> funcs.mmio_writew = x_write16; ((uncore))->funcs.mmio_writel = x_write32; } while (0); \ |
1490 | (uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \ |
1491 | } while (0) |
1492 | |
1493 | #define ASSIGN_READ_MMIO_VFUNCS(uncore, x)do { do { (uncore)->funcs.mmio_readb = x_read8; (uncore)-> funcs.mmio_readw = x_read16; (uncore)->funcs.mmio_readl = x_read32 ; (uncore)->funcs.mmio_readq = x_read64; } while (0); (uncore )->funcs.read_fw_domains = x_reg_read_fw_domains; } while ( 0) \ |
1494 | do { \ |
1495 | ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x)do { (uncore)->funcs.mmio_readb = x_read8; (uncore)->funcs .mmio_readw = x_read16; (uncore)->funcs.mmio_readl = x_read32 ; (uncore)->funcs.mmio_readq = x_read64; } while (0); \ |
1496 | (uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \ |
1497 | } while (0) |
1498 | |
1499 | static int __fw_domain_init(struct intel_uncore *uncore, |
1500 | enum forcewake_domain_id domain_id, |
1501 | i915_reg_t reg_set, |
1502 | i915_reg_t reg_ack) |
1503 | { |
1504 | struct intel_uncore_forcewake_domain *d; |
1505 | |
1506 | GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT)((void)0); |
1507 | GEM_BUG_ON(uncore->fw_domain[domain_id])((void)0); |
1508 | |
1509 | if (i915_inject_probe_failure(uncore->i915)({ ((void)0); 0; })) |
1510 | return -ENOMEM12; |
1511 | |
1512 | d = kzalloc(sizeof(*d), GFP_KERNEL(0x0001 | 0x0004)); |
1513 | if (!d) |
1514 | return -ENOMEM12; |
1515 | |
1516 | drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set))({ int __ret = !!((!i915_mmio_reg_valid(reg_set))); if (__ret ) printf("%s %s: " "%s", dev_driver_string(((&uncore-> i915->drm))->dev), "", "drm_WARN_ON(" "!i915_mmio_reg_valid(reg_set)" ")"); __builtin_expect(!!(__ret), 0); }); |
1517 | drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack))({ int __ret = !!((!i915_mmio_reg_valid(reg_ack))); if (__ret ) printf("%s %s: " "%s", dev_driver_string(((&uncore-> i915->drm))->dev), "", "drm_WARN_ON(" "!i915_mmio_reg_valid(reg_ack)" ")"); __builtin_expect(!!(__ret), 0); }); |
1518 | |
1519 | d->uncore = uncore; |
1520 | d->wake_count = 0; |
1521 | d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set); |
1522 | d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack); |
1523 | |
1524 | d->id = domain_id; |
1525 | |
1526 | BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER))extern char _ctassert[(!(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER ))) ? 1 : -1 ] __attribute__((__unused__)); |
1527 | BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER))extern char _ctassert[(!(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER ))) ? 1 : -1 ] __attribute__((__unused__)); |
1528 | BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA))extern char _ctassert[(!(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA ))) ? 1 : -1 ] __attribute__((__unused__)); |
1529 | BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0))extern char _ctassert[(!(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0))) ? 1 : -1 ] __attribute__((__unused__ )); |
1530 | BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1))extern char _ctassert[(!(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1))) ? 1 : -1 ] __attribute__((__unused__ )); |
1531 | BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2))extern char _ctassert[(!(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2))) ? 1 : -1 ] __attribute__((__unused__ )); |
1532 | BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3))extern char _ctassert[(!(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3))) ? 1 : -1 ] __attribute__((__unused__ )); |
1533 | BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0))extern char _ctassert[(!(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0))) ? 1 : -1 ] __attribute__((__unused__ )); |
1534 | BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1))extern char _ctassert[(!(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1))) ? 1 : -1 ] __attribute__((__unused__ )); |
1535 | |
1536 | d->mask = BIT(domain_id)(1UL << (domain_id)); |
1537 | |
1538 | #ifdef __linux__ |
1539 | hrtimer_init(&d->timer, CLOCK_MONOTONIC3, HRTIMER_MODE_REL1); |
1540 | d->timer.function = intel_uncore_fw_release_timer; |
1541 | #else |
1542 | timeout_set(&d->timer, intel_uncore_fw_release_timer, d); |
1543 | #endif |
1544 | |
1545 | uncore->fw_domains |= BIT(domain_id)(1UL << (domain_id)); |
1546 | |
1547 | fw_domain_reset(d); |
1548 | |
1549 | uncore->fw_domain[domain_id] = d; |
1550 | |
1551 | return 0; |
1552 | } |
1553 | |
1554 | static void fw_domain_fini(struct intel_uncore *uncore, |
1555 | enum forcewake_domain_id domain_id) |
1556 | { |
1557 | struct intel_uncore_forcewake_domain *d; |
1558 | |
1559 | GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT)((void)0); |
1560 | |
1561 | d = fetch_and_zero(&uncore->fw_domain[domain_id])({ typeof(*&uncore->fw_domain[domain_id]) __T = *(& uncore->fw_domain[domain_id]); *(&uncore->fw_domain [domain_id]) = (typeof(*&uncore->fw_domain[domain_id]) )0; __T; }); |
1562 | if (!d) |
1563 | return; |
1564 | |
1565 | uncore->fw_domains &= ~BIT(domain_id)(1UL << (domain_id)); |
1566 | drm_WARN_ON(&uncore->i915->drm, d->wake_count)({ int __ret = !!((d->wake_count)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&uncore->i915->drm))-> dev), "", "drm_WARN_ON(" "d->wake_count" ")"); __builtin_expect (!!(__ret), 0); }); |
1567 | drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer))({ int __ret = !!((timeout_del(&d->timer))); if (__ret ) printf("%s %s: " "%s", dev_driver_string(((&uncore-> i915->drm))->dev), "", "drm_WARN_ON(" "timeout_del(&d->timer)" ")"); __builtin_expect(!!(__ret), 0); }); |
1568 | kfree(d); |
1569 | } |
1570 | |
1571 | static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore) |
1572 | { |
1573 | struct intel_uncore_forcewake_domain *d; |
1574 | int tmp; |
1575 | |
1576 | for_each_fw_domain(d, uncore, tmp)for (tmp = ((uncore)->fw_domains); tmp ;) if (!(d = (uncore )->fw_domain[({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx; })])) {} else |
1577 | fw_domain_fini(uncore, d->id); |
1578 | } |
1579 | |
1580 | static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) |
1581 | { |
1582 | struct drm_i915_privateinteldrm_softc *i915 = uncore->i915; |
1583 | int ret = 0; |
1584 | |
1585 | GEM_BUG_ON(!intel_uncore_has_forcewake(uncore))((void)0); |
1586 | |
1587 | #define fw_domain_init(uncore__, id__, set__, ack__) \ |
1588 | (ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__)))) |
1589 | |
1590 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 11) { |
1591 | /* we'll prune the domains of missing engines later */ |
1592 | intel_engine_mask_t emask = INTEL_INFO(i915)(&(i915)->__info)->platform_engine_mask; |
1593 | int i; |
1594 | |
1595 | uncore->funcs.force_wake_get = fw_domains_get_with_fallback; |
1596 | uncore->funcs.force_wake_put = fw_domains_put; |
1597 | fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1598 | FORCEWAKE_RENDER_GEN9((const i915_reg_t){ .reg = (0xa278) }), |
1599 | FORCEWAKE_ACK_RENDER_GEN9((const i915_reg_t){ .reg = (0x0D84) })); |
1600 | fw_domain_init(uncore, FW_DOMAIN_ID_BLITTER, |
1601 | FORCEWAKE_BLITTER_GEN9((const i915_reg_t){ .reg = (0xa188) }), |
1602 | FORCEWAKE_ACK_BLITTER_GEN9((const i915_reg_t){ .reg = (0x130044) })); |
1603 | |
1604 | for (i = 0; i < I915_MAX_VCS4; i++) { |
1605 | if (!__HAS_ENGINE(emask, _VCS(i))((emask) & (1UL << ((VCS0 + (i)))))) |
1606 | continue; |
1607 | |
1608 | fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i, |
1609 | FORCEWAKE_MEDIA_VDBOX_GEN11(i)((const i915_reg_t){ .reg = (0xa540 + (i) * 4) }), |
1610 | FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i)((const i915_reg_t){ .reg = (0x0D50 + (i) * 4) })); |
1611 | } |
1612 | for (i = 0; i < I915_MAX_VECS2; i++) { |
1613 | if (!__HAS_ENGINE(emask, _VECS(i))((emask) & (1UL << ((VECS0 + (i)))))) |
1614 | continue; |
1615 | |
1616 | fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i, |
1617 | FORCEWAKE_MEDIA_VEBOX_GEN11(i)((const i915_reg_t){ .reg = (0xa560 + (i) * 4) }), |
1618 | FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i)((const i915_reg_t){ .reg = (0x0D70 + (i) * 4) })); |
1619 | } |
1620 | } else if (IS_GEN_RANGE(i915, 9, 10)(!!((&(i915)->__info)->gen_mask & ( 0 + 0 + ((( ~0UL) >> (64 - (((10)) - 1) - 1)) & ((~0UL) << (((9)) - 1))))))) { |
1621 | uncore->funcs.force_wake_get = fw_domains_get_with_fallback; |
1622 | uncore->funcs.force_wake_put = fw_domains_put; |
1623 | fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1624 | FORCEWAKE_RENDER_GEN9((const i915_reg_t){ .reg = (0xa278) }), |
1625 | FORCEWAKE_ACK_RENDER_GEN9((const i915_reg_t){ .reg = (0x0D84) })); |
1626 | fw_domain_init(uncore, FW_DOMAIN_ID_BLITTER, |
1627 | FORCEWAKE_BLITTER_GEN9((const i915_reg_t){ .reg = (0xa188) }), |
1628 | FORCEWAKE_ACK_BLITTER_GEN9((const i915_reg_t){ .reg = (0x130044) })); |
1629 | fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA, |
1630 | FORCEWAKE_MEDIA_GEN9((const i915_reg_t){ .reg = (0xa270) }), FORCEWAKE_ACK_MEDIA_GEN9((const i915_reg_t){ .reg = (0x0D88) })); |
1631 | } else if (IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW) || IS_CHERRYVIEW(i915)IS_PLATFORM(i915, INTEL_CHERRYVIEW)) { |
1632 | uncore->funcs.force_wake_get = fw_domains_get; |
1633 | uncore->funcs.force_wake_put = fw_domains_put; |
1634 | fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1635 | FORCEWAKE_VLV((const i915_reg_t){ .reg = (0x1300b0) }), FORCEWAKE_ACK_VLV((const i915_reg_t){ .reg = (0x1300b4) })); |
1636 | fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA, |
1637 | FORCEWAKE_MEDIA_VLV((const i915_reg_t){ .reg = (0x1300b8) }), FORCEWAKE_ACK_MEDIA_VLV((const i915_reg_t){ .reg = (0x1300bc) })); |
1638 | } else if (IS_HASWELL(i915)IS_PLATFORM(i915, INTEL_HASWELL) || IS_BROADWELL(i915)IS_PLATFORM(i915, INTEL_BROADWELL)) { |
1639 | uncore->funcs.force_wake_get = |
1640 | fw_domains_get_with_thread_status; |
1641 | uncore->funcs.force_wake_put = fw_domains_put; |
1642 | fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1643 | FORCEWAKE_MT((const i915_reg_t){ .reg = (0xa188) }), FORCEWAKE_ACK_HSW((const i915_reg_t){ .reg = (0x130044) })); |
1644 | } else if (IS_IVYBRIDGE(i915)IS_PLATFORM(i915, INTEL_IVYBRIDGE)) { |
1645 | u32 ecobus; |
1646 | |
1647 | /* IVB configs may use multi-threaded forcewake */ |
1648 | |
1649 | /* A small trick here - if the bios hasn't configured |
1650 | * MT forcewake, and if the device is in RC6, then |
1651 | * force_wake_mt_get will not wake the device and the |
1652 | * ECOBUS read will return zero. Which will be |
1653 | * (correctly) interpreted by the test below as MT |
1654 | * forcewake being disabled. |
1655 | */ |
1656 | uncore->funcs.force_wake_get = |
1657 | fw_domains_get_with_thread_status; |
1658 | uncore->funcs.force_wake_put = fw_domains_put; |
1659 | |
1660 | /* We need to init first for ECOBUS access and then |
1661 | * determine later if we want to reinit, in case of MT access is |
1662 | * not working. In this stage we don't know which flavour this |
1663 | * ivb is, so it is better to reset also the gen6 fw registers |
1664 | * before the ecobus check. |
1665 | */ |
1666 | |
1667 | __raw_uncore_write32(uncore, FORCEWAKE((const i915_reg_t){ .reg = (0xA18C) }), 0); |
1668 | __raw_posting_read(uncore, ECOBUS)((void)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = (0xa180) }))); |
1669 | |
1670 | ret = __fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1671 | FORCEWAKE_MT((const i915_reg_t){ .reg = (0xa188) }), FORCEWAKE_MT_ACK((const i915_reg_t){ .reg = (0x130040) })); |
1672 | if (ret) |
1673 | goto out; |
1674 | |
1675 | spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock); |
1676 | fw_domains_get_with_thread_status(uncore, FORCEWAKE_RENDER); |
1677 | ecobus = __raw_uncore_read32(uncore, ECOBUS((const i915_reg_t){ .reg = (0xa180) })); |
1678 | fw_domains_put(uncore, FORCEWAKE_RENDER); |
1679 | spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock); |
1680 | |
1681 | if (!(ecobus & FORCEWAKE_MT_ENABLE(1 << 5))) { |
1682 | drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n")do { } while(0); |
1683 | drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n")do { } while(0); |
1684 | fw_domain_fini(uncore, FW_DOMAIN_ID_RENDER); |
1685 | fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1686 | FORCEWAKE((const i915_reg_t){ .reg = (0xA18C) }), FORCEWAKE_ACK((const i915_reg_t){ .reg = (0x130090) })); |
1687 | } |
1688 | } else if (IS_GEN(i915, 6)(0 + (&(i915)->__info)->gen == (6))) { |
1689 | uncore->funcs.force_wake_get = |
1690 | fw_domains_get_with_thread_status; |
1691 | uncore->funcs.force_wake_put = fw_domains_put; |
1692 | fw_domain_init(uncore, FW_DOMAIN_ID_RENDER, |
1693 | FORCEWAKE((const i915_reg_t){ .reg = (0xA18C) }), FORCEWAKE_ACK((const i915_reg_t){ .reg = (0x130090) })); |
1694 | } |
1695 | |
1696 | #undef fw_domain_init |
1697 | |
1698 | /* All future platforms are expected to require complex power gating */ |
1699 | drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0)({ int __ret = !!((!ret && uncore->fw_domains == 0 )); if (__ret) printf("%s %s: " "%s", dev_driver_string(((& i915->drm))->dev), "", "drm_WARN_ON(" "!ret && uncore->fw_domains == 0" ")"); __builtin_expect(!!(__ret), 0); }); |
1700 | |
1701 | out: |
1702 | if (ret) |
1703 | intel_uncore_fw_domains_fini(uncore); |
1704 | |
1705 | return ret; |
1706 | } |
1707 | |
1708 | #define ASSIGN_FW_DOMAINS_TABLE(uncore, d){ (uncore)->fw_domains_table = (struct intel_forcewake_range *)(d); (uncore)->fw_domains_table_entries = (sizeof(((d)) ) / sizeof(((d))[0])); } \ |
1709 | { \ |
1710 | (uncore)->fw_domains_table = \ |
1711 | (struct intel_forcewake_range *)(d); \ |
1712 | (uncore)->fw_domains_table_entries = ARRAY_SIZE((d))(sizeof(((d))) / sizeof(((d))[0])); \ |
1713 | } |
1714 | |
1715 | static int i915_pmic_bus_access_notifier(struct notifier_block *nb, |
1716 | unsigned long action, void *data) |
1717 | { |
1718 | struct intel_uncore *uncore = container_of(nb,({ const __typeof( ((struct intel_uncore *)0)->pmic_bus_access_nb ) *__mptr = (nb); (struct intel_uncore *)( (char *)__mptr - __builtin_offsetof (struct intel_uncore, pmic_bus_access_nb) );}) |
1719 | struct intel_uncore, pmic_bus_access_nb)({ const __typeof( ((struct intel_uncore *)0)->pmic_bus_access_nb ) *__mptr = (nb); (struct intel_uncore *)( (char *)__mptr - __builtin_offsetof (struct intel_uncore, pmic_bus_access_nb) );}); |
1720 | |
1721 | switch (action) { |
1722 | case MBI_PMIC_BUS_ACCESS_BEGIN1: |
1723 | /* |
1724 | * forcewake all now to make sure that we don't need to do a |
1725 | * forcewake later which on systems where this notifier gets |
1726 | * called requires the punit to access to the shared pmic i2c |
1727 | * bus, which will be busy after this notification, leading to: |
1728 | * "render: timed out waiting for forcewake ack request." |
1729 | * errors. |
1730 | * |
1731 | * The notifier is unregistered during intel_runtime_suspend(), |
1732 | * so it's ok to access the HW here without holding a RPM |
1733 | * wake reference -> disable wakeref asserts for the time of |
1734 | * the access. |
1735 | */ |
1736 | disable_rpm_wakeref_asserts(uncore->rpm); |
1737 | intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); |
1738 | enable_rpm_wakeref_asserts(uncore->rpm); |
1739 | break; |
1740 | case MBI_PMIC_BUS_ACCESS_END2: |
1741 | intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); |
1742 | break; |
1743 | } |
1744 | |
1745 | return NOTIFY_OK1; |
1746 | } |
1747 | |
1748 | static int uncore_mmio_setup(struct intel_uncore *uncore) |
1749 | { |
1750 | struct drm_i915_privateinteldrm_softc *i915 = uncore->i915; |
1751 | struct pci_dev *pdev = i915->drm.pdev; |
1752 | int mmio_bar; |
1753 | int mmio_size; |
1754 | |
1755 | mmio_bar = IS_GEN(i915, 2)(0 + (&(i915)->__info)->gen == (2)) ? 1 : 0; |
Value stored to 'mmio_bar' is never read | |
1756 | /* |
1757 | * Before gen4, the registers and the GTT are behind different BARs. |
1758 | * However, from gen4 onwards, the registers and the GTT are shared |
1759 | * in the same BAR, so we want to restrict this ioremap from |
1760 | * clobbering the GTT which we want ioremap_wc instead. Fortunately, |
1761 | * the register BAR remains the same size for all the earlier |
1762 | * generations up to Ironlake. |
1763 | */ |
1764 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) < 5) |
1765 | mmio_size = 512 * 1024; |
1766 | else |
1767 | mmio_size = 2 * 1024 * 1024; |
1768 | #ifdef __linux__ |
1769 | uncore->regs = pci_iomap(pdev, mmio_bar, mmio_size); |
1770 | if (uncore->regs == NULL((void *)0)) { |
1771 | drm_err(&i915->drm, "failed to map registers\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to map registers\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__); |
1772 | return -EIO5; |
1773 | } |
1774 | #endif |
1775 | |
1776 | return 0; |
1777 | } |
1778 | |
1779 | static void uncore_mmio_cleanup(struct intel_uncore *uncore) |
1780 | { |
1781 | #ifdef __linux__ |
1782 | struct pci_dev *pdev = uncore->i915->drm.pdev; |
1783 | |
1784 | pci_iounmap(pdev, uncore->regs); |
1785 | #endif |
1786 | } |
1787 | |
1788 | void intel_uncore_init_early(struct intel_uncore *uncore, |
1789 | struct drm_i915_privateinteldrm_softc *i915) |
1790 | { |
1791 | mtx_init(&uncore->lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&uncore-> lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9) ? 0x9 : ((0x9)))); } while (0); |
1792 | uncore->i915 = i915; |
1793 | uncore->rpm = &i915->runtime_pm; |
1794 | uncore->debug = &i915->mmio_debug; |
1795 | } |
1796 | |
1797 | static void uncore_raw_init(struct intel_uncore *uncore) |
1798 | { |
1799 | GEM_BUG_ON(intel_uncore_has_forcewake(uncore))((void)0); |
1800 | |
1801 | if (intel_vgpu_active(uncore->i915)) { |
1802 | ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu)do { (uncore)->funcs.mmio_writeb = vgpu_write8; (uncore)-> funcs.mmio_writew = vgpu_write16; (uncore)->funcs.mmio_writel = vgpu_write32; } while (0); |
1803 | ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu)do { (uncore)->funcs.mmio_readb = vgpu_read8; (uncore)-> funcs.mmio_readw = vgpu_read16; (uncore)->funcs.mmio_readl = vgpu_read32; (uncore)->funcs.mmio_readq = vgpu_read64; } while (0); |
1804 | } else if (IS_GEN(uncore->i915, 5)(0 + (&(uncore->i915)->__info)->gen == (5))) { |
1805 | ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5)do { (uncore)->funcs.mmio_writeb = gen5_write8; (uncore)-> funcs.mmio_writew = gen5_write16; (uncore)->funcs.mmio_writel = gen5_write32; } while (0); |
1806 | ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5)do { (uncore)->funcs.mmio_readb = gen5_read8; (uncore)-> funcs.mmio_readw = gen5_read16; (uncore)->funcs.mmio_readl = gen5_read32; (uncore)->funcs.mmio_readq = gen5_read64; } while (0); |
1807 | } else { |
1808 | ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2)do { (uncore)->funcs.mmio_writeb = gen2_write8; (uncore)-> funcs.mmio_writew = gen2_write16; (uncore)->funcs.mmio_writel = gen2_write32; } while (0); |
1809 | ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2)do { (uncore)->funcs.mmio_readb = gen2_read8; (uncore)-> funcs.mmio_readw = gen2_read16; (uncore)->funcs.mmio_readl = gen2_read32; (uncore)->funcs.mmio_readq = gen2_read64; } while (0); |
1810 | } |
1811 | } |
1812 | |
1813 | static int uncore_forcewake_init(struct intel_uncore *uncore) |
1814 | { |
1815 | struct drm_i915_privateinteldrm_softc *i915 = uncore->i915; |
1816 | int ret; |
1817 | |
1818 | GEM_BUG_ON(!intel_uncore_has_forcewake(uncore))((void)0); |
1819 | |
1820 | ret = intel_uncore_fw_domains_init(uncore); |
1821 | if (ret) |
1822 | return ret; |
1823 | forcewake_early_sanitize(uncore, 0); |
1824 | |
1825 | if (IS_GEN_RANGE(i915, 6, 7)(!!((&(i915)->__info)->gen_mask & ( 0 + 0 + ((( ~0UL) >> (64 - (((7)) - 1) - 1)) & ((~0UL) << (((6)) - 1))))))) { |
1826 | ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6)do { do { ((uncore))->funcs.mmio_writeb = gen6_write8; ((uncore ))->funcs.mmio_writew = gen6_write16; ((uncore))->funcs .mmio_writel = gen6_write32; } while (0); (uncore)->funcs. write_fw_domains = gen6_reg_write_fw_domains; } while (0); |
1827 | |
1828 | if (IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW)) { |
1829 | ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges){ (uncore)->fw_domains_table = (struct intel_forcewake_range *)(__vlv_fw_ranges); (uncore)->fw_domains_table_entries = (sizeof(((__vlv_fw_ranges))) / sizeof(((__vlv_fw_ranges))[0] )); }; |
1830 | ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable)do { do { (uncore)->funcs.mmio_readb = fwtable_read8; (uncore )->funcs.mmio_readw = fwtable_read16; (uncore)->funcs.mmio_readl = fwtable_read32; (uncore)->funcs.mmio_readq = fwtable_read64 ; } while (0); (uncore)->funcs.read_fw_domains = fwtable_reg_read_fw_domains ; } while (0); |
1831 | } else { |
1832 | ASSIGN_READ_MMIO_VFUNCS(uncore, gen6)do { do { (uncore)->funcs.mmio_readb = gen6_read8; (uncore )->funcs.mmio_readw = gen6_read16; (uncore)->funcs.mmio_readl = gen6_read32; (uncore)->funcs.mmio_readq = gen6_read64; } while (0); (uncore)->funcs.read_fw_domains = gen6_reg_read_fw_domains ; } while (0); |
1833 | } |
1834 | } else if (IS_GEN(i915, 8)(0 + (&(i915)->__info)->gen == (8))) { |
1835 | if (IS_CHERRYVIEW(i915)IS_PLATFORM(i915, INTEL_CHERRYVIEW)) { |
1836 | ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges){ (uncore)->fw_domains_table = (struct intel_forcewake_range *)(__chv_fw_ranges); (uncore)->fw_domains_table_entries = (sizeof(((__chv_fw_ranges))) / sizeof(((__chv_fw_ranges))[0] )); }; |
1837 | ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable)do { do { ((uncore))->funcs.mmio_writeb = fwtable_write8; ( (uncore))->funcs.mmio_writew = fwtable_write16; ((uncore)) ->funcs.mmio_writel = fwtable_write32; } while (0); (uncore )->funcs.write_fw_domains = fwtable_reg_write_fw_domains; } while (0); |
1838 | ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable)do { do { (uncore)->funcs.mmio_readb = fwtable_read8; (uncore )->funcs.mmio_readw = fwtable_read16; (uncore)->funcs.mmio_readl = fwtable_read32; (uncore)->funcs.mmio_readq = fwtable_read64 ; } while (0); (uncore)->funcs.read_fw_domains = fwtable_reg_read_fw_domains ; } while (0); |
1839 | } else { |
1840 | ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen8)do { do { ((uncore))->funcs.mmio_writeb = gen8_write8; ((uncore ))->funcs.mmio_writew = gen8_write16; ((uncore))->funcs .mmio_writel = gen8_write32; } while (0); (uncore)->funcs. write_fw_domains = gen8_reg_write_fw_domains; } while (0); |
1841 | ASSIGN_READ_MMIO_VFUNCS(uncore, gen6)do { do { (uncore)->funcs.mmio_readb = gen6_read8; (uncore )->funcs.mmio_readw = gen6_read16; (uncore)->funcs.mmio_readl = gen6_read32; (uncore)->funcs.mmio_readq = gen6_read64; } while (0); (uncore)->funcs.read_fw_domains = gen6_reg_read_fw_domains ; } while (0); |
1842 | } |
1843 | } else if (IS_GEN_RANGE(i915, 9, 10)(!!((&(i915)->__info)->gen_mask & ( 0 + 0 + ((( ~0UL) >> (64 - (((10)) - 1) - 1)) & ((~0UL) << (((9)) - 1))))))) { |
1844 | ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges){ (uncore)->fw_domains_table = (struct intel_forcewake_range *)(__gen9_fw_ranges); (uncore)->fw_domains_table_entries = (sizeof(((__gen9_fw_ranges))) / sizeof(((__gen9_fw_ranges))[ 0])); }; |
1845 | ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable)do { do { ((uncore))->funcs.mmio_writeb = fwtable_write8; ( (uncore))->funcs.mmio_writew = fwtable_write16; ((uncore)) ->funcs.mmio_writel = fwtable_write32; } while (0); (uncore )->funcs.write_fw_domains = fwtable_reg_write_fw_domains; } while (0); |
1846 | ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable)do { do { (uncore)->funcs.mmio_readb = fwtable_read8; (uncore )->funcs.mmio_readw = fwtable_read16; (uncore)->funcs.mmio_readl = fwtable_read32; (uncore)->funcs.mmio_readq = fwtable_read64 ; } while (0); (uncore)->funcs.read_fw_domains = fwtable_reg_read_fw_domains ; } while (0); |
1847 | } else if (IS_GEN(i915, 11)(0 + (&(i915)->__info)->gen == (11))) { |
1848 | ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges){ (uncore)->fw_domains_table = (struct intel_forcewake_range *)(__gen11_fw_ranges); (uncore)->fw_domains_table_entries = (sizeof(((__gen11_fw_ranges))) / sizeof(((__gen11_fw_ranges ))[0])); }; |
1849 | ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen11_fwtable)do { do { ((uncore))->funcs.mmio_writeb = gen11_fwtable_write8 ; ((uncore))->funcs.mmio_writew = gen11_fwtable_write16; ( (uncore))->funcs.mmio_writel = gen11_fwtable_write32; } while (0); (uncore)->funcs.write_fw_domains = gen11_fwtable_reg_write_fw_domains ; } while (0); |
1850 | ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable)do { do { (uncore)->funcs.mmio_readb = gen11_fwtable_read8 ; (uncore)->funcs.mmio_readw = gen11_fwtable_read16; (uncore )->funcs.mmio_readl = gen11_fwtable_read32; (uncore)->funcs .mmio_readq = gen11_fwtable_read64; } while (0); (uncore)-> funcs.read_fw_domains = gen11_fwtable_reg_read_fw_domains; } while (0); |
1851 | } else { |
1852 | ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges){ (uncore)->fw_domains_table = (struct intel_forcewake_range *)(__gen12_fw_ranges); (uncore)->fw_domains_table_entries = (sizeof(((__gen12_fw_ranges))) / sizeof(((__gen12_fw_ranges ))[0])); }; |
1853 | ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen12_fwtable)do { do { ((uncore))->funcs.mmio_writeb = gen12_fwtable_write8 ; ((uncore))->funcs.mmio_writew = gen12_fwtable_write16; ( (uncore))->funcs.mmio_writel = gen12_fwtable_write32; } while (0); (uncore)->funcs.write_fw_domains = gen12_fwtable_reg_write_fw_domains ; } while (0); |
1854 | ASSIGN_READ_MMIO_VFUNCS(uncore, gen12_fwtable)do { do { (uncore)->funcs.mmio_readb = gen12_fwtable_read8 ; (uncore)->funcs.mmio_readw = gen12_fwtable_read16; (uncore )->funcs.mmio_readl = gen12_fwtable_read32; (uncore)->funcs .mmio_readq = gen12_fwtable_read64; } while (0); (uncore)-> funcs.read_fw_domains = gen12_fwtable_reg_read_fw_domains; } while (0); |
1855 | } |
1856 | |
1857 | uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier; |
1858 | iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb)0; |
1859 | |
1860 | return 0; |
1861 | } |
1862 | |
1863 | int intel_uncore_init_mmio(struct intel_uncore *uncore) |
1864 | { |
1865 | struct drm_i915_privateinteldrm_softc *i915 = uncore->i915; |
1866 | int ret; |
1867 | |
1868 | ret = uncore_mmio_setup(uncore); |
1869 | if (ret) |
1870 | return ret; |
1871 | |
1872 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) > 5 && !intel_vgpu_active(i915)) |
1873 | uncore->flags |= UNCORE_HAS_FORCEWAKE(1UL << (0)); |
1874 | |
1875 | if (!intel_uncore_has_forcewake(uncore)) { |
1876 | uncore_raw_init(uncore); |
1877 | } else { |
1878 | ret = uncore_forcewake_init(uncore); |
1879 | if (ret) |
1880 | goto out_mmio_cleanup; |
1881 | } |
1882 | |
1883 | /* make sure fw funcs are set if and only if we have fw*/ |
1884 | GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.force_wake_get)((void)0); |
1885 | GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.force_wake_put)((void)0); |
1886 | GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains)((void)0); |
1887 | GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains)((void)0); |
1888 | |
1889 | if (HAS_FPGA_DBG_UNCLAIMED(i915)((&(i915)->__info)->has_fpga_dbg)) |
1890 | uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED(1UL << (1)); |
1891 | |
1892 | if (IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW) || IS_CHERRYVIEW(i915)IS_PLATFORM(i915, INTEL_CHERRYVIEW)) |
1893 | uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED(1UL << (2)); |
1894 | |
1895 | if (IS_GEN_RANGE(i915, 6, 7)(!!((&(i915)->__info)->gen_mask & ( 0 + 0 + ((( ~0UL) >> (64 - (((7)) - 1) - 1)) & ((~0UL) << (((6)) - 1))))))) |
1896 | uncore->flags |= UNCORE_HAS_FIFO(1UL << (3)); |
1897 | |
1898 | /* clear out unclaimed reg detection bit */ |
1899 | if (intel_uncore_unclaimed_mmio(uncore)) |
1900 | drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n")drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "unclaimed mmio detected on uncore init, clearing\n" ); |
1901 | |
1902 | return 0; |
1903 | |
1904 | out_mmio_cleanup: |
1905 | uncore_mmio_cleanup(uncore); |
1906 | |
1907 | return ret; |
1908 | } |
1909 | |
1910 | /* |
1911 | * We might have detected that some engines are fused off after we initialized |
1912 | * the forcewake domains. Prune them, to make sure they only reference existing |
1913 | * engines. |
1914 | */ |
1915 | void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore, |
1916 | struct intel_gt *gt) |
1917 | { |
1918 | enum forcewake_domains fw_domains = uncore->fw_domains; |
1919 | enum forcewake_domain_id domain_id; |
1920 | int i; |
1921 | |
1922 | if (!intel_uncore_has_forcewake(uncore) || INTEL_GEN(uncore->i915)((&(uncore->i915)->__info)->gen) < 11) |
1923 | return; |
1924 | |
1925 | for (i = 0; i < I915_MAX_VCS4; i++) { |
1926 | domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i; |
1927 | |
1928 | if (HAS_ENGINE(gt, _VCS(i))(((gt)->info.engine_mask) & (1UL << ((VCS0 + (i) ))))) |
1929 | continue; |
1930 | |
1931 | if (fw_domains & BIT(domain_id)(1UL << (domain_id))) |
1932 | fw_domain_fini(uncore, domain_id); |
1933 | } |
1934 | |
1935 | for (i = 0; i < I915_MAX_VECS2; i++) { |
1936 | domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i; |
1937 | |
1938 | if (HAS_ENGINE(gt, _VECS(i))(((gt)->info.engine_mask) & (1UL << ((VECS0 + (i )))))) |
1939 | continue; |
1940 | |
1941 | if (fw_domains & BIT(domain_id)(1UL << (domain_id))) |
1942 | fw_domain_fini(uncore, domain_id); |
1943 | } |
1944 | } |
1945 | |
1946 | void intel_uncore_fini_mmio(struct intel_uncore *uncore) |
1947 | { |
1948 | if (intel_uncore_has_forcewake(uncore)) { |
1949 | iosf_mbi_punit_acquire(); |
1950 | iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(0 |
1951 | &uncore->pmic_bus_access_nb)0; |
1952 | intel_uncore_forcewake_reset(uncore); |
1953 | intel_uncore_fw_domains_fini(uncore); |
1954 | iosf_mbi_punit_release(); |
1955 | } |
1956 | |
1957 | uncore_mmio_cleanup(uncore); |
1958 | } |
1959 | |
1960 | static const struct reg_whitelist { |
1961 | i915_reg_t offset_ldw; |
1962 | i915_reg_t offset_udw; |
1963 | u16 gen_mask; |
1964 | u8 size; |
1965 | } reg_read_whitelist[] = { { |
1966 | .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x358) }), |
1967 | .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x358 + 4) }), |
1968 | .gen_mask = INTEL_GEN_MASK(4, 12)( 0 + 0 + (((~0UL) >> (64 - ((12) - 1) - 1)) & ((~0UL ) << ((4) - 1)))), |
1969 | .size = 8 |
1970 | } }; |
1971 | |
1972 | int i915_reg_read_ioctl(struct drm_device *dev, |
1973 | void *data, struct drm_file *file) |
1974 | { |
1975 | struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev); |
1976 | struct intel_uncore *uncore = &i915->uncore; |
1977 | struct drm_i915_reg_read *reg = data; |
1978 | struct reg_whitelist const *entry; |
1979 | intel_wakeref_t wakeref; |
1980 | unsigned int flags; |
1981 | int remain; |
1982 | int ret = 0; |
1983 | |
1984 | entry = reg_read_whitelist; |
1985 | remain = ARRAY_SIZE(reg_read_whitelist)(sizeof((reg_read_whitelist)) / sizeof((reg_read_whitelist)[0 ])); |
1986 | while (remain) { |
1987 | u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw); |
1988 | |
1989 | GEM_BUG_ON(!is_power_of_2(entry->size))((void)0); |
1990 | GEM_BUG_ON(entry->size > 8)((void)0); |
1991 | GEM_BUG_ON(entry_offset & (entry->size - 1))((void)0); |
1992 | |
1993 | if (INTEL_INFO(i915)(&(i915)->__info)->gen_mask & entry->gen_mask && |
1994 | entry_offset == (reg->offset & -entry->size)) |
1995 | break; |
1996 | entry++; |
1997 | remain--; |
1998 | } |
1999 | |
2000 | if (!remain) |
2001 | return -EINVAL22; |
2002 | |
2003 | flags = reg->offset & (entry->size - 1); |
2004 | |
2005 | with_intel_runtime_pm(&i915->runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(&i915->runtime_pm ); (wakeref); intel_runtime_pm_put((&i915->runtime_pm) , (wakeref)), (wakeref) = 0) { |
2006 | if (entry->size == 8 && flags == I915_REG_READ_8B_WA(1ul << 0)) |
2007 | reg->val = intel_uncore_read64_2x32(uncore, |
2008 | entry->offset_ldw, |
2009 | entry->offset_udw); |
2010 | else if (entry->size == 8 && flags == 0) |
2011 | reg->val = intel_uncore_read64(uncore, |
2012 | entry->offset_ldw); |
2013 | else if (entry->size == 4 && flags == 0) |
2014 | reg->val = intel_uncore_read(uncore, entry->offset_ldw); |
2015 | else if (entry->size == 2 && flags == 0) |
2016 | reg->val = intel_uncore_read16(uncore, |
2017 | entry->offset_ldw); |
2018 | else if (entry->size == 1 && flags == 0) |
2019 | reg->val = intel_uncore_read8(uncore, |
2020 | entry->offset_ldw); |
2021 | else |
2022 | ret = -EINVAL22; |
2023 | } |
2024 | |
2025 | return ret; |
2026 | } |
2027 | |
2028 | /** |
2029 | * __intel_wait_for_register_fw - wait until register matches expected state |
2030 | * @uncore: the struct intel_uncore |
2031 | * @reg: the register to read |
2032 | * @mask: mask to apply to register value |
2033 | * @value: expected value |
2034 | * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait |
2035 | * @slow_timeout_ms: slow timeout in millisecond |
2036 | * @out_value: optional placeholder to hold registry value |
2037 | * |
2038 | * This routine waits until the target register @reg contains the expected |
2039 | * @value after applying the @mask, i.e. it waits until :: |
2040 | * |
2041 | * (I915_READ_FW(reg) & mask) == value |
2042 | * |
2043 | * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. |
2044 | * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us |
2045 | * must be not larger than 20,0000 microseconds. |
2046 | * |
2047 | * Note that this routine assumes the caller holds forcewake asserted, it is |
2048 | * not suitable for very long waits. See intel_wait_for_register() if you |
2049 | * wish to wait without holding forcewake for the duration (i.e. you expect |
2050 | * the wait to be slow). |
2051 | * |
2052 | * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. |
2053 | */ |
2054 | int __intel_wait_for_register_fw(struct intel_uncore *uncore, |
2055 | i915_reg_t reg, |
2056 | u32 mask, |
2057 | u32 value, |
2058 | unsigned int fast_timeout_us, |
2059 | unsigned int slow_timeout_ms, |
2060 | u32 *out_value) |
2061 | { |
2062 | u32 reg_value = 0; |
2063 | #define done (((reg_value = intel_uncore_read_fw(uncore, reg)__raw_uncore_read32(uncore, reg)) & mask) == value) |
2064 | int ret; |
2065 | |
2066 | /* Catch any overuse of this function */ |
2067 | might_sleep_if(slow_timeout_ms)do { if (slow_timeout_ms) assertwaitok(); } while (0); |
2068 | GEM_BUG_ON(fast_timeout_us > 20000)((void)0); |
2069 | GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms)((void)0); |
2070 | |
2071 | ret = -ETIMEDOUT60; |
2072 | if (fast_timeout_us && fast_timeout_us <= 20000) |
2073 | ret = _wait_for_atomic(done, fast_timeout_us, 0)({ int cpu, ret, timeout = (fast_timeout_us) * 1000; u64 base ; do { } while (0); if (!(0)) { ; cpu = (({struct cpu_info *__ci ; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(0)) ; __asm volatile("" : : : "memory"); if (done) { ret = 0; break ; } if (now - base >= timeout) { ret = -60; break; } cpu_relax (); if (!(0)) { ; if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile( "movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cpuid); base = local_clock (); } } } ret; }); |
2074 | if (ret && slow_timeout_ms) |
2075 | ret = wait_for(done, slow_timeout_ms)({ const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (((slow_timeout_ms) * 1000))); long wait__ = ((10)); int ret__ ; assertwaitok(); for (;;) { const _Bool expired__ = ktime_after (ktime_get_raw(), end__); ; __asm volatile("" : : : "memory") ; if (((done))) { ret__ = 0; break; } if (expired__) { ret__ = -60; break; } usleep_range(wait__, wait__ * 2); if (wait__ < ((1000))) wait__ <<= 1; } ret__; }); |
2076 | |
2077 | if (out_value) |
2078 | *out_value = reg_value; |
2079 | |
2080 | return ret; |
2081 | #undef done |
2082 | } |
2083 | |
2084 | /** |
2085 | * __intel_wait_for_register - wait until register matches expected state |
2086 | * @uncore: the struct intel_uncore |
2087 | * @reg: the register to read |
2088 | * @mask: mask to apply to register value |
2089 | * @value: expected value |
2090 | * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait |
2091 | * @slow_timeout_ms: slow timeout in millisecond |
2092 | * @out_value: optional placeholder to hold registry value |
2093 | * |
2094 | * This routine waits until the target register @reg contains the expected |
2095 | * @value after applying the @mask, i.e. it waits until :: |
2096 | * |
2097 | * (I915_READ(reg) & mask) == value |
2098 | * |
2099 | * Otherwise, the wait will timeout after @timeout_ms milliseconds. |
2100 | * |
2101 | * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. |
2102 | */ |
2103 | int __intel_wait_for_register(struct intel_uncore *uncore, |
2104 | i915_reg_t reg, |
2105 | u32 mask, |
2106 | u32 value, |
2107 | unsigned int fast_timeout_us, |
2108 | unsigned int slow_timeout_ms, |
2109 | u32 *out_value) |
2110 | { |
2111 | unsigned fw = |
2112 | intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ(1)); |
2113 | u32 reg_value; |
2114 | int ret; |
2115 | |
2116 | might_sleep_if(slow_timeout_ms)do { if (slow_timeout_ms) assertwaitok(); } while (0); |
2117 | |
2118 | spin_lock_irq(&uncore->lock)mtx_enter(&uncore->lock); |
2119 | intel_uncore_forcewake_get__locked(uncore, fw); |
2120 | |
2121 | ret = __intel_wait_for_register_fw(uncore, |
2122 | reg, mask, value, |
2123 | fast_timeout_us, 0, ®_value); |
2124 | |
2125 | intel_uncore_forcewake_put__locked(uncore, fw); |
2126 | spin_unlock_irq(&uncore->lock)mtx_leave(&uncore->lock); |
2127 | |
2128 | if (ret && slow_timeout_ms) |
2129 | ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,({ const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (slow_timeout_ms * 1000)); long wait__ = (10); int ret__; assertwaitok (); for (;;) { const _Bool expired__ = ktime_after(ktime_get_raw (), end__); reg_value = intel_uncore_read_notrace(uncore, reg ); __asm volatile("" : : : "memory"); if ((reg_value & mask ) == value) { ret__ = 0; break; } if (expired__) { ret__ = -60 ; break; } usleep_range(wait__, wait__ * 2); if (wait__ < ( 1000)) wait__ <<= 1; } ret__; }) |
2130 | reg),({ const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (slow_timeout_ms * 1000)); long wait__ = (10); int ret__; assertwaitok (); for (;;) { const _Bool expired__ = ktime_after(ktime_get_raw (), end__); reg_value = intel_uncore_read_notrace(uncore, reg ); __asm volatile("" : : : "memory"); if ((reg_value & mask ) == value) { ret__ = 0; break; } if (expired__) { ret__ = -60 ; break; } usleep_range(wait__, wait__ * 2); if (wait__ < ( 1000)) wait__ <<= 1; } ret__; }) |
2131 | (reg_value & mask) == value,({ const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (slow_timeout_ms * 1000)); long wait__ = (10); int ret__; assertwaitok (); for (;;) { const _Bool expired__ = ktime_after(ktime_get_raw (), end__); reg_value = intel_uncore_read_notrace(uncore, reg ); __asm volatile("" : : : "memory"); if ((reg_value & mask ) == value) { ret__ = 0; break; } if (expired__) { ret__ = -60 ; break; } usleep_range(wait__, wait__ * 2); if (wait__ < ( 1000)) wait__ <<= 1; } ret__; }) |
2132 | slow_timeout_ms * 1000, 10, 1000)({ const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (slow_timeout_ms * 1000)); long wait__ = (10); int ret__; assertwaitok (); for (;;) { const _Bool expired__ = ktime_after(ktime_get_raw (), end__); reg_value = intel_uncore_read_notrace(uncore, reg ); __asm volatile("" : : : "memory"); if ((reg_value & mask ) == value) { ret__ = 0; break; } if (expired__) { ret__ = -60 ; break; } usleep_range(wait__, wait__ * 2); if (wait__ < ( 1000)) wait__ <<= 1; } ret__; }); |
2133 | |
2134 | /* just trace the final value */ |
2135 | trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); |
2136 | |
2137 | if (out_value) |
2138 | *out_value = reg_value; |
2139 | |
2140 | return ret; |
2141 | } |
2142 | |
2143 | bool_Bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore) |
2144 | { |
2145 | bool_Bool ret; |
2146 | |
2147 | spin_lock_irq(&uncore->debug->lock)mtx_enter(&uncore->debug->lock); |
2148 | ret = check_for_unclaimed_mmio(uncore); |
2149 | spin_unlock_irq(&uncore->debug->lock)mtx_leave(&uncore->debug->lock); |
2150 | |
2151 | return ret; |
2152 | } |
2153 | |
2154 | bool_Bool |
2155 | intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore) |
2156 | { |
2157 | bool_Bool ret = false0; |
2158 | |
2159 | spin_lock_irq(&uncore->debug->lock)mtx_enter(&uncore->debug->lock); |
2160 | |
2161 | if (unlikely(uncore->debug->unclaimed_mmio_check <= 0)__builtin_expect(!!(uncore->debug->unclaimed_mmio_check <= 0), 0)) |
2162 | goto out; |
2163 | |
2164 | if (unlikely(check_for_unclaimed_mmio(uncore))__builtin_expect(!!(check_for_unclaimed_mmio(uncore)), 0)) { |
2165 | if (!uncore->i915->params.mmio_debug) { |
2166 | drm_dbg(&uncore->i915->drm,drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "Unclaimed register detected, " "enabling oneshot unclaimed register reporting. " "Please use i915.mmio_debug=N for more information.\n") |
2167 | "Unclaimed register detected, "drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "Unclaimed register detected, " "enabling oneshot unclaimed register reporting. " "Please use i915.mmio_debug=N for more information.\n") |
2168 | "enabling oneshot unclaimed register reporting. "drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "Unclaimed register detected, " "enabling oneshot unclaimed register reporting. " "Please use i915.mmio_debug=N for more information.\n") |
2169 | "Please use i915.mmio_debug=N for more information.\n")drm_dev_dbg((&uncore->i915->drm)->dev, DRM_UT_DRIVER , "Unclaimed register detected, " "enabling oneshot unclaimed register reporting. " "Please use i915.mmio_debug=N for more information.\n"); |
2170 | uncore->i915->params.mmio_debug++; |
2171 | } |
2172 | uncore->debug->unclaimed_mmio_check--; |
2173 | ret = true1; |
2174 | } |
2175 | |
2176 | out: |
2177 | spin_unlock_irq(&uncore->debug->lock)mtx_leave(&uncore->debug->lock); |
2178 | |
2179 | return ret; |
2180 | } |
2181 | |
2182 | /** |
2183 | * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access |
2184 | * a register |
2185 | * @uncore: pointer to struct intel_uncore |
2186 | * @reg: register in question |
2187 | * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE |
2188 | * |
2189 | * Returns a set of forcewake domains required to be taken with for example |
2190 | * intel_uncore_forcewake_get for the specified register to be accessible in the |
2191 | * specified mode (read, write or read/write) with raw mmio accessors. |
2192 | * |
2193 | * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the |
2194 | * callers to do FIFO management on their own or risk losing writes. |
2195 | */ |
2196 | enum forcewake_domains |
2197 | intel_uncore_forcewake_for_reg(struct intel_uncore *uncore, |
2198 | i915_reg_t reg, unsigned int op) |
2199 | { |
2200 | enum forcewake_domains fw_domains = 0; |
2201 | |
2202 | drm_WARN_ON(&uncore->i915->drm, !op)({ int __ret = !!((!op)); if (__ret) printf("%s %s: " "%s", dev_driver_string (((&uncore->i915->drm))->dev), "", "drm_WARN_ON(" "!op" ")"); __builtin_expect(!!(__ret), 0); }); |
2203 | |
2204 | if (!intel_uncore_has_forcewake(uncore)) |
2205 | return 0; |
2206 | |
2207 | if (op & FW_REG_READ(1)) |
2208 | fw_domains = uncore->funcs.read_fw_domains(uncore, reg); |
2209 | |
2210 | if (op & FW_REG_WRITE(2)) |
2211 | fw_domains |= uncore->funcs.write_fw_domains(uncore, reg); |
2212 | |
2213 | drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains)({ int __ret = !!((fw_domains & ~uncore->fw_domains)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&uncore ->i915->drm))->dev), "", "drm_WARN_ON(" "fw_domains & ~uncore->fw_domains" ")"); __builtin_expect(!!(__ret), 0); }); |
2214 | |
2215 | return fw_domains; |
2216 | } |
2217 | |
2218 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0 |
2219 | #include "selftests/mock_uncore.c" |
2220 | #include "selftests/intel_uncore.c" |
2221 | #endif |