File: | dev/pci/drm/i915/gt/intel_reset.c |
Warning: | line 800, column 3 Value stored to 'node' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2008-2018 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/sched/mm.h> |
7 | #include <linux/stop_machine.h> |
8 | #include <linux/string_helpers.h> |
9 | |
10 | #include "display/intel_display.h" |
11 | #include "display/intel_overlay.h" |
12 | |
13 | #include "gem/i915_gem_context.h" |
14 | |
15 | #include "gt/intel_gt_regs.h" |
16 | |
17 | #include "i915_drv.h" |
18 | #include "i915_file_private.h" |
19 | #include "i915_gpu_error.h" |
20 | #include "i915_irq.h" |
21 | #include "intel_breadcrumbs.h" |
22 | #include "intel_engine_pm.h" |
23 | #include "intel_engine_regs.h" |
24 | #include "intel_gt.h" |
25 | #include "intel_gt_pm.h" |
26 | #include "intel_gt_requests.h" |
27 | #include "intel_mchbar_regs.h" |
28 | #include "intel_pci_config.h" |
29 | #include "intel_reset.h" |
30 | |
31 | #include "uc/intel_guc.h" |
32 | |
33 | #define RESET_MAX_RETRIES3 3 |
34 | |
35 | /* XXX How to handle concurrent GGTT updates using tiling registers? */ |
36 | #define RESET_UNDER_STOP_MACHINE0 0 |
37 | |
38 | static void rmw_set_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 set) |
39 | { |
40 | intel_uncore_rmw_fw(uncore, reg, 0, set); |
41 | } |
42 | |
43 | static void rmw_clear_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 clr) |
44 | { |
45 | intel_uncore_rmw_fw(uncore, reg, clr, 0); |
46 | } |
47 | |
48 | static void client_mark_guilty(struct i915_gem_context *ctx, bool_Bool banned) |
49 | { |
50 | struct drm_i915_file_private *file_priv = ctx->file_priv; |
51 | unsigned long prev_hang; |
52 | unsigned int score; |
53 | |
54 | if (IS_ERR_OR_NULL(file_priv)) |
55 | return; |
56 | |
57 | score = 0; |
58 | if (banned) |
59 | score = I915_CLIENT_SCORE_CONTEXT_BAN3; |
60 | |
61 | prev_hang = xchg(&file_priv->hang_timestamp, jiffies)__sync_lock_test_and_set(&file_priv->hang_timestamp, jiffies ); |
62 | if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES)time_after(prev_hang + (60 * hz),jiffies)) |
63 | score += I915_CLIENT_SCORE_HANG_FAST1; |
64 | |
65 | if (score) { |
66 | atomic_add(score, &file_priv->ban_score)__sync_fetch_and_add(&file_priv->ban_score, score); |
67 | |
68 | drm_dbg(&ctx->i915->drm,__drm_dev_dbg(((void *)0), (&ctx->i915->drm) ? (& ctx->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "client %s: gained %u ban score, now %u\n" , ctx->name, score, ({ typeof(*(&file_priv->ban_score )) __tmp = *(volatile typeof(*(&file_priv->ban_score)) *)&(*(&file_priv->ban_score)); membar_datadep_consumer (); __tmp; })) |
69 | "client %s: gained %u ban score, now %u\n",__drm_dev_dbg(((void *)0), (&ctx->i915->drm) ? (& ctx->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "client %s: gained %u ban score, now %u\n" , ctx->name, score, ({ typeof(*(&file_priv->ban_score )) __tmp = *(volatile typeof(*(&file_priv->ban_score)) *)&(*(&file_priv->ban_score)); membar_datadep_consumer (); __tmp; })) |
70 | ctx->name, score,__drm_dev_dbg(((void *)0), (&ctx->i915->drm) ? (& ctx->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "client %s: gained %u ban score, now %u\n" , ctx->name, score, ({ typeof(*(&file_priv->ban_score )) __tmp = *(volatile typeof(*(&file_priv->ban_score)) *)&(*(&file_priv->ban_score)); membar_datadep_consumer (); __tmp; })) |
71 | atomic_read(&file_priv->ban_score))__drm_dev_dbg(((void *)0), (&ctx->i915->drm) ? (& ctx->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "client %s: gained %u ban score, now %u\n" , ctx->name, score, ({ typeof(*(&file_priv->ban_score )) __tmp = *(volatile typeof(*(&file_priv->ban_score)) *)&(*(&file_priv->ban_score)); membar_datadep_consumer (); __tmp; })); |
72 | } |
73 | } |
74 | |
75 | static bool_Bool mark_guilty(struct i915_request *rq) |
76 | { |
77 | struct i915_gem_context *ctx; |
78 | unsigned long prev_hang; |
79 | bool_Bool banned; |
80 | int i; |
81 | |
82 | if (intel_context_is_closed(rq->context)) |
83 | return true1; |
84 | |
85 | rcu_read_lock(); |
86 | ctx = rcu_dereference(rq->context->gem_context)(rq->context->gem_context); |
87 | if (ctx && !kref_get_unless_zero(&ctx->ref)) |
88 | ctx = NULL((void *)0); |
89 | rcu_read_unlock(); |
90 | if (!ctx) |
91 | return intel_context_is_banned(rq->context); |
92 | |
93 | atomic_inc(&ctx->guilty_count)__sync_fetch_and_add(&ctx->guilty_count, 1); |
94 | |
95 | /* Cool contexts are too cool to be banned! (Used for reset testing.) */ |
96 | if (!i915_gem_context_is_bannable(ctx)) { |
97 | banned = false0; |
98 | goto out; |
99 | } |
100 | |
101 | drm_notice(&ctx->i915->drm,printf("drm:pid%d:%s *NOTICE* " "[drm] " "%s context reset due to GPU hang\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx-> name) |
102 | "%s context reset due to GPU hang\n",printf("drm:pid%d:%s *NOTICE* " "[drm] " "%s context reset due to GPU hang\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx-> name) |
103 | ctx->name)printf("drm:pid%d:%s *NOTICE* " "[drm] " "%s context reset due to GPU hang\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx-> name); |
104 | |
105 | /* Record the timestamp for the last N hangs */ |
106 | prev_hang = ctx->hang_timestamp[0]; |
107 | for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp)(sizeof((ctx->hang_timestamp)) / sizeof((ctx->hang_timestamp )[0])) - 1; i++) |
108 | ctx->hang_timestamp[i] = ctx->hang_timestamp[i + 1]; |
109 | ctx->hang_timestamp[i] = jiffies; |
110 | |
111 | /* If we have hung N+1 times in rapid succession, we ban the context! */ |
112 | banned = !i915_gem_context_is_recoverable(ctx); |
113 | if (time_before(jiffies, prev_hang + CONTEXT_FAST_HANG_JIFFIES)time_after(prev_hang + (120 * hz),jiffies)) |
114 | banned = true1; |
115 | if (banned) |
116 | drm_dbg(&ctx->i915->drm, "context %s: guilty %d, banned\n",__drm_dev_dbg(((void *)0), (&ctx->i915->drm) ? (& ctx->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "context %s: guilty %d, banned\n" , ctx->name, ({ typeof(*(&ctx->guilty_count)) __tmp = *(volatile typeof(*(&ctx->guilty_count)) *)&(*( &ctx->guilty_count)); membar_datadep_consumer(); __tmp ; })) |
117 | ctx->name, atomic_read(&ctx->guilty_count))__drm_dev_dbg(((void *)0), (&ctx->i915->drm) ? (& ctx->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "context %s: guilty %d, banned\n" , ctx->name, ({ typeof(*(&ctx->guilty_count)) __tmp = *(volatile typeof(*(&ctx->guilty_count)) *)&(*( &ctx->guilty_count)); membar_datadep_consumer(); __tmp ; })); |
118 | |
119 | client_mark_guilty(ctx, banned); |
120 | |
121 | out: |
122 | i915_gem_context_put(ctx); |
123 | return banned; |
124 | } |
125 | |
126 | static void mark_innocent(struct i915_request *rq) |
127 | { |
128 | struct i915_gem_context *ctx; |
129 | |
130 | rcu_read_lock(); |
131 | ctx = rcu_dereference(rq->context->gem_context)(rq->context->gem_context); |
132 | if (ctx) |
133 | atomic_inc(&ctx->active_count)__sync_fetch_and_add(&ctx->active_count, 1); |
134 | rcu_read_unlock(); |
135 | } |
136 | |
137 | void __i915_request_reset(struct i915_request *rq, bool_Bool guilty) |
138 | { |
139 | bool_Bool banned = false0; |
140 | |
141 | RQ_TRACE(rq, "guilty? %s\n", str_yes_no(guilty))do { const struct i915_request *rq__ = (rq); do { const struct intel_engine_cs *e__ __attribute__((__unused__)) = (rq__-> engine); do { } while (0); } while (0); } while (0); |
142 | GEM_BUG_ON(__i915_request_is_complete(rq))((void)0); |
143 | |
144 | rcu_read_lock(); /* protect the GEM context */ |
145 | if (guilty) { |
146 | i915_request_set_error_once(rq, -EIO5); |
147 | __i915_request_skip(rq); |
148 | banned = mark_guilty(rq); |
149 | } else { |
150 | i915_request_set_error_once(rq, -EAGAIN35); |
151 | mark_innocent(rq); |
152 | } |
153 | rcu_read_unlock(); |
154 | |
155 | if (banned) |
156 | intel_context_ban(rq->context, rq); |
157 | } |
158 | |
159 | static bool_Bool i915_in_reset(struct pci_dev *pdev) |
160 | { |
161 | u8 gdrst; |
162 | |
163 | pci_read_config_byte(pdev, I915_GDRST0xc0, &gdrst); |
164 | return gdrst & GRDOM_RESET_STATUS(1 << 1); |
165 | } |
166 | |
167 | static int i915_do_reset(struct intel_gt *gt, |
168 | intel_engine_mask_t engine_mask, |
169 | unsigned int retry) |
170 | { |
171 | struct pci_dev *pdev = gt->i915->drm.pdev; |
172 | int err; |
173 | |
174 | /* Assert reset for at least 20 usec, and wait for acknowledgement. */ |
175 | pci_write_config_byte(pdev, I915_GDRST0xc0, GRDOM_RESET_ENABLE(1 << 0)); |
176 | udelay(50); |
177 | err = wait_for_atomic(i915_in_reset(pdev), 50)({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { preempt_disable(); cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci ) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci; })->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) preempt_enable(); __asm volatile("" : : : "memory"); if (((i915_in_reset(pdev)))) { ret = 0; break ; } if (now - base >= timeout) { ret = -60; break; } cpu_relax (); if (!(1)) { preempt_disable(); if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }); |
178 | |
179 | /* Clear the reset request. */ |
180 | pci_write_config_byte(pdev, I915_GDRST0xc0, 0); |
181 | udelay(50); |
182 | if (!err) |
183 | err = wait_for_atomic(!i915_in_reset(pdev), 50)({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { preempt_disable(); cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci ) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci; })->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) preempt_enable(); __asm volatile("" : : : "memory"); if (((!i915_in_reset(pdev)))) { ret = 0; break ; } if (now - base >= timeout) { ret = -60; break; } cpu_relax (); if (!(1)) { preempt_disable(); if (__builtin_expect(!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }); |
184 | |
185 | return err; |
186 | } |
187 | |
188 | static bool_Bool g4x_reset_complete(struct pci_dev *pdev) |
189 | { |
190 | u8 gdrst; |
191 | |
192 | pci_read_config_byte(pdev, I915_GDRST0xc0, &gdrst); |
193 | return (gdrst & GRDOM_RESET_ENABLE(1 << 0)) == 0; |
194 | } |
195 | |
196 | static int g33_do_reset(struct intel_gt *gt, |
197 | intel_engine_mask_t engine_mask, |
198 | unsigned int retry) |
199 | { |
200 | struct pci_dev *pdev = gt->i915->drm.pdev; |
201 | |
202 | pci_write_config_byte(pdev, I915_GDRST0xc0, GRDOM_RESET_ENABLE(1 << 0)); |
203 | return wait_for_atomic(g4x_reset_complete(pdev), 50)({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { preempt_disable(); cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci ) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci; })->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) preempt_enable(); __asm volatile("" : : : "memory"); if (((g4x_reset_complete(pdev)))) { ret = 0 ; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { preempt_disable(); if (__builtin_expect (!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }); |
204 | } |
205 | |
206 | static int g4x_do_reset(struct intel_gt *gt, |
207 | intel_engine_mask_t engine_mask, |
208 | unsigned int retry) |
209 | { |
210 | struct pci_dev *pdev = gt->i915->drm.pdev; |
211 | struct intel_uncore *uncore = gt->uncore; |
212 | int ret; |
213 | |
214 | /* WaVcpClkGateDisableForMediaReset:ctg,elk */ |
215 | rmw_set_fw(uncore, VDECCLK_GATE_D((const i915_reg_t){ .reg = (0x620C) }), VCP_UNIT_CLOCK_GATE_DISABLE(1 << 4)); |
216 | intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D)((void)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = (0x620C) }))); |
217 | |
218 | pci_write_config_byte(pdev, I915_GDRST0xc0, |
219 | GRDOM_MEDIA(3 << 2) | GRDOM_RESET_ENABLE(1 << 0)); |
220 | ret = wait_for_atomic(g4x_reset_complete(pdev), 50)({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { preempt_disable(); cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci ) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci; })->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) preempt_enable(); __asm volatile("" : : : "memory"); if (((g4x_reset_complete(pdev)))) { ret = 0 ; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { preempt_disable(); if (__builtin_expect (!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }); |
221 | if (ret) { |
222 | GT_TRACE(gt, "Wait for media reset failed\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
223 | goto out; |
224 | } |
225 | |
226 | pci_write_config_byte(pdev, I915_GDRST0xc0, |
227 | GRDOM_RENDER(1 << 2) | GRDOM_RESET_ENABLE(1 << 0)); |
228 | ret = wait_for_atomic(g4x_reset_complete(pdev), 50)({ extern char _ctassert[(!(!__builtin_constant_p((50) * 1000 ))) ? 1 : -1 ] __attribute__((__unused__)); extern char _ctassert [(!(((50) * 1000) > 50000)) ? 1 : -1 ] __attribute__((__unused__ )); ({ int cpu, ret, timeout = (((50) * 1000)) * 1000; u64 base ; do { } while (0); if (!(1)) { preempt_disable(); cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci ) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci; })->ci_cpuid); } base = local_clock(); for (;;) { u64 now = local_clock(); if (!(1)) preempt_enable(); __asm volatile("" : : : "memory"); if (((g4x_reset_complete(pdev)))) { ret = 0 ; break; } if (now - base >= timeout) { ret = -60; break; } cpu_relax(); if (!(1)) { preempt_disable(); if (__builtin_expect (!!(cpu != (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid)), 0)) { timeout -= now - base; cpu = (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_cpuid); base = local_clock(); } } } ret; } ); }); |
229 | if (ret) { |
230 | GT_TRACE(gt, "Wait for render reset failed\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
231 | goto out; |
232 | } |
233 | |
234 | out: |
235 | pci_write_config_byte(pdev, I915_GDRST0xc0, 0); |
236 | |
237 | rmw_clear_fw(uncore, VDECCLK_GATE_D((const i915_reg_t){ .reg = (0x620C) }), VCP_UNIT_CLOCK_GATE_DISABLE(1 << 4)); |
238 | intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D)((void)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = (0x620C) }))); |
239 | |
240 | return ret; |
241 | } |
242 | |
243 | static int ilk_do_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask, |
244 | unsigned int retry) |
245 | { |
246 | struct intel_uncore *uncore = gt->uncore; |
247 | int ret; |
248 | |
249 | intel_uncore_write_fw(uncore, ILK_GDSR,__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), (1 << 1) | (1 << 0)) |
250 | ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), (1 << 1) | (1 << 0)); |
251 | ret = __intel_wait_for_register_fw(uncore, ILK_GDSR((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), |
252 | ILK_GRDOM_RESET_ENABLE(1 << 0), 0, |
253 | 5000, 0, |
254 | NULL((void *)0)); |
255 | if (ret) { |
256 | GT_TRACE(gt, "Wait for render reset failed\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
257 | goto out; |
258 | } |
259 | |
260 | intel_uncore_write_fw(uncore, ILK_GDSR,__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), (3 << 1) | (1 << 0)) |
261 | ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), (3 << 1) | (1 << 0)); |
262 | ret = __intel_wait_for_register_fw(uncore, ILK_GDSR((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), |
263 | ILK_GRDOM_RESET_ENABLE(1 << 0), 0, |
264 | 5000, 0, |
265 | NULL((void *)0)); |
266 | if (ret) { |
267 | GT_TRACE(gt, "Wait for media reset failed\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
268 | goto out; |
269 | } |
270 | |
271 | out: |
272 | intel_uncore_write_fw(uncore, ILK_GDSR, 0)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }), 0); |
273 | intel_uncore_posting_read_fw(uncore, ILK_GDSR)((void)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = (0x10000 + 0x2ca4) }))); |
274 | return ret; |
275 | } |
276 | |
277 | /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */ |
278 | static int gen6_hw_domain_reset(struct intel_gt *gt, u32 hw_domain_mask) |
279 | { |
280 | struct intel_uncore *uncore = gt->uncore; |
281 | int loops = 2; |
282 | int err; |
283 | |
284 | /* |
285 | * GEN6_GDRST is not in the gt power well, no need to check |
286 | * for fifo space for the write or forcewake the chip for |
287 | * the read |
288 | */ |
289 | do { |
290 | intel_uncore_write_fw(uncore, GEN6_GDRST, hw_domain_mask)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x941c ) }), hw_domain_mask); |
291 | |
292 | /* |
293 | * Wait for the device to ack the reset requests. |
294 | * |
295 | * On some platforms, e.g. Jasperlake, we see that the |
296 | * engine register state is not cleared until shortly after |
297 | * GDRST reports completion, causing a failure as we try |
298 | * to immediately resume while the internal state is still |
299 | * in flux. If we immediately repeat the reset, the second |
300 | * reset appears to serialise with the first, and since |
301 | * it is a no-op, the registers should retain their reset |
302 | * value. However, there is still a concern that upon |
303 | * leaving the second reset, the internal engine state |
304 | * is still in flux and not ready for resuming. |
305 | */ |
306 | err = __intel_wait_for_register_fw(uncore, GEN6_GDRST((const i915_reg_t){ .reg = (0x941c) }), |
307 | hw_domain_mask, 0, |
308 | 2000, 0, |
309 | NULL((void *)0)); |
310 | } while (err == 0 && --loops); |
311 | if (err) |
312 | GT_TRACE(gt,do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0) |
313 | "Wait for 0x%08x engines reset failed\n",do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0) |
314 | hw_domain_mask)do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
315 | |
316 | /* |
317 | * As we have observed that the engine state is still volatile |
318 | * after GDRST is acked, impose a small delay to let everything settle. |
319 | */ |
320 | udelay(50); |
321 | |
322 | return err; |
323 | } |
324 | |
325 | static int __gen6_reset_engines(struct intel_gt *gt, |
326 | intel_engine_mask_t engine_mask, |
327 | unsigned int retry) |
328 | { |
329 | struct intel_engine_cs *engine; |
330 | u32 hw_mask; |
331 | |
332 | if (engine_mask == ALL_ENGINES((intel_engine_mask_t)~0ul)) { |
333 | hw_mask = GEN6_GRDOM_FULL(1 << 0); |
334 | } else { |
335 | intel_engine_mask_t tmp; |
336 | |
337 | hw_mask = 0; |
338 | for_each_engine_masked(engine, gt, engine_mask, tmp)for ((tmp) = (engine_mask) & (gt)->info.engine_mask; ( tmp) ? ((engine) = (gt)->engine[({ int __idx = ffs(tmp) - 1 ; tmp &= ~(1UL << (__idx)); __idx; })]), 1 : 0;) { |
339 | hw_mask |= engine->reset_domain; |
340 | } |
341 | } |
342 | |
343 | return gen6_hw_domain_reset(gt, hw_mask); |
344 | } |
345 | |
346 | static int gen6_reset_engines(struct intel_gt *gt, |
347 | intel_engine_mask_t engine_mask, |
348 | unsigned int retry) |
349 | { |
350 | unsigned long flags; |
351 | int ret; |
352 | |
353 | spin_lock_irqsave(>->uncore->lock, flags)do { flags = 0; mtx_enter(>->uncore->lock); } while (0); |
354 | ret = __gen6_reset_engines(gt, engine_mask, retry); |
355 | spin_unlock_irqrestore(>->uncore->lock, flags)do { (void)(flags); mtx_leave(>->uncore->lock); } while (0); |
356 | |
357 | return ret; |
358 | } |
359 | |
360 | static struct intel_engine_cs *find_sfc_paired_vecs_engine(struct intel_engine_cs *engine) |
361 | { |
362 | int vecs_id; |
363 | |
364 | GEM_BUG_ON(engine->class != VIDEO_DECODE_CLASS)((void)0); |
365 | |
366 | vecs_id = _VECS((engine->instance) / 2)(VECS0 + ((engine->instance) / 2)); |
367 | |
368 | return engine->gt->engine[vecs_id]; |
369 | } |
370 | |
371 | struct sfc_lock_data { |
372 | i915_reg_t lock_reg; |
373 | i915_reg_t ack_reg; |
374 | i915_reg_t usage_reg; |
375 | u32 lock_bit; |
376 | u32 ack_bit; |
377 | u32 usage_bit; |
378 | u32 reset_bit; |
379 | }; |
380 | |
381 | static void get_sfc_forced_lock_data(struct intel_engine_cs *engine, |
382 | struct sfc_lock_data *sfc_lock) |
383 | { |
384 | switch (engine->class) { |
385 | default: |
386 | MISSING_CASE(engine->class)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n" , "engine->class", (long)(engine->class)); __builtin_expect (!!(__ret), 0); }); |
387 | fallthroughdo {} while (0); |
388 | case VIDEO_DECODE_CLASS1: |
389 | sfc_lock->lock_reg = GEN11_VCS_SFC_FORCED_LOCK(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x88c) } ); |
390 | sfc_lock->lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT(1 << 0); |
391 | |
392 | sfc_lock->ack_reg = GEN11_VCS_SFC_LOCK_STATUS(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x890) } ); |
393 | sfc_lock->ack_bit = GEN11_VCS_SFC_LOCK_ACK_BIT(1 << 1); |
394 | |
395 | sfc_lock->usage_reg = GEN11_VCS_SFC_LOCK_STATUS(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x890) } ); |
396 | sfc_lock->usage_bit = GEN11_VCS_SFC_USAGE_BIT(1 << 0); |
397 | sfc_lock->reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance)(((u32)((1UL << (17)) + 0)) << ((engine->instance ) >> 1)); |
398 | |
399 | break; |
400 | case VIDEO_ENHANCEMENT_CLASS2: |
401 | sfc_lock->lock_reg = GEN11_VECS_SFC_FORCED_LOCK(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x201c) }); |
402 | sfc_lock->lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT(1 << 0); |
403 | |
404 | sfc_lock->ack_reg = GEN11_VECS_SFC_LOCK_ACK(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x2018) }); |
405 | sfc_lock->ack_bit = GEN11_VECS_SFC_LOCK_ACK_BIT(1 << 0); |
406 | |
407 | sfc_lock->usage_reg = GEN11_VECS_SFC_USAGE(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x2014) }); |
408 | sfc_lock->usage_bit = GEN11_VECS_SFC_USAGE_BIT(1 << 0); |
409 | sfc_lock->reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance)(((u32)((1UL << (17)) + 0)) << (engine->instance )); |
410 | |
411 | break; |
412 | } |
413 | } |
414 | |
415 | static int gen11_lock_sfc(struct intel_engine_cs *engine, |
416 | u32 *reset_mask, |
417 | u32 *unlock_mask) |
418 | { |
419 | struct intel_uncore *uncore = engine->uncore; |
420 | u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access; |
421 | struct sfc_lock_data sfc_lock; |
422 | bool_Bool lock_obtained, lock_to_other = false0; |
423 | int ret; |
424 | |
425 | switch (engine->class) { |
426 | case VIDEO_DECODE_CLASS1: |
427 | if ((BIT(engine->instance)(1UL << (engine->instance)) & vdbox_sfc_access) == 0) |
428 | return 0; |
429 | |
430 | fallthroughdo {} while (0); |
431 | case VIDEO_ENHANCEMENT_CLASS2: |
432 | get_sfc_forced_lock_data(engine, &sfc_lock); |
433 | |
434 | break; |
435 | default: |
436 | return 0; |
437 | } |
438 | |
439 | if (!(intel_uncore_read_fw(uncore, sfc_lock.usage_reg)__raw_uncore_read32(uncore, sfc_lock.usage_reg) & sfc_lock.usage_bit)) { |
440 | struct intel_engine_cs *paired_vecs; |
441 | |
442 | if (engine->class != VIDEO_DECODE_CLASS1 || |
443 | GRAPHICS_VER(engine->i915)((&(engine->i915)->__runtime)->graphics.ip.ver) != 12) |
444 | return 0; |
445 | |
446 | /* |
447 | * Wa_14010733141 |
448 | * |
449 | * If the VCS-MFX isn't using the SFC, we also need to check |
450 | * whether VCS-HCP is using it. If so, we need to issue a *VE* |
451 | * forced lock on the VE engine that shares the same SFC. |
452 | */ |
453 | if (!(intel_uncore_read_fw(uncore,__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = ((engine ->mmio_base) + 0x2914) })) |
454 | GEN12_HCP_SFC_LOCK_STATUS(engine->mmio_base))__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = ((engine ->mmio_base) + 0x2914) })) & |
455 | GEN12_HCP_SFC_USAGE_BIT((u32)((1UL << (0)) + 0)))) |
456 | return 0; |
457 | |
458 | paired_vecs = find_sfc_paired_vecs_engine(engine); |
459 | get_sfc_forced_lock_data(paired_vecs, &sfc_lock); |
460 | lock_to_other = true1; |
461 | *unlock_mask |= paired_vecs->mask; |
462 | } else { |
463 | *unlock_mask |= engine->mask; |
464 | } |
465 | |
466 | /* |
467 | * If the engine is using an SFC, tell the engine that a software reset |
468 | * is going to happen. The engine will then try to force lock the SFC. |
469 | * If SFC ends up being locked to the engine we want to reset, we have |
470 | * to reset it as well (we will unlock it once the reset sequence is |
471 | * completed). |
472 | */ |
473 | rmw_set_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); |
474 | |
475 | ret = __intel_wait_for_register_fw(uncore, |
476 | sfc_lock.ack_reg, |
477 | sfc_lock.ack_bit, |
478 | sfc_lock.ack_bit, |
479 | 1000, 0, NULL((void *)0)); |
480 | |
481 | /* |
482 | * Was the SFC released while we were trying to lock it? |
483 | * |
484 | * We should reset both the engine and the SFC if: |
485 | * - We were locking the SFC to this engine and the lock succeeded |
486 | * OR |
487 | * - We were locking the SFC to a different engine (Wa_14010733141) |
488 | * but the SFC was released before the lock was obtained. |
489 | * |
490 | * Otherwise we need only reset the engine by itself and we can |
491 | * leave the SFC alone. |
492 | */ |
493 | lock_obtained = (intel_uncore_read_fw(uncore, sfc_lock.usage_reg)__raw_uncore_read32(uncore, sfc_lock.usage_reg) & |
494 | sfc_lock.usage_bit) != 0; |
495 | if (lock_obtained == lock_to_other) |
496 | return 0; |
497 | |
498 | if (ret) { |
499 | ENGINE_TRACE(engine, "Wait for SFC forced lock ack failed\n")do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
500 | return ret; |
501 | } |
502 | |
503 | *reset_mask |= sfc_lock.reset_bit; |
504 | return 0; |
505 | } |
506 | |
507 | static void gen11_unlock_sfc(struct intel_engine_cs *engine) |
508 | { |
509 | struct intel_uncore *uncore = engine->uncore; |
510 | u8 vdbox_sfc_access = engine->gt->info.vdbox_sfc_access; |
511 | struct sfc_lock_data sfc_lock = {}; |
512 | |
513 | if (engine->class != VIDEO_DECODE_CLASS1 && |
514 | engine->class != VIDEO_ENHANCEMENT_CLASS2) |
515 | return; |
516 | |
517 | if (engine->class == VIDEO_DECODE_CLASS1 && |
518 | (BIT(engine->instance)(1UL << (engine->instance)) & vdbox_sfc_access) == 0) |
519 | return; |
520 | |
521 | get_sfc_forced_lock_data(engine, &sfc_lock); |
522 | |
523 | rmw_clear_fw(uncore, sfc_lock.lock_reg, sfc_lock.lock_bit); |
524 | } |
525 | |
526 | static int __gen11_reset_engines(struct intel_gt *gt, |
527 | intel_engine_mask_t engine_mask, |
528 | unsigned int retry) |
529 | { |
530 | struct intel_engine_cs *engine; |
531 | intel_engine_mask_t tmp; |
532 | u32 reset_mask, unlock_mask = 0; |
533 | int ret; |
534 | |
535 | if (engine_mask == ALL_ENGINES((intel_engine_mask_t)~0ul)) { |
536 | reset_mask = GEN11_GRDOM_FULL(1 << 0); |
537 | } else { |
538 | reset_mask = 0; |
539 | for_each_engine_masked(engine, gt, engine_mask, tmp)for ((tmp) = (engine_mask) & (gt)->info.engine_mask; ( tmp) ? ((engine) = (gt)->engine[({ int __idx = ffs(tmp) - 1 ; tmp &= ~(1UL << (__idx)); __idx; })]), 1 : 0;) { |
540 | reset_mask |= engine->reset_domain; |
541 | ret = gen11_lock_sfc(engine, &reset_mask, &unlock_mask); |
542 | if (ret) |
543 | goto sfc_unlock; |
544 | } |
545 | } |
546 | |
547 | ret = gen6_hw_domain_reset(gt, reset_mask); |
548 | |
549 | sfc_unlock: |
550 | /* |
551 | * We unlock the SFC based on the lock status and not the result of |
552 | * gen11_lock_sfc to make sure that we clean properly if something |
553 | * wrong happened during the lock (e.g. lock acquired after timeout |
554 | * expiration). |
555 | * |
556 | * Due to Wa_14010733141, we may have locked an SFC to an engine that |
557 | * wasn't being reset. So instead of calling gen11_unlock_sfc() |
558 | * on engine_mask, we instead call it on the mask of engines that our |
559 | * gen11_lock_sfc() calls told us actually had locks attempted. |
560 | */ |
561 | for_each_engine_masked(engine, gt, unlock_mask, tmp)for ((tmp) = (unlock_mask) & (gt)->info.engine_mask; ( tmp) ? ((engine) = (gt)->engine[({ int __idx = ffs(tmp) - 1 ; tmp &= ~(1UL << (__idx)); __idx; })]), 1 : 0;) |
562 | gen11_unlock_sfc(engine); |
563 | |
564 | return ret; |
565 | } |
566 | |
567 | static int gen8_engine_reset_prepare(struct intel_engine_cs *engine) |
568 | { |
569 | struct intel_uncore *uncore = engine->uncore; |
570 | const i915_reg_t reg = RING_RESET_CTL(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0xd0) } ); |
571 | u32 request, mask, ack; |
572 | int ret; |
573 | |
574 | if (I915_SELFTEST_ONLY(should_fail(&engine->reset_timeout, 1))0) |
575 | return -ETIMEDOUT60; |
576 | |
577 | ack = intel_uncore_read_fw(uncore, reg)__raw_uncore_read32(uncore, reg); |
578 | if (ack & RESET_CTL_CAT_ERROR((u32)((1UL << (2)) + 0))) { |
579 | /* |
580 | * For catastrophic errors, ready-for-reset sequence |
581 | * needs to be bypassed: HAS#396813 |
582 | */ |
583 | request = RESET_CTL_CAT_ERROR((u32)((1UL << (2)) + 0)); |
584 | mask = RESET_CTL_CAT_ERROR((u32)((1UL << (2)) + 0)); |
585 | |
586 | /* Catastrophic errors need to be cleared by HW */ |
587 | ack = 0; |
588 | } else if (!(ack & RESET_CTL_READY_TO_RESET((u32)((1UL << (1)) + 0)))) { |
589 | request = RESET_CTL_REQUEST_RESET((u32)((1UL << (0)) + 0)); |
590 | mask = RESET_CTL_READY_TO_RESET((u32)((1UL << (1)) + 0)); |
591 | ack = RESET_CTL_READY_TO_RESET((u32)((1UL << (1)) + 0)); |
592 | } else { |
593 | return 0; |
594 | } |
595 | |
596 | intel_uncore_write_fw(uncore, reg, _MASKED_BIT_ENABLE(request))__raw_uncore_write32(uncore, reg, ({ typeof(request) _a = (request ); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p(_a)) do { } while (0); ((_a) << 16 | (_a)); }); })); |
597 | ret = __intel_wait_for_register_fw(uncore, reg, mask, ack, |
598 | 700, 0, NULL((void *)0)); |
599 | if (ret) |
600 | drm_err(&engine->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, request, __raw_uncore_read32(uncore, reg)) |
601 | "%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, request, __raw_uncore_read32(uncore, reg)) |
602 | engine->name, request,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, request, __raw_uncore_read32(uncore, reg)) |
603 | intel_uncore_read_fw(uncore, reg))printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, request, __raw_uncore_read32(uncore, reg)); |
604 | |
605 | return ret; |
606 | } |
607 | |
608 | static void gen8_engine_reset_cancel(struct intel_engine_cs *engine) |
609 | { |
610 | intel_uncore_write_fw(engine->uncore,__raw_uncore_write32(engine->uncore, ((const i915_reg_t){ . reg = ((engine->mmio_base) + 0xd0) }), (({ if (__builtin_constant_p ((((u32)((1UL << (0)) + 0))))) do { } while (0); if (__builtin_constant_p (0)) do { } while (0); if (__builtin_constant_p((((u32)((1UL << (0)) + 0)))) && __builtin_constant_p(0)) do { } while (0); (((((u32)((1UL << (0)) + 0)))) << 16 | (0)) ; }))) |
611 | RING_RESET_CTL(engine->mmio_base),__raw_uncore_write32(engine->uncore, ((const i915_reg_t){ . reg = ((engine->mmio_base) + 0xd0) }), (({ if (__builtin_constant_p ((((u32)((1UL << (0)) + 0))))) do { } while (0); if (__builtin_constant_p (0)) do { } while (0); if (__builtin_constant_p((((u32)((1UL << (0)) + 0)))) && __builtin_constant_p(0)) do { } while (0); (((((u32)((1UL << (0)) + 0)))) << 16 | (0)) ; }))) |
612 | _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET))__raw_uncore_write32(engine->uncore, ((const i915_reg_t){ . reg = ((engine->mmio_base) + 0xd0) }), (({ if (__builtin_constant_p ((((u32)((1UL << (0)) + 0))))) do { } while (0); if (__builtin_constant_p (0)) do { } while (0); if (__builtin_constant_p((((u32)((1UL << (0)) + 0)))) && __builtin_constant_p(0)) do { } while (0); (((((u32)((1UL << (0)) + 0)))) << 16 | (0)) ; }))); |
613 | } |
614 | |
615 | static int gen8_reset_engines(struct intel_gt *gt, |
616 | intel_engine_mask_t engine_mask, |
617 | unsigned int retry) |
618 | { |
619 | struct intel_engine_cs *engine; |
620 | const bool_Bool reset_non_ready = retry >= 1; |
621 | intel_engine_mask_t tmp; |
622 | unsigned long flags; |
623 | int ret; |
624 | |
625 | spin_lock_irqsave(>->uncore->lock, flags)do { flags = 0; mtx_enter(>->uncore->lock); } while (0); |
626 | |
627 | for_each_engine_masked(engine, gt, engine_mask, tmp)for ((tmp) = (engine_mask) & (gt)->info.engine_mask; ( tmp) ? ((engine) = (gt)->engine[({ int __idx = ffs(tmp) - 1 ; tmp &= ~(1UL << (__idx)); __idx; })]), 1 : 0;) { |
628 | ret = gen8_engine_reset_prepare(engine); |
629 | if (ret && !reset_non_ready) |
630 | goto skip_reset; |
631 | |
632 | /* |
633 | * If this is not the first failed attempt to prepare, |
634 | * we decide to proceed anyway. |
635 | * |
636 | * By doing so we risk context corruption and with |
637 | * some gens (kbl), possible system hang if reset |
638 | * happens during active bb execution. |
639 | * |
640 | * We rather take context corruption instead of |
641 | * failed reset with a wedged driver/gpu. And |
642 | * active bb execution case should be covered by |
643 | * stop_engines() we have before the reset. |
644 | */ |
645 | } |
646 | |
647 | /* |
648 | * Wa_22011100796:dg2, whenever Full soft reset is required, |
649 | * reset all individual engines firstly, and then do a full soft reset. |
650 | * |
651 | * This is best effort, so ignore any error from the initial reset. |
652 | */ |
653 | if (IS_DG2(gt->i915)IS_PLATFORM(gt->i915, INTEL_DG2) && engine_mask == ALL_ENGINES((intel_engine_mask_t)~0ul)) |
654 | __gen11_reset_engines(gt, gt->info.engine_mask, 0); |
655 | |
656 | if (GRAPHICS_VER(gt->i915)((&(gt->i915)->__runtime)->graphics.ip.ver) >= 11) |
657 | ret = __gen11_reset_engines(gt, engine_mask, retry); |
658 | else |
659 | ret = __gen6_reset_engines(gt, engine_mask, retry); |
660 | |
661 | skip_reset: |
662 | for_each_engine_masked(engine, gt, engine_mask, tmp)for ((tmp) = (engine_mask) & (gt)->info.engine_mask; ( tmp) ? ((engine) = (gt)->engine[({ int __idx = ffs(tmp) - 1 ; tmp &= ~(1UL << (__idx)); __idx; })]), 1 : 0;) |
663 | gen8_engine_reset_cancel(engine); |
664 | |
665 | spin_unlock_irqrestore(>->uncore->lock, flags)do { (void)(flags); mtx_leave(>->uncore->lock); } while (0); |
666 | |
667 | return ret; |
668 | } |
669 | |
670 | static int mock_reset(struct intel_gt *gt, |
671 | intel_engine_mask_t mask, |
672 | unsigned int retry) |
673 | { |
674 | return 0; |
675 | } |
676 | |
677 | typedef int (*reset_func)(struct intel_gt *, |
678 | intel_engine_mask_t engine_mask, |
679 | unsigned int retry); |
680 | |
681 | static reset_func intel_get_gpu_reset(const struct intel_gt *gt) |
682 | { |
683 | struct drm_i915_privateinteldrm_softc *i915 = gt->i915; |
684 | |
685 | if (is_mock_gt(gt)) |
686 | return mock_reset; |
687 | else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 8) |
688 | return gen8_reset_engines; |
689 | else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 6) |
690 | return gen6_reset_engines; |
691 | else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 5) |
692 | return ilk_do_reset; |
693 | else if (IS_G4X(i915)(IS_PLATFORM(i915, INTEL_G45) || IS_PLATFORM(i915, INTEL_GM45 ))) |
694 | return g4x_do_reset; |
695 | else if (IS_G33(i915)IS_PLATFORM(i915, INTEL_G33) || IS_PINEVIEW(i915)IS_PLATFORM(i915, INTEL_PINEVIEW)) |
696 | return g33_do_reset; |
697 | else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 3) |
698 | return i915_do_reset; |
699 | else |
700 | return NULL((void *)0); |
701 | } |
702 | |
703 | int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask) |
704 | { |
705 | const int retries = engine_mask == ALL_ENGINES((intel_engine_mask_t)~0ul) ? RESET_MAX_RETRIES3 : 1; |
706 | reset_func reset; |
707 | int ret = -ETIMEDOUT60; |
708 | int retry; |
709 | |
710 | reset = intel_get_gpu_reset(gt); |
711 | if (!reset) |
712 | return -ENODEV19; |
713 | |
714 | /* |
715 | * If the power well sleeps during the reset, the reset |
716 | * request may be dropped and never completes (causing -EIO). |
717 | */ |
718 | intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); |
719 | for (retry = 0; ret == -ETIMEDOUT60 && retry < retries; retry++) { |
720 | GT_TRACE(gt, "engine_mask=%x\n", engine_mask)do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
721 | preempt_disable(); |
722 | ret = reset(gt, engine_mask, retry); |
723 | preempt_enable(); |
724 | } |
725 | intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); |
726 | |
727 | return ret; |
728 | } |
729 | |
730 | bool_Bool intel_has_gpu_reset(const struct intel_gt *gt) |
731 | { |
732 | if (!gt->i915->params.reset) |
733 | return NULL((void *)0); |
734 | |
735 | return intel_get_gpu_reset(gt); |
736 | } |
737 | |
738 | bool_Bool intel_has_reset_engine(const struct intel_gt *gt) |
739 | { |
740 | if (gt->i915->params.reset < 2) |
741 | return false0; |
742 | |
743 | return INTEL_INFO(gt->i915)(&(gt->i915)->__info)->has_reset_engine; |
744 | } |
745 | |
746 | int intel_reset_guc(struct intel_gt *gt) |
747 | { |
748 | u32 guc_domain = |
749 | GRAPHICS_VER(gt->i915)((&(gt->i915)->__runtime)->graphics.ip.ver) >= 11 ? GEN11_GRDOM_GUC((u32)((1UL << (3)) + 0)) : GEN9_GRDOM_GUC(1 << 5); |
750 | int ret; |
751 | |
752 | GEM_BUG_ON(!HAS_GT_UC(gt->i915))((void)0); |
753 | |
754 | intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); |
755 | ret = gen6_hw_domain_reset(gt, guc_domain); |
756 | intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); |
757 | |
758 | return ret; |
759 | } |
760 | |
761 | /* |
762 | * Ensure irq handler finishes, and not run again. |
763 | * Also return the active request so that we only search for it once. |
764 | */ |
765 | static void reset_prepare_engine(struct intel_engine_cs *engine) |
766 | { |
767 | /* |
768 | * During the reset sequence, we must prevent the engine from |
769 | * entering RC6. As the context state is undefined until we restart |
770 | * the engine, if it does enter RC6 during the reset, the state |
771 | * written to the powercontext is undefined and so we may lose |
772 | * GPU state upon resume, i.e. fail to restart after a reset. |
773 | */ |
774 | intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); |
775 | if (engine->reset.prepare) |
776 | engine->reset.prepare(engine); |
777 | } |
778 | |
779 | static void revoke_mmaps(struct intel_gt *gt) |
780 | { |
781 | int i; |
782 | |
783 | for (i = 0; i < gt->ggtt->num_fences; i++) { |
784 | struct drm_vma_offset_node *node; |
785 | struct i915_vma *vma; |
786 | u64 vma_offset; |
787 | |
788 | vma = READ_ONCE(gt->ggtt->fence_regs[i].vma)({ typeof(gt->ggtt->fence_regs[i].vma) __tmp = *(volatile typeof(gt->ggtt->fence_regs[i].vma) *)&(gt->ggtt ->fence_regs[i].vma); membar_datadep_consumer(); __tmp; }); |
789 | if (!vma) |
790 | continue; |
791 | |
792 | if (!i915_vma_has_userfault(vma)) |
793 | continue; |
794 | |
795 | GEM_BUG_ON(vma->fence != >->ggtt->fence_regs[i])((void)0); |
796 | |
797 | if (!vma->mmo) |
798 | continue; |
799 | |
800 | node = &vma->mmo->vma_node; |
Value stored to 'node' is never read | |
801 | vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT12; |
802 | |
803 | #ifdef __linux__ |
804 | unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping, |
805 | drm_vma_node_offset_addr(node) + vma_offset, |
806 | vma->size, |
807 | 1); |
808 | #else |
809 | { |
810 | struct drm_i915_privateinteldrm_softc *dev_priv = vma->obj->base.dev->dev_private; |
811 | struct vm_page *pg; |
812 | |
813 | for (pg = &dev_priv->pgs[atop(vma->node.start)((vma->node.start) >> 12)]; |
814 | pg != &dev_priv->pgs[atop(vma->node.start + vma->size)((vma->node.start + vma->size) >> 12)]; |
815 | pg++) |
816 | pmap_page_protect(pg, PROT_NONE0x00); |
817 | } |
818 | #endif |
819 | } |
820 | } |
821 | |
822 | static intel_engine_mask_t reset_prepare(struct intel_gt *gt) |
823 | { |
824 | struct intel_engine_cs *engine; |
825 | intel_engine_mask_t awake = 0; |
826 | enum intel_engine_id id; |
827 | |
828 | /* For GuC mode, ensure submission is disabled before stopping ring */ |
829 | intel_uc_reset_prepare(>->uc); |
830 | |
831 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else { |
832 | if (intel_engine_pm_get_if_awake(engine)) |
833 | awake |= engine->mask; |
834 | reset_prepare_engine(engine); |
835 | } |
836 | |
837 | return awake; |
838 | } |
839 | |
840 | static void gt_revoke(struct intel_gt *gt) |
841 | { |
842 | revoke_mmaps(gt); |
843 | } |
844 | |
845 | static int gt_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask) |
846 | { |
847 | struct intel_engine_cs *engine; |
848 | enum intel_engine_id id; |
849 | int err; |
850 | |
851 | /* |
852 | * Everything depends on having the GTT running, so we need to start |
853 | * there. |
854 | */ |
855 | err = i915_ggtt_enable_hw(gt->i915); |
856 | if (err) |
857 | return err; |
858 | |
859 | local_bh_disable(); |
860 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else |
861 | __intel_engine_reset(engine, stalled_mask & engine->mask); |
862 | local_bh_enable(); |
863 | |
864 | intel_uc_reset(>->uc, ALL_ENGINES((intel_engine_mask_t)~0ul)); |
865 | |
866 | intel_ggtt_restore_fences(gt->ggtt); |
867 | |
868 | return err; |
869 | } |
870 | |
871 | static void reset_finish_engine(struct intel_engine_cs *engine) |
872 | { |
873 | if (engine->reset.finish) |
874 | engine->reset.finish(engine); |
875 | intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); |
876 | |
877 | intel_engine_signal_breadcrumbs(engine); |
878 | } |
879 | |
880 | static void reset_finish(struct intel_gt *gt, intel_engine_mask_t awake) |
881 | { |
882 | struct intel_engine_cs *engine; |
883 | enum intel_engine_id id; |
884 | |
885 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else { |
886 | reset_finish_engine(engine); |
887 | if (awake & engine->mask) |
888 | intel_engine_pm_put(engine); |
889 | } |
890 | |
891 | intel_uc_reset_finish(>->uc); |
892 | } |
893 | |
894 | static void nop_submit_request(struct i915_request *request) |
895 | { |
896 | RQ_TRACE(request, "-EIO\n")do { const struct i915_request *rq__ = (request); do { const struct intel_engine_cs *e__ __attribute__((__unused__)) = (rq__-> engine); do { } while (0); } while (0); } while (0); |
897 | |
898 | request = i915_request_mark_eio(request); |
899 | if (request) { |
900 | i915_request_submit(request); |
901 | intel_engine_signal_breadcrumbs(request->engine); |
902 | |
903 | i915_request_put(request); |
904 | } |
905 | } |
906 | |
907 | static void __intel_gt_set_wedged(struct intel_gt *gt) |
908 | { |
909 | struct intel_engine_cs *engine; |
910 | intel_engine_mask_t awake; |
911 | enum intel_engine_id id; |
912 | |
913 | if (test_bit(I915_WEDGED(64 - 1), >->reset.flags)) |
914 | return; |
915 | |
916 | GT_TRACE(gt, "start\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
917 | |
918 | /* |
919 | * First, stop submission to hw, but do not yet complete requests by |
920 | * rolling the global seqno forward (since this would complete requests |
921 | * for which we haven't set the fence error to EIO yet). |
922 | */ |
923 | awake = reset_prepare(gt); |
924 | |
925 | /* Even if the GPU reset fails, it should still stop the engines */ |
926 | if (!INTEL_INFO(gt->i915)(&(gt->i915)->__info)->gpu_reset_clobbers_display) |
927 | __intel_gt_reset(gt, ALL_ENGINES((intel_engine_mask_t)~0ul)); |
928 | |
929 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else |
930 | engine->submit_request = nop_submit_request; |
931 | |
932 | /* |
933 | * Make sure no request can slip through without getting completed by |
934 | * either this call here to intel_engine_write_global_seqno, or the one |
935 | * in nop_submit_request. |
936 | */ |
937 | synchronize_rcu_expedited(); |
938 | set_bit(I915_WEDGED(64 - 1), >->reset.flags); |
939 | |
940 | /* Mark all executing requests as skipped */ |
941 | local_bh_disable(); |
942 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else |
943 | if (engine->reset.cancel) |
944 | engine->reset.cancel(engine); |
945 | intel_uc_cancel_requests(>->uc); |
946 | local_bh_enable(); |
947 | |
948 | reset_finish(gt, awake); |
949 | |
950 | GT_TRACE(gt, "end\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
951 | } |
952 | |
953 | void intel_gt_set_wedged(struct intel_gt *gt) |
954 | { |
955 | intel_wakeref_t wakeref; |
956 | |
957 | if (test_bit(I915_WEDGED(64 - 1), >->reset.flags)) |
958 | return; |
959 | |
960 | wakeref = intel_runtime_pm_get(gt->uncore->rpm); |
961 | mutex_lock(>->reset.mutex)rw_enter_write(>->reset.mutex); |
962 | |
963 | if (GEM_SHOW_DEBUG()(0)) { |
964 | struct drm_printer p = drm_debug_printer(__func__); |
965 | struct intel_engine_cs *engine; |
966 | enum intel_engine_id id; |
967 | |
968 | drm_printf(&p, "called from %pS\n", (void *)_RET_IP___builtin_return_address(0)); |
969 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else { |
970 | if (intel_engine_is_idle(engine)) |
971 | continue; |
972 | |
973 | intel_engine_dump(engine, &p, "%s\n", engine->name); |
974 | } |
975 | } |
976 | |
977 | __intel_gt_set_wedged(gt); |
978 | |
979 | mutex_unlock(>->reset.mutex)rw_exit_write(>->reset.mutex); |
980 | intel_runtime_pm_put(gt->uncore->rpm, wakeref); |
981 | } |
982 | |
983 | static bool_Bool __intel_gt_unset_wedged(struct intel_gt *gt) |
984 | { |
985 | struct intel_gt_timelines *timelines = >->timelines; |
986 | struct intel_timeline *tl; |
987 | bool_Bool ok; |
988 | |
989 | if (!test_bit(I915_WEDGED(64 - 1), >->reset.flags)) |
990 | return true1; |
991 | |
992 | /* Never fully initialised, recovery impossible */ |
993 | if (intel_gt_has_unrecoverable_error(gt)) |
994 | return false0; |
995 | |
996 | GT_TRACE(gt, "start\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
997 | |
998 | /* |
999 | * Before unwedging, make sure that all pending operations |
1000 | * are flushed and errored out - we may have requests waiting upon |
1001 | * third party fences. We marked all inflight requests as EIO, and |
1002 | * every execbuf since returned EIO, for consistency we want all |
1003 | * the currently pending requests to also be marked as EIO, which |
1004 | * is done inside our nop_submit_request - and so we must wait. |
1005 | * |
1006 | * No more can be submitted until we reset the wedged bit. |
1007 | */ |
1008 | spin_lock(&timelines->lock)mtx_enter(&timelines->lock); |
1009 | list_for_each_entry(tl, &timelines->active_list, link)for (tl = ({ const __typeof( ((__typeof(*tl) *)0)->link ) * __mptr = ((&timelines->active_list)->next); (__typeof (*tl) *)( (char *)__mptr - __builtin_offsetof(__typeof(*tl), link ) );}); &tl->link != (&timelines->active_list); tl = ({ const __typeof( ((__typeof(*tl) *)0)->link ) *__mptr = (tl->link.next); (__typeof(*tl) *)( (char *)__mptr - __builtin_offsetof (__typeof(*tl), link) );})) { |
1010 | struct dma_fence *fence; |
1011 | |
1012 | fence = i915_active_fence_get(&tl->last_request); |
1013 | if (!fence) |
1014 | continue; |
1015 | |
1016 | spin_unlock(&timelines->lock)mtx_leave(&timelines->lock); |
1017 | |
1018 | /* |
1019 | * All internal dependencies (i915_requests) will have |
1020 | * been flushed by the set-wedge, but we may be stuck waiting |
1021 | * for external fences. These should all be capped to 10s |
1022 | * (I915_FENCE_TIMEOUT) so this wait should not be unbounded |
1023 | * in the worst case. |
1024 | */ |
1025 | dma_fence_default_wait(fence, false0, MAX_SCHEDULE_TIMEOUT(0x7fffffff)); |
1026 | dma_fence_put(fence); |
1027 | |
1028 | /* Restart iteration after droping lock */ |
1029 | spin_lock(&timelines->lock)mtx_enter(&timelines->lock); |
1030 | tl = list_entry(&timelines->active_list, typeof(*tl), link)({ const __typeof( ((typeof(*tl) *)0)->link ) *__mptr = (& timelines->active_list); (typeof(*tl) *)( (char *)__mptr - __builtin_offsetof(typeof(*tl), link) );}); |
1031 | } |
1032 | spin_unlock(&timelines->lock)mtx_leave(&timelines->lock); |
1033 | |
1034 | /* We must reset pending GPU events before restoring our submission */ |
1035 | ok = !HAS_EXECLISTS(gt->i915)((&(gt->i915)->__info)->has_logical_ring_contexts ); /* XXX better agnosticism desired */ |
1036 | if (!INTEL_INFO(gt->i915)(&(gt->i915)->__info)->gpu_reset_clobbers_display) |
1037 | ok = __intel_gt_reset(gt, ALL_ENGINES((intel_engine_mask_t)~0ul)) == 0; |
1038 | if (!ok) { |
1039 | /* |
1040 | * Warn CI about the unrecoverable wedged condition. |
1041 | * Time for a reboot. |
1042 | */ |
1043 | add_taint_for_CI(gt->i915, TAINT_WARN1); |
1044 | return false0; |
1045 | } |
1046 | |
1047 | /* |
1048 | * Undo nop_submit_request. We prevent all new i915 requests from |
1049 | * being queued (by disallowing execbuf whilst wedged) so having |
1050 | * waited for all active requests above, we know the system is idle |
1051 | * and do not have to worry about a thread being inside |
1052 | * engine->submit_request() as we swap over. So unlike installing |
1053 | * the nop_submit_request on reset, we can do this from normal |
1054 | * context and do not require stop_machine(). |
1055 | */ |
1056 | intel_engines_reset_default_submission(gt); |
1057 | |
1058 | GT_TRACE(gt, "end\n")do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
1059 | |
1060 | smp_mb__before_atomic()do { } while (0); /* complete takeover before enabling execbuf */ |
1061 | clear_bit(I915_WEDGED(64 - 1), >->reset.flags); |
1062 | |
1063 | return true1; |
1064 | } |
1065 | |
1066 | bool_Bool intel_gt_unset_wedged(struct intel_gt *gt) |
1067 | { |
1068 | bool_Bool result; |
1069 | |
1070 | mutex_lock(>->reset.mutex)rw_enter_write(>->reset.mutex); |
1071 | result = __intel_gt_unset_wedged(gt); |
1072 | mutex_unlock(>->reset.mutex)rw_exit_write(>->reset.mutex); |
1073 | |
1074 | return result; |
1075 | } |
1076 | |
1077 | static int do_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask) |
1078 | { |
1079 | int err, i; |
1080 | |
1081 | err = __intel_gt_reset(gt, ALL_ENGINES((intel_engine_mask_t)~0ul)); |
1082 | for (i = 0; err && i < RESET_MAX_RETRIES3; i++) { |
1083 | drm_msleep(10 * (i + 1))mdelay(10 * (i + 1)); |
1084 | err = __intel_gt_reset(gt, ALL_ENGINES((intel_engine_mask_t)~0ul)); |
1085 | } |
1086 | if (err) |
1087 | return err; |
1088 | |
1089 | return gt_reset(gt, stalled_mask); |
1090 | } |
1091 | |
1092 | static int resume(struct intel_gt *gt) |
1093 | { |
1094 | struct intel_engine_cs *engine; |
1095 | enum intel_engine_id id; |
1096 | int ret; |
1097 | |
1098 | for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine ) = (gt)->engine[(id)])) {} else { |
1099 | ret = intel_engine_resume(engine); |
1100 | if (ret) |
1101 | return ret; |
1102 | } |
1103 | |
1104 | return 0; |
1105 | } |
1106 | |
1107 | /** |
1108 | * intel_gt_reset - reset chip after a hang |
1109 | * @gt: #intel_gt to reset |
1110 | * @stalled_mask: mask of the stalled engines with the guilty requests |
1111 | * @reason: user error message for why we are resetting |
1112 | * |
1113 | * Reset the chip. Useful if a hang is detected. Marks the device as wedged |
1114 | * on failure. |
1115 | * |
1116 | * Procedure is fairly simple: |
1117 | * - reset the chip using the reset reg |
1118 | * - re-init context state |
1119 | * - re-init hardware status page |
1120 | * - re-init ring buffer |
1121 | * - re-init interrupt state |
1122 | * - re-init display |
1123 | */ |
1124 | void intel_gt_reset(struct intel_gt *gt, |
1125 | intel_engine_mask_t stalled_mask, |
1126 | const char *reason) |
1127 | { |
1128 | intel_engine_mask_t awake; |
1129 | int ret; |
1130 | |
1131 | GT_TRACE(gt, "flags=%lx\n", gt->reset.flags)do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
1132 | |
1133 | might_sleep()assertwaitok(); |
1134 | GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, >->reset.flags))((void)0); |
1135 | |
1136 | /* |
1137 | * FIXME: Revoking cpu mmap ptes cannot be done from a dma_fence |
1138 | * critical section like gpu reset. |
1139 | */ |
1140 | gt_revoke(gt); |
1141 | |
1142 | mutex_lock(>->reset.mutex)rw_enter_write(>->reset.mutex); |
1143 | |
1144 | /* Clear any previous failed attempts at recovery. Time to try again. */ |
1145 | if (!__intel_gt_unset_wedged(gt)) |
1146 | goto unlock; |
1147 | |
1148 | if (reason) |
1149 | drm_notice(>->i915->drm,printf("drm:pid%d:%s *NOTICE* " "[drm] " "Resetting chip for %s\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , reason ) |
1150 | "Resetting chip for %s\n", reason)printf("drm:pid%d:%s *NOTICE* " "[drm] " "Resetting chip for %s\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , reason ); |
1151 | atomic_inc(>->i915->gpu_error.reset_count)__sync_fetch_and_add(>->i915->gpu_error.reset_count , 1); |
1152 | |
1153 | awake = reset_prepare(gt); |
1154 | |
1155 | if (!intel_has_gpu_reset(gt)) { |
1156 | if (gt->i915->params.reset) |
1157 | drm_err(>->i915->drm, "GPU reset not supported\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "GPU reset not supported\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__); |
1158 | else |
1159 | drm_dbg(>->i915->drm, "GPU reset disabled\n")__drm_dev_dbg(((void *)0), (>->i915->drm) ? (& gt->i915->drm)->dev : ((void *)0), DRM_UT_DRIVER, "GPU reset disabled\n" ); |
1160 | goto error; |
1161 | } |
1162 | |
1163 | if (INTEL_INFO(gt->i915)(&(gt->i915)->__info)->gpu_reset_clobbers_display) |
1164 | intel_runtime_pm_disable_interrupts(gt->i915); |
1165 | |
1166 | if (do_reset(gt, stalled_mask)) { |
1167 | drm_err(>->i915->drm, "Failed to reset chip\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to reset chip\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__); |
1168 | goto taint; |
1169 | } |
1170 | |
1171 | if (INTEL_INFO(gt->i915)(&(gt->i915)->__info)->gpu_reset_clobbers_display) |
1172 | intel_runtime_pm_enable_interrupts(gt->i915); |
1173 | |
1174 | intel_overlay_reset(gt->i915); |
1175 | |
1176 | /* |
1177 | * Next we need to restore the context, but we don't use those |
1178 | * yet either... |
1179 | * |
1180 | * Ring buffer needs to be re-initialized in the KMS case, or if X |
1181 | * was running at the time of the reset (i.e. we weren't VT |
1182 | * switched away). |
1183 | */ |
1184 | ret = intel_gt_init_hw(gt); |
1185 | if (ret) { |
1186 | drm_err(>->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to initialise HW following reset (%d)\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , ret) |
1187 | "Failed to initialise HW following reset (%d)\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to initialise HW following reset (%d)\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , ret) |
1188 | ret)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to initialise HW following reset (%d)\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , ret); |
1189 | goto taint; |
1190 | } |
1191 | |
1192 | ret = resume(gt); |
1193 | if (ret) |
1194 | goto taint; |
1195 | |
1196 | finish: |
1197 | reset_finish(gt, awake); |
1198 | unlock: |
1199 | mutex_unlock(>->reset.mutex)rw_exit_write(>->reset.mutex); |
1200 | return; |
1201 | |
1202 | taint: |
1203 | /* |
1204 | * History tells us that if we cannot reset the GPU now, we |
1205 | * never will. This then impacts everything that is run |
1206 | * subsequently. On failing the reset, we mark the driver |
1207 | * as wedged, preventing further execution on the GPU. |
1208 | * We also want to go one step further and add a taint to the |
1209 | * kernel so that any subsequent faults can be traced back to |
1210 | * this failure. This is important for CI, where if the |
1211 | * GPU/driver fails we would like to reboot and restart testing |
1212 | * rather than continue on into oblivion. For everyone else, |
1213 | * the system should still plod along, but they have been warned! |
1214 | */ |
1215 | add_taint_for_CI(gt->i915, TAINT_WARN1); |
1216 | error: |
1217 | __intel_gt_set_wedged(gt); |
1218 | goto finish; |
1219 | } |
1220 | |
1221 | static int intel_gt_reset_engine(struct intel_engine_cs *engine) |
1222 | { |
1223 | return __intel_gt_reset(engine->gt, engine->mask); |
1224 | } |
1225 | |
1226 | int __intel_engine_reset_bh(struct intel_engine_cs *engine, const char *msg) |
1227 | { |
1228 | struct intel_gt *gt = engine->gt; |
1229 | int ret; |
1230 | |
1231 | ENGINE_TRACE(engine, "flags=%lx\n", gt->reset.flags)do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
1232 | GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, >->reset.flags))((void)0); |
1233 | |
1234 | if (intel_engine_uses_guc(engine)) |
1235 | return -ENODEV19; |
1236 | |
1237 | if (!intel_engine_pm_get_if_awake(engine)) |
1238 | return 0; |
1239 | |
1240 | reset_prepare_engine(engine); |
1241 | |
1242 | if (msg) |
1243 | drm_notice(&engine->i915->drm,printf("drm:pid%d:%s *NOTICE* " "[drm] " "Resetting %s for %s\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, msg) |
1244 | "Resetting %s for %s\n", engine->name, msg)printf("drm:pid%d:%s *NOTICE* " "[drm] " "Resetting %s for %s\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, msg); |
1245 | atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class])__sync_fetch_and_add(&engine->i915->gpu_error.reset_engine_count [engine->uabi_class], 1); |
1246 | |
1247 | ret = intel_gt_reset_engine(engine); |
1248 | if (ret) { |
1249 | /* If we fail here, we expect to fallback to a global reset */ |
1250 | ENGINE_TRACE(engine, "Failed to reset %s, err: %d\n", engine->name, ret)do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
1251 | goto out; |
1252 | } |
1253 | |
1254 | /* |
1255 | * The request that caused the hang is stuck on elsp, we know the |
1256 | * active request and can drop it, adjust head to skip the offending |
1257 | * request to resume executing remaining requests in the queue. |
1258 | */ |
1259 | __intel_engine_reset(engine, true1); |
1260 | |
1261 | /* |
1262 | * The engine and its registers (and workarounds in case of render) |
1263 | * have been reset to their default values. Follow the init_ring |
1264 | * process to program RING_MODE, HWSP and re-enable submission. |
1265 | */ |
1266 | ret = intel_engine_resume(engine); |
1267 | |
1268 | out: |
1269 | intel_engine_cancel_stop_cs(engine); |
1270 | reset_finish_engine(engine); |
1271 | intel_engine_pm_put_async(engine); |
1272 | return ret; |
1273 | } |
1274 | |
1275 | /** |
1276 | * intel_engine_reset - reset GPU engine to recover from a hang |
1277 | * @engine: engine to reset |
1278 | * @msg: reason for GPU reset; or NULL for no drm_notice() |
1279 | * |
1280 | * Reset a specific GPU engine. Useful if a hang is detected. |
1281 | * Returns zero on successful reset or otherwise an error code. |
1282 | * |
1283 | * Procedure is: |
1284 | * - identifies the request that caused the hang and it is dropped |
1285 | * - reset engine (which will force the engine to idle) |
1286 | * - re-init/configure engine |
1287 | */ |
1288 | int intel_engine_reset(struct intel_engine_cs *engine, const char *msg) |
1289 | { |
1290 | int err; |
1291 | |
1292 | local_bh_disable(); |
1293 | err = __intel_engine_reset_bh(engine, msg); |
1294 | local_bh_enable(); |
1295 | |
1296 | return err; |
1297 | } |
1298 | |
1299 | static void intel_gt_reset_global(struct intel_gt *gt, |
1300 | u32 engine_mask, |
1301 | const char *reason) |
1302 | { |
1303 | #ifdef notyet |
1304 | struct kobject *kobj = >->i915->drm.primary->kdev->kobj; |
1305 | char *error_event[] = { I915_ERROR_UEVENT"ERROR" "=1", NULL((void *)0) }; |
1306 | char *reset_event[] = { I915_RESET_UEVENT"RESET" "=1", NULL((void *)0) }; |
1307 | char *reset_done_event[] = { I915_ERROR_UEVENT"ERROR" "=0", NULL((void *)0) }; |
1308 | #endif |
1309 | struct intel_wedge_me w; |
1310 | |
1311 | kobject_uevent_env(kobj, KOBJ_CHANGE, error_event); |
1312 | |
1313 | GT_TRACE(gt, "resetting chip, engines=%x\n", engine_mask)do { const struct intel_gt *gt__ __attribute__((__unused__)) = (gt); do { } while (0); } while (0); |
1314 | kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event); |
1315 | |
1316 | /* Use a watchdog to ensure that our reset completes */ |
1317 | intel_wedge_on_timeout(&w, gt, 5 * HZ)for (__intel_init_wedge((&w), (gt), (5 * hz), __func__); ( &w)->gt; __intel_fini_wedge((&w))) { |
1318 | intel_display_prepare_reset(gt->i915); |
1319 | |
1320 | intel_gt_reset(gt, engine_mask, reason); |
1321 | |
1322 | intel_display_finish_reset(gt->i915); |
1323 | } |
1324 | |
1325 | if (!test_bit(I915_WEDGED(64 - 1), >->reset.flags)) |
1326 | kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event); |
1327 | } |
1328 | |
1329 | /** |
1330 | * intel_gt_handle_error - handle a gpu error |
1331 | * @gt: the intel_gt |
1332 | * @engine_mask: mask representing engines that are hung |
1333 | * @flags: control flags |
1334 | * @fmt: Error message format string |
1335 | * |
1336 | * Do some basic checking of register state at error time and |
1337 | * dump it to the syslog. Also call i915_capture_error_state() to make |
1338 | * sure we get a record and make it available in debugfs. Fire a uevent |
1339 | * so userspace knows something bad happened (should trigger collection |
1340 | * of a ring dump etc.). |
1341 | */ |
1342 | void intel_gt_handle_error(struct intel_gt *gt, |
1343 | intel_engine_mask_t engine_mask, |
1344 | unsigned long flags, |
1345 | const char *fmt, ...) |
1346 | { |
1347 | struct intel_engine_cs *engine; |
1348 | intel_wakeref_t wakeref; |
1349 | intel_engine_mask_t tmp; |
1350 | char error_msg[80]; |
1351 | char *msg = NULL((void *)0); |
1352 | |
1353 | if (fmt) { |
1354 | va_list args; |
1355 | |
1356 | va_start(args, fmt)__builtin_va_start((args), fmt); |
1357 | vscnprintf(error_msg, sizeof(error_msg), fmt, args); |
1358 | va_end(args)__builtin_va_end((args)); |
1359 | |
1360 | msg = error_msg; |
1361 | } |
1362 | |
1363 | /* |
1364 | * In most cases it's guaranteed that we get here with an RPM |
1365 | * reference held, for example because there is a pending GPU |
1366 | * request that won't finish until the reset is done. This |
1367 | * isn't the case at least when we get here by doing a |
1368 | * simulated reset via debugfs, so get an RPM reference. |
1369 | */ |
1370 | wakeref = intel_runtime_pm_get(gt->uncore->rpm); |
1371 | |
1372 | engine_mask &= gt->info.engine_mask; |
1373 | |
1374 | if (flags & I915_ERROR_CAPTURE(1UL << (0))) { |
1375 | i915_capture_error_state(gt, engine_mask, CORE_DUMP_FLAG_NONE0x0); |
1376 | intel_gt_clear_error_registers(gt, engine_mask); |
1377 | } |
1378 | |
1379 | /* |
1380 | * Try engine reset when available. We fall back to full reset if |
1381 | * single reset fails. |
1382 | */ |
1383 | if (!intel_uc_uses_guc_submission(>->uc) && |
1384 | intel_has_reset_engine(gt) && !intel_gt_is_wedged(gt)) { |
1385 | local_bh_disable(); |
1386 | for_each_engine_masked(engine, gt, engine_mask, tmp)for ((tmp) = (engine_mask) & (gt)->info.engine_mask; ( tmp) ? ((engine) = (gt)->engine[({ int __idx = ffs(tmp) - 1 ; tmp &= ~(1UL << (__idx)); __idx; })]), 1 : 0;) { |
1387 | BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE)extern char _ctassert[(!(1 >= 2)) ? 1 : -1 ] __attribute__ ((__unused__)); |
1388 | if (test_and_set_bit(I915_RESET_ENGINE2 + engine->id, |
1389 | >->reset.flags)) |
1390 | continue; |
1391 | |
1392 | if (__intel_engine_reset_bh(engine, msg) == 0) |
1393 | engine_mask &= ~engine->mask; |
1394 | |
1395 | clear_and_wake_up_bit(I915_RESET_ENGINE2 + engine->id, |
1396 | >->reset.flags); |
1397 | } |
1398 | local_bh_enable(); |
1399 | } |
1400 | |
1401 | if (!engine_mask) |
1402 | goto out; |
1403 | |
1404 | /* Full reset needs the mutex, stop any other user trying to do so. */ |
1405 | if (test_and_set_bit(I915_RESET_BACKOFF0, >->reset.flags)) { |
1406 | wait_event(gt->reset.queue,do { if (!(!test_bit(0, >->reset.flags))) ({ long __ret = 0; struct wait_queue_entry __wq_entry; init_wait_entry(& __wq_entry, 0); do { int __error, __wait; unsigned long deadline ; ((!cold) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c" , 1407, "!cold")); prepare_to_wait(>->reset.queue, & __wq_entry, 0); deadline = jiffies + __ret; __wait = !(!test_bit (0, >->reset.flags)); __error = sleep_finish(__ret, __wait ); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); } while (0) |
1407 | !test_bit(I915_RESET_BACKOFF, >->reset.flags))do { if (!(!test_bit(0, >->reset.flags))) ({ long __ret = 0; struct wait_queue_entry __wq_entry; init_wait_entry(& __wq_entry, 0); do { int __error, __wait; unsigned long deadline ; ((!cold) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c" , 1407, "!cold")); prepare_to_wait(>->reset.queue, & __wq_entry, 0); deadline = jiffies + __ret; __wait = !(!test_bit (0, >->reset.flags)); __error = sleep_finish(__ret, __wait ); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); } while (0); |
1408 | goto out; /* piggy-back on the other reset */ |
1409 | } |
1410 | |
1411 | /* Make sure i915_reset_trylock() sees the I915_RESET_BACKOFF */ |
1412 | synchronize_rcu_expedited(); |
1413 | |
1414 | /* |
1415 | * Prevent any other reset-engine attempt. We don't do this for GuC |
1416 | * submission the GuC owns the per-engine reset, not the i915. |
1417 | */ |
1418 | if (!intel_uc_uses_guc_submission(>->uc)) { |
1419 | for_each_engine(engine, gt, tmp)for ((tmp) = 0; (tmp) < I915_NUM_ENGINES; (tmp)++) if (!(( engine) = (gt)->engine[(tmp)])) {} else { |
1420 | while (test_and_set_bit(I915_RESET_ENGINE2 + engine->id, |
1421 | >->reset.flags)) |
1422 | wait_on_bit(>->reset.flags, |
1423 | I915_RESET_ENGINE2 + engine->id, |
1424 | TASK_UNINTERRUPTIBLE0); |
1425 | } |
1426 | } |
1427 | |
1428 | /* Flush everyone using a resource about to be clobbered */ |
1429 | synchronize_srcu_expedited(>->reset.backoff_srcu); |
1430 | |
1431 | intel_gt_reset_global(gt, engine_mask, msg); |
1432 | |
1433 | if (!intel_uc_uses_guc_submission(>->uc)) { |
1434 | for_each_engine(engine, gt, tmp)for ((tmp) = 0; (tmp) < I915_NUM_ENGINES; (tmp)++) if (!(( engine) = (gt)->engine[(tmp)])) {} else |
1435 | clear_bit_unlock(I915_RESET_ENGINE2 + engine->id, |
1436 | >->reset.flags); |
1437 | } |
1438 | clear_bit_unlock(I915_RESET_BACKOFF0, >->reset.flags); |
1439 | smp_mb__after_atomic()do { } while (0); |
1440 | wake_up_all(>->reset.queue)wake_up(>->reset.queue); |
1441 | |
1442 | out: |
1443 | intel_runtime_pm_put(gt->uncore->rpm, wakeref); |
1444 | } |
1445 | |
1446 | int intel_gt_reset_trylock(struct intel_gt *gt, int *srcu) |
1447 | { |
1448 | might_lock(>->reset.backoff_srcu); |
1449 | might_sleep()assertwaitok(); |
1450 | |
1451 | rcu_read_lock(); |
1452 | while (test_bit(I915_RESET_BACKOFF0, >->reset.flags)) { |
1453 | rcu_read_unlock(); |
1454 | |
1455 | if (wait_event_interruptible(gt->reset.queue,({ int __ret = 0; if (!(!test_bit(0, >->reset.flags) )) __ret = ({ long __ret = 0; struct wait_queue_entry __wq_entry ; init_wait_entry(&__wq_entry, 0); do { int __error, __wait ; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c", 1457, "!cold" )); prepare_to_wait(>->reset.queue, &__wq_entry, 0x100); deadline = jiffies + __ret; __wait = !(!test_bit(0, & gt->reset.flags)); __error = sleep_finish(__ret, __wait); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); __ret; }) |
1456 | !test_bit(I915_RESET_BACKOFF,({ int __ret = 0; if (!(!test_bit(0, >->reset.flags) )) __ret = ({ long __ret = 0; struct wait_queue_entry __wq_entry ; init_wait_entry(&__wq_entry, 0); do { int __error, __wait ; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c", 1457, "!cold" )); prepare_to_wait(>->reset.queue, &__wq_entry, 0x100); deadline = jiffies + __ret; __wait = !(!test_bit(0, & gt->reset.flags)); __error = sleep_finish(__ret, __wait); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); __ret; }) |
1457 | >->reset.flags))({ int __ret = 0; if (!(!test_bit(0, >->reset.flags) )) __ret = ({ long __ret = 0; struct wait_queue_entry __wq_entry ; init_wait_entry(&__wq_entry, 0); do { int __error, __wait ; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c", 1457, "!cold" )); prepare_to_wait(>->reset.queue, &__wq_entry, 0x100); deadline = jiffies + __ret; __wait = !(!test_bit(0, & gt->reset.flags)); __error = sleep_finish(__ret, __wait); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); __ret; })) |
1458 | return -EINTR4; |
1459 | |
1460 | rcu_read_lock(); |
1461 | } |
1462 | *srcu = srcu_read_lock(>->reset.backoff_srcu)0; |
1463 | rcu_read_unlock(); |
1464 | |
1465 | return 0; |
1466 | } |
1467 | |
1468 | void intel_gt_reset_unlock(struct intel_gt *gt, int tag) |
1469 | __releases(>->reset.backoff_srcu) |
1470 | { |
1471 | srcu_read_unlock(>->reset.backoff_srcu, tag); |
1472 | } |
1473 | |
1474 | int intel_gt_terminally_wedged(struct intel_gt *gt) |
1475 | { |
1476 | might_sleep()assertwaitok(); |
1477 | |
1478 | if (!intel_gt_is_wedged(gt)) |
1479 | return 0; |
1480 | |
1481 | if (intel_gt_has_unrecoverable_error(gt)) |
1482 | return -EIO5; |
1483 | |
1484 | /* Reset still in progress? Maybe we will recover? */ |
1485 | if (wait_event_interruptible(gt->reset.queue,({ int __ret = 0; if (!(!test_bit(0, >->reset.flags) )) __ret = ({ long __ret = 0; struct wait_queue_entry __wq_entry ; init_wait_entry(&__wq_entry, 0); do { int __error, __wait ; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c", 1487, "!cold" )); prepare_to_wait(>->reset.queue, &__wq_entry, 0x100); deadline = jiffies + __ret; __wait = !(!test_bit(0, & gt->reset.flags)); __error = sleep_finish(__ret, __wait); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); __ret; }) |
1486 | !test_bit(I915_RESET_BACKOFF,({ int __ret = 0; if (!(!test_bit(0, >->reset.flags) )) __ret = ({ long __ret = 0; struct wait_queue_entry __wq_entry ; init_wait_entry(&__wq_entry, 0); do { int __error, __wait ; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c", 1487, "!cold" )); prepare_to_wait(>->reset.queue, &__wq_entry, 0x100); deadline = jiffies + __ret; __wait = !(!test_bit(0, & gt->reset.flags)); __error = sleep_finish(__ret, __wait); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); __ret; }) |
1487 | >->reset.flags))({ int __ret = 0; if (!(!test_bit(0, >->reset.flags) )) __ret = ({ long __ret = 0; struct wait_queue_entry __wq_entry ; init_wait_entry(&__wq_entry, 0); do { int __error, __wait ; unsigned long deadline; ((!cold) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/dev/pci/drm/i915/gt/intel_reset.c", 1487, "!cold" )); prepare_to_wait(>->reset.queue, &__wq_entry, 0x100); deadline = jiffies + __ret; __wait = !(!test_bit(0, & gt->reset.flags)); __error = sleep_finish(__ret, __wait); if ((0) > 0) __ret = deadline - jiffies; if (__error == -1 || __error == 4) { __ret = -4; break; } if ((0) > 0 && (__ret <= 0 || __error == 35)) { __ret = ((!test_bit(0, & gt->reset.flags))) ? 1 : 0; break; } } while (__ret > 0 && !(!test_bit(0, >->reset.flags))); finish_wait (>->reset.queue, &__wq_entry); __ret; }); __ret; })) |
1488 | return -EINTR4; |
1489 | |
1490 | return intel_gt_is_wedged(gt) ? -EIO5 : 0; |
1491 | } |
1492 | |
1493 | void intel_gt_set_wedged_on_init(struct intel_gt *gt) |
1494 | { |
1495 | BUILD_BUG_ON(I915_RESET_ENGINE + I915_NUM_ENGINES >extern char _ctassert[(!(2 + I915_NUM_ENGINES > (64 - 3))) ? 1 : -1 ] __attribute__((__unused__)) |
1496 | I915_WEDGED_ON_INIT)extern char _ctassert[(!(2 + I915_NUM_ENGINES > (64 - 3))) ? 1 : -1 ] __attribute__((__unused__)); |
1497 | intel_gt_set_wedged(gt); |
1498 | i915_disable_error_state(gt->i915, -ENODEV19); |
1499 | set_bit(I915_WEDGED_ON_INIT(64 - 3), >->reset.flags); |
1500 | |
1501 | /* Wedged on init is non-recoverable */ |
1502 | add_taint_for_CI(gt->i915, TAINT_WARN1); |
1503 | } |
1504 | |
1505 | void intel_gt_set_wedged_on_fini(struct intel_gt *gt) |
1506 | { |
1507 | intel_gt_set_wedged(gt); |
1508 | i915_disable_error_state(gt->i915, -ENODEV19); |
1509 | set_bit(I915_WEDGED_ON_FINI(64 - 2), >->reset.flags); |
1510 | intel_gt_retire_requests(gt); /* cleanup any wedged requests */ |
1511 | } |
1512 | |
1513 | void intel_gt_init_reset(struct intel_gt *gt) |
1514 | { |
1515 | init_waitqueue_head(>->reset.queue); |
1516 | rw_init(>->reset.mutex, "gtres")_rw_init_flags(>->reset.mutex, "gtres", 0, ((void *) 0)); |
1517 | init_srcu_struct(>->reset.backoff_srcu); |
1518 | |
1519 | /* |
1520 | * While undesirable to wait inside the shrinker, complain anyway. |
1521 | * |
1522 | * If we have to wait during shrinking, we guarantee forward progress |
1523 | * by forcing the reset. Therefore during the reset we must not |
1524 | * re-enter the shrinker. By declaring that we take the reset mutex |
1525 | * within the shrinker, we forbid ourselves from performing any |
1526 | * fs-reclaim or taking related locks during reset. |
1527 | */ |
1528 | i915_gem_shrinker_taints_mutex(gt->i915, >->reset.mutex); |
1529 | |
1530 | /* no GPU until we are ready! */ |
1531 | __set_bit(I915_WEDGED(64 - 1), >->reset.flags); |
1532 | } |
1533 | |
1534 | void intel_gt_fini_reset(struct intel_gt *gt) |
1535 | { |
1536 | cleanup_srcu_struct(>->reset.backoff_srcu); |
1537 | } |
1538 | |
1539 | static void intel_wedge_me(struct work_struct *work) |
1540 | { |
1541 | struct intel_wedge_me *w = container_of(work, typeof(*w), work.work)({ const __typeof( ((typeof(*w) *)0)->work.work ) *__mptr = (work); (typeof(*w) *)( (char *)__mptr - __builtin_offsetof( typeof(*w), work.work) );}); |
1542 | |
1543 | drm_err(&w->gt->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s timed out, cancelling all in-flight rendering.\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , w-> name) |
1544 | "%s timed out, cancelling all in-flight rendering.\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s timed out, cancelling all in-flight rendering.\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , w-> name) |
1545 | w->name)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s timed out, cancelling all in-flight rendering.\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , w-> name); |
1546 | intel_gt_set_wedged(w->gt); |
1547 | } |
1548 | |
1549 | void __intel_init_wedge(struct intel_wedge_me *w, |
1550 | struct intel_gt *gt, |
1551 | long timeout, |
1552 | const char *name) |
1553 | { |
1554 | w->gt = gt; |
1555 | w->name = name; |
1556 | |
1557 | INIT_DELAYED_WORK_ONSTACK(&w->work, intel_wedge_me); |
1558 | schedule_delayed_work(&w->work, timeout); |
1559 | } |
1560 | |
1561 | void __intel_fini_wedge(struct intel_wedge_me *w) |
1562 | { |
1563 | cancel_delayed_work_sync(&w->work); |
1564 | destroy_delayed_work_on_stack(&w->work); |
1565 | w->gt = NULL((void *)0); |
1566 | } |
1567 | |
1568 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0 |
1569 | #include "selftest_reset.c" |
1570 | #include "selftest_hangcheck.c" |
1571 | #endif |