File: | dev/pci/drm/i915/gt/intel_ring_submission.c |
Warning: | line 1175, column 27 Value stored to 'i915' during its initialization is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* |
2 | * Copyright © 2008-2010 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Eric Anholt <eric@anholt.net> |
25 | * Zou Nan hai <nanhai.zou@intel.com> |
26 | * Xiang Hai hao<haihao.xiang@intel.com> |
27 | * |
28 | */ |
29 | |
30 | #include "gen2_engine_cs.h" |
31 | #include "gen6_engine_cs.h" |
32 | #include "gen6_ppgtt.h" |
33 | #include "gen7_renderclear.h" |
34 | #include "i915_drv.h" |
35 | #include "i915_mitigations.h" |
36 | #include "intel_breadcrumbs.h" |
37 | #include "intel_context.h" |
38 | #include "intel_gt.h" |
39 | #include "intel_reset.h" |
40 | #include "intel_ring.h" |
41 | #include "shmem_utils.h" |
42 | |
43 | /* Rough estimate of the typical request size, performing a flush, |
44 | * set-context and then emitting the batch. |
45 | */ |
46 | #define LEGACY_REQUEST_SIZE200 200 |
47 | |
48 | static void set_hwstam(struct intel_engine_cs *engine, u32 mask) |
49 | { |
50 | /* |
51 | * Keep the render interrupt unmasked as this papers over |
52 | * lost interrupts following a reset. |
53 | */ |
54 | if (engine->class == RENDER_CLASS0) { |
55 | if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) >= 6) |
56 | mask &= ~BIT(0)(1UL << (0)); |
57 | else |
58 | mask &= ~I915_USER_INTERRUPT(1 << 1); |
59 | } |
60 | |
61 | intel_engine_set_hwsp_writemask(engine, mask); |
62 | } |
63 | |
64 | static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) |
65 | { |
66 | u32 addr; |
67 | |
68 | addr = lower_32_bits(phys)((u32)(phys)); |
69 | if (INTEL_GEN(engine->i915)((&(engine->i915)->__info)->gen) >= 4) |
70 | addr |= (phys >> 28) & 0xf0; |
71 | |
72 | intel_uncore_write(engine->uncore, HWS_PGA((const i915_reg_t){ .reg = (0x2080) }), addr); |
73 | } |
74 | |
75 | static struct vm_page *status_page(struct intel_engine_cs *engine) |
76 | { |
77 | struct drm_i915_gem_object *obj = engine->status_page.vma->obj; |
78 | |
79 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj))((void)0); |
80 | return sg_page(obj->mm.pages->sgl); |
81 | } |
82 | |
83 | static void ring_setup_phys_status_page(struct intel_engine_cs *engine) |
84 | { |
85 | set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))(((((status_page(engine))->phys_addr) / (1 << 12))) << 12)); |
86 | set_hwstam(engine, ~0u); |
87 | } |
88 | |
89 | static void set_hwsp(struct intel_engine_cs *engine, u32 offset) |
90 | { |
91 | i915_reg_t hwsp; |
92 | |
93 | /* |
94 | * The ring status page addresses are no longer next to the rest of |
95 | * the ring registers as of gen7. |
96 | */ |
97 | if (IS_GEN(engine->i915, 7)(0 + (&(engine->i915)->__info)->gen == (7))) { |
98 | switch (engine->id) { |
99 | /* |
100 | * No more rings exist on Gen7. Default case is only to shut up |
101 | * gcc switch check warning. |
102 | */ |
103 | default: |
104 | GEM_BUG_ON(engine->id)((void)0); |
105 | fallthroughdo {} while (0); |
106 | case RCS0: |
107 | hwsp = RENDER_HWS_PGA_GEN7((const i915_reg_t){ .reg = (0x04080) }); |
108 | break; |
109 | case BCS0: |
110 | hwsp = BLT_HWS_PGA_GEN7((const i915_reg_t){ .reg = (0x04280) }); |
111 | break; |
112 | case VCS0: |
113 | hwsp = BSD_HWS_PGA_GEN7((const i915_reg_t){ .reg = (0x04180) }); |
114 | break; |
115 | case VECS0: |
116 | hwsp = VEBOX_HWS_PGA_GEN7((const i915_reg_t){ .reg = (0x04380) }); |
117 | break; |
118 | } |
119 | } else if (IS_GEN(engine->i915, 6)(0 + (&(engine->i915)->__info)->gen == (6))) { |
120 | hwsp = RING_HWS_PGA_GEN6(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x2080) }); |
121 | } else { |
122 | hwsp = RING_HWS_PGA(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x80) } ); |
123 | } |
124 | |
125 | intel_uncore_write(engine->uncore, hwsp, offset); |
126 | intel_uncore_posting_read(engine->uncore, hwsp)((void)intel_uncore_read_notrace(engine->uncore, hwsp)); |
127 | } |
128 | |
129 | static void flush_cs_tlb(struct intel_engine_cs *engine) |
130 | { |
131 | struct drm_i915_privateinteldrm_softc *dev_priv = engine->i915; |
132 | |
133 | if (!IS_GEN_RANGE(dev_priv, 6, 7)(!!((&(dev_priv)->__info)->gen_mask & ( 0 + 0 + (((~0UL) >> (64 - (((7)) - 1) - 1)) & ((~0UL) << (((6)) - 1))))))) |
134 | return; |
135 | |
136 | /* ring should be idle before issuing a sync flush*/ |
137 | drm_WARN_ON(&dev_priv->drm,({ int __ret = !!(((intel_uncore_read(((engine))->uncore, ( (const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) } )) & (1 << 9)) == 0)); if (__ret) printf("%s %s: " "%s" , dev_driver_string(((&dev_priv->drm))->dev), "", "drm_WARN_ON(" "(intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) })) & (1 << 9)) == 0" ")"); __builtin_expect(!!(__ret), 0); }) |
138 | (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0)({ int __ret = !!(((intel_uncore_read(((engine))->uncore, ( (const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) } )) & (1 << 9)) == 0)); if (__ret) printf("%s %s: " "%s" , dev_driver_string(((&dev_priv->drm))->dev), "", "drm_WARN_ON(" "(intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) })) & (1 << 9)) == 0" ")"); __builtin_expect(!!(__ret), 0); }); |
139 | |
140 | ENGINE_WRITE(engine, RING_INSTPM,intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0xc0) }), (({ typeof((1 << 9) | (1 << 5)) _a = ((1 << 9) | (1 << 5)); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p(_a)) do { } while (0); ((_a) << 16 | (_a)); }); }))) |
141 | _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0xc0) }), (({ typeof((1 << 9) | (1 << 5)) _a = ((1 << 9) | (1 << 5)); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p(_a)) do { } while (0); ((_a) << 16 | (_a)); }); }))) |
142 | INSTPM_SYNC_FLUSH))intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0xc0) }), (({ typeof((1 << 9) | (1 << 5)) _a = ((1 << 9) | (1 << 5)); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p(_a)) do { } while (0); ((_a) << 16 | (_a)); }); }))); |
143 | if (intel_wait_for_register(engine->uncore, |
144 | RING_INSTPM(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0xc0) } ), |
145 | INSTPM_SYNC_FLUSH(1 << 5), 0, |
146 | 1000)) |
147 | drm_err(&dev_priv->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s: wait for SyncFlush to complete for TLB invalidation timed out\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name) |
148 | "%s: wait for SyncFlush to complete for TLB invalidation timed out\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s: wait for SyncFlush to complete for TLB invalidation timed out\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name) |
149 | engine->name)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s: wait for SyncFlush to complete for TLB invalidation timed out\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name); |
150 | } |
151 | |
152 | static void ring_setup_status_page(struct intel_engine_cs *engine) |
153 | { |
154 | set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); |
155 | set_hwstam(engine, ~0u); |
156 | |
157 | flush_cs_tlb(engine); |
158 | } |
159 | |
160 | static bool_Bool stop_ring(struct intel_engine_cs *engine) |
161 | { |
162 | struct drm_i915_privateinteldrm_softc *dev_priv = engine->i915; |
163 | |
164 | if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) > 2) { |
165 | ENGINE_WRITE(engine,intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x9c) }), (({ typeof((1 << 8)) _a = ((1 << 8)); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0 ); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }))) |
166 | RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING))intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x9c) }), (({ typeof((1 << 8)) _a = ((1 << 8)); ({ if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0 ); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }))); |
167 | if (intel_wait_for_register(engine->uncore, |
168 | RING_MI_MODE(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x9c) } ), |
169 | MODE_IDLE(1 << 9), |
170 | MODE_IDLE(1 << 9), |
171 | 1000)) { |
172 | drm_err(&dev_priv->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s : timed out trying to stop ring\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name) |
173 | "%s : timed out trying to stop ring\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s : timed out trying to stop ring\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name) |
174 | engine->name)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s : timed out trying to stop ring\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name); |
175 | |
176 | /* |
177 | * Sometimes we observe that the idle flag is not |
178 | * set even though the ring is empty. So double |
179 | * check before giving up. |
180 | */ |
181 | if (ENGINE_READ(engine, RING_HEAD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })) != |
182 | ENGINE_READ(engine, RING_TAIL)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) }))) |
183 | return false0; |
184 | } |
185 | } |
186 | |
187 | ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL))intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x34) }), (intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })))); |
188 | |
189 | ENGINE_WRITE(engine, RING_HEAD, 0)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x34) }), (0)); |
190 | ENGINE_WRITE(engine, RING_TAIL, 0)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x30) }), (0)); |
191 | |
192 | /* The ring must be empty before it is disabled */ |
193 | ENGINE_WRITE(engine, RING_CTL, 0)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x3c) }), (0)); |
194 | |
195 | return (ENGINE_READ(engine, RING_HEAD)intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })) & HEAD_ADDR0x001FFFFC) == 0; |
196 | } |
197 | |
198 | static struct i915_address_space *vm_alias(struct i915_address_space *vm) |
199 | { |
200 | if (i915_is_ggtt(vm)((vm)->is_ggtt)) |
201 | vm = &i915_vm_to_ggtt(vm)->alias->vm; |
202 | |
203 | return vm; |
204 | } |
205 | |
206 | static u32 pp_dir(struct i915_address_space *vm) |
207 | { |
208 | return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir; |
209 | } |
210 | |
211 | static void set_pp_dir(struct intel_engine_cs *engine) |
212 | { |
213 | struct i915_address_space *vm = vm_alias(engine->gt->vm); |
214 | |
215 | if (vm) { |
216 | ENGINE_WRITE(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x220) }), (0xffffffff)); |
217 | ENGINE_WRITE(engine, RING_PP_DIR_BASE, pp_dir(vm))intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x228) }), (pp_dir(vm))); |
218 | } |
219 | } |
220 | |
221 | static int xcs_resume(struct intel_engine_cs *engine) |
222 | { |
223 | struct drm_i915_privateinteldrm_softc *dev_priv = engine->i915; |
224 | struct intel_ring *ring = engine->legacy.ring; |
225 | int ret = 0; |
226 | |
227 | ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0) |
228 | ring->head, ring->tail)do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
229 | |
230 | intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); |
231 | |
232 | /* WaClearRingBufHeadRegAtInit:ctg,elk */ |
233 | if (!stop_ring(engine)) { |
234 | /* G45 ring initialization often fails to reset head to zero */ |
235 | drm_dbg(&dev_priv->drm, "%s head not reset to zero "drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
236 | "ctl %08x head %08x tail %08x start %08x\n",drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
237 | engine->name,drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
238 | ENGINE_READ(engine, RING_CTL),drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
239 | ENGINE_READ(engine, RING_HEAD),drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
240 | ENGINE_READ(engine, RING_TAIL),drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
241 | ENGINE_READ(engine, RING_START))drm_dev_dbg((&dev_priv->drm)->dev, DRM_UT_DRIVER, "%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name , intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x30) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))); |
242 | |
243 | if (!stop_ring(engine)) { |
244 | drm_err(&dev_priv->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
245 | "failed to set %s head to zero "printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
246 | "ctl %08x head %08x tail %08x start %08x\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
247 | engine->name,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
248 | ENGINE_READ(engine, RING_CTL),printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
249 | ENGINE_READ(engine, RING_HEAD),printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
250 | ENGINE_READ(engine, RING_TAIL),printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))) |
251 | ENGINE_READ(engine, RING_START))printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p-> ps_pid, __func__ , engine->name, intel_uncore_read(((engine ))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base ) + 0x3c) })), intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x34) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), intel_uncore_read(((engine))-> uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x38) }))); |
252 | ret = -EIO5; |
253 | goto out; |
254 | } |
255 | } |
256 | |
257 | if (HWS_NEEDS_PHYSICAL(dev_priv)((&(dev_priv)->__info)->hws_needs_physical)) |
258 | ring_setup_phys_status_page(engine); |
259 | else |
260 | ring_setup_status_page(engine); |
261 | |
262 | intel_breadcrumbs_reset(engine->breadcrumbs); |
263 | |
264 | /* Enforce ordering by reading HEAD register back */ |
265 | ENGINE_POSTING_READ(engine, RING_HEAD)((void)__raw_uncore_read32(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x34) }))); |
266 | |
267 | /* |
268 | * Initialize the ring. This must happen _after_ we've cleared the ring |
269 | * registers with the above sequence (the readback of the HEAD registers |
270 | * also enforces ordering), otherwise the hw might lose the new ring |
271 | * register values. |
272 | */ |
273 | ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma))intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x38) }), (i915_ggtt_offset (ring->vma))); |
274 | |
275 | /* Check that the ring offsets point within the ring! */ |
276 | GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head))((void)0); |
277 | GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail))((void)0); |
278 | intel_ring_update_space(ring); |
279 | |
280 | set_pp_dir(engine); |
281 | |
282 | /* First wake the ring up to an empty/idle ring */ |
283 | ENGINE_WRITE(engine, RING_HEAD, ring->head)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x34) }), (ring->head )); |
284 | ENGINE_WRITE(engine, RING_TAIL, ring->head)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x30) }), (ring->head )); |
285 | ENGINE_POSTING_READ(engine, RING_TAIL)((void)__raw_uncore_read32(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x30) }))); |
286 | |
287 | ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x3c) }), (((ring->size ) - (1 << 12)) | 0x00000001)); |
288 | |
289 | /* If the head is still not zero, the ring is dead */ |
290 | if (intel_wait_for_register(engine->uncore, |
291 | RING_CTL(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x3c) } ), |
292 | RING_VALID0x00000001, RING_VALID0x00000001, |
293 | 50)) { |
294 | drm_err(&dev_priv->drm, "%s initialization failed "printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
295 | "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
296 | engine->name,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
297 | ENGINE_READ(engine, RING_CTL),printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
298 | ENGINE_READ(engine, RING_CTL) & RING_VALID,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
299 | ENGINE_READ(engine, RING_HEAD), ring->head,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
300 | ENGINE_READ(engine, RING_TAIL), ring->tail,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
301 | ENGINE_READ(engine, RING_START),printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)) |
302 | i915_ggtt_offset(ring->vma))printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__ , engine ->name, intel_uncore_read(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x3c) })), intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x3c) })) & 0x00000001, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x34) })), ring->head, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x30) })), ring->tail, intel_uncore_read (((engine))->uncore, ((const i915_reg_t){ .reg = (((engine )->mmio_base) + 0x38) })), i915_ggtt_offset(ring->vma)); |
303 | ret = -EIO5; |
304 | goto out; |
305 | } |
306 | |
307 | if (INTEL_GEN(dev_priv)((&(dev_priv)->__info)->gen) > 2) |
308 | ENGINE_WRITE(engine,intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x9c) }), ((({ if (__builtin_constant_p (((1 << 8)))) do { } while (0); if (__builtin_constant_p (0)) do { } while (0); if (__builtin_constant_p(((1 << 8 ))) && __builtin_constant_p(0)) do { } while (0); ((( (1 << 8))) << 16 | (0)); })))) |
309 | RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING))intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x9c) }), ((({ if (__builtin_constant_p (((1 << 8)))) do { } while (0); if (__builtin_constant_p (0)) do { } while (0); if (__builtin_constant_p(((1 << 8 ))) && __builtin_constant_p(0)) do { } while (0); ((( (1 << 8))) << 16 | (0)); })))); |
310 | |
311 | /* Now awake, let it get started */ |
312 | if (ring->tail != ring->head) { |
313 | ENGINE_WRITE(engine, RING_TAIL, ring->tail)intel_uncore_write(((engine))->uncore, ((const i915_reg_t) { .reg = (((engine)->mmio_base) + 0x30) }), (ring->tail )); |
314 | ENGINE_POSTING_READ(engine, RING_TAIL)((void)__raw_uncore_read32(((engine))->uncore, ((const i915_reg_t ){ .reg = (((engine)->mmio_base) + 0x30) }))); |
315 | } |
316 | |
317 | /* Papering over lost _interrupts_ immediately following the restart */ |
318 | intel_engine_signal_breadcrumbs(engine); |
319 | out: |
320 | intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); |
321 | |
322 | return ret; |
323 | } |
324 | |
325 | static void reset_prepare(struct intel_engine_cs *engine) |
326 | { |
327 | struct intel_uncore *uncore = engine->uncore; |
328 | const u32 base = engine->mmio_base; |
329 | |
330 | /* |
331 | * We stop engines, otherwise we might get failed reset and a |
332 | * dead gpu (on elk). Also as modern gpu as kbl can suffer |
333 | * from system hang if batchbuffer is progressing when |
334 | * the reset is issued, regardless of READY_TO_RESET ack. |
335 | * Thus assume it is best to stop engines on all gens |
336 | * where we have a gpu reset. |
337 | * |
338 | * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) |
339 | * |
340 | * WaMediaResetMainRingCleanup:ctg,elk (presumably) |
341 | * |
342 | * FIXME: Wa for more modern gens needs to be validated |
343 | */ |
344 | ENGINE_TRACE(engine, "\n")do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
345 | |
346 | if (intel_engine_stop_cs(engine)) |
347 | ENGINE_TRACE(engine, "timed out on STOP_RING\n")do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
348 | |
349 | intel_uncore_write_fw(uncore,__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x34) }), __raw_uncore_read32(uncore, ((const i915_reg_t) { .reg = ((base) + 0x30) }))) |
350 | RING_HEAD(base),__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x34) }), __raw_uncore_read32(uncore, ((const i915_reg_t) { .reg = ((base) + 0x30) }))) |
351 | intel_uncore_read_fw(uncore, RING_TAIL(base)))__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x34) }), __raw_uncore_read32(uncore, ((const i915_reg_t) { .reg = ((base) + 0x30) }))); |
352 | intel_uncore_posting_read_fw(uncore, RING_HEAD(base))((void)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = ((base) + 0x34) }))); /* paranoia */ |
353 | |
354 | intel_uncore_write_fw(uncore, RING_HEAD(base), 0)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x34) }), 0); |
355 | intel_uncore_write_fw(uncore, RING_TAIL(base), 0)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x30) }), 0); |
356 | intel_uncore_posting_read_fw(uncore, RING_TAIL(base))((void)__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = ((base) + 0x30) }))); |
357 | |
358 | /* The ring must be empty before it is disabled */ |
359 | intel_uncore_write_fw(uncore, RING_CTL(base), 0)__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x3c) }), 0); |
360 | |
361 | /* Check acts as a post */ |
362 | if (intel_uncore_read_fw(uncore, RING_HEAD(base))__raw_uncore_read32(uncore, ((const i915_reg_t){ .reg = ((base ) + 0x34) }))) |
363 | ENGINE_TRACE(engine, "ring head [%x] not parked\n",do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0) |
364 | intel_uncore_read_fw(uncore, RING_HEAD(base)))do { const struct intel_engine_cs *e__ __attribute__((__unused__ )) = (engine); do { } while (0); } while (0); |
365 | } |
366 | |
367 | static void reset_rewind(struct intel_engine_cs *engine, bool_Bool stalled) |
368 | { |
369 | struct i915_request *pos, *rq; |
370 | unsigned long flags; |
371 | u32 head; |
372 | |
373 | rq = NULL((void *)0); |
374 | spin_lock_irqsave(&engine->active.lock, flags)do { flags = 0; mtx_enter(&engine->active.lock); } while (0); |
375 | list_for_each_entry(pos, &engine->active.requests, sched.link)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->sched .link ) *__mptr = ((&engine->active.requests)->next ); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof (*pos), sched.link) );}); &pos->sched.link != (&engine ->active.requests); pos = ({ const __typeof( ((__typeof(*pos ) *)0)->sched.link ) *__mptr = (pos->sched.link.next); ( __typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof (*pos), sched.link) );})) { |
376 | if (!i915_request_completed(pos)) { |
377 | rq = pos; |
378 | break; |
379 | } |
380 | } |
381 | |
382 | /* |
383 | * The guilty request will get skipped on a hung engine. |
384 | * |
385 | * Users of client default contexts do not rely on logical |
386 | * state preserved between batches so it is safe to execute |
387 | * queued requests following the hang. Non default contexts |
388 | * rely on preserved state, so skipping a batch loses the |
389 | * evolution of the state and it needs to be considered corrupted. |
390 | * Executing more queued batches on top of corrupted state is |
391 | * risky. But we take the risk by trying to advance through |
392 | * the queued requests in order to make the client behaviour |
393 | * more predictable around resets, by not throwing away random |
394 | * amount of batches it has prepared for execution. Sophisticated |
395 | * clients can use gem_reset_stats_ioctl and dma fence status |
396 | * (exported via sync_file info ioctl on explicit fences) to observe |
397 | * when it loses the context state and should rebuild accordingly. |
398 | * |
399 | * The context ban, and ultimately the client ban, mechanism are safety |
400 | * valves if client submission ends up resulting in nothing more than |
401 | * subsequent hangs. |
402 | */ |
403 | |
404 | if (rq) { |
405 | /* |
406 | * Try to restore the logical GPU state to match the |
407 | * continuation of the request queue. If we skip the |
408 | * context/PD restore, then the next request may try to execute |
409 | * assuming that its context is valid and loaded on the GPU and |
410 | * so may try to access invalid memory, prompting repeated GPU |
411 | * hangs. |
412 | * |
413 | * If the request was guilty, we still restore the logical |
414 | * state in case the next request requires it (e.g. the |
415 | * aliasing ppgtt), but skip over the hung batch. |
416 | * |
417 | * If the request was innocent, we try to replay the request |
418 | * with the restored context. |
419 | */ |
420 | __i915_request_reset(rq, stalled); |
421 | |
422 | GEM_BUG_ON(rq->ring != engine->legacy.ring)((void)0); |
423 | head = rq->head; |
424 | } else { |
425 | head = engine->legacy.ring->tail; |
426 | } |
427 | engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); |
428 | |
429 | spin_unlock_irqrestore(&engine->active.lock, flags)do { (void)(flags); mtx_leave(&engine->active.lock); } while (0); |
430 | } |
431 | |
432 | static void reset_finish(struct intel_engine_cs *engine) |
433 | { |
434 | } |
435 | |
436 | static void reset_cancel(struct intel_engine_cs *engine) |
437 | { |
438 | struct i915_request *request; |
439 | unsigned long flags; |
440 | |
441 | spin_lock_irqsave(&engine->active.lock, flags)do { flags = 0; mtx_enter(&engine->active.lock); } while (0); |
442 | |
443 | /* Mark all submitted requests as skipped. */ |
444 | list_for_each_entry(request, &engine->active.requests, sched.link)for (request = ({ const __typeof( ((__typeof(*request) *)0)-> sched.link ) *__mptr = ((&engine->active.requests)-> next); (__typeof(*request) *)( (char *)__mptr - __builtin_offsetof (__typeof(*request), sched.link) );}); &request->sched .link != (&engine->active.requests); request = ({ const __typeof( ((__typeof(*request) *)0)->sched.link ) *__mptr = (request->sched.link.next); (__typeof(*request) *)( (char *)__mptr - __builtin_offsetof(__typeof(*request), sched.link ) );})) { |
445 | i915_request_set_error_once(request, -EIO5); |
446 | i915_request_mark_complete(request); |
447 | } |
448 | |
449 | /* Remaining _unready_ requests will be nop'ed when submitted */ |
450 | |
451 | spin_unlock_irqrestore(&engine->active.lock, flags)do { (void)(flags); mtx_leave(&engine->active.lock); } while (0); |
452 | } |
453 | |
454 | static void i9xx_submit_request(struct i915_request *request) |
455 | { |
456 | i915_request_submit(request); |
457 | wmb()do { __asm volatile("sfence" ::: "memory"); } while (0); /* paranoid flush writes out of the WCB before mmio */ |
458 | |
459 | ENGINE_WRITE(request->engine, RING_TAIL,intel_uncore_write(((request->engine))->uncore, ((const i915_reg_t){ .reg = (((request->engine)->mmio_base) + 0x30 ) }), (intel_ring_set_tail(request->ring, request->tail ))) |
460 | intel_ring_set_tail(request->ring, request->tail))intel_uncore_write(((request->engine))->uncore, ((const i915_reg_t){ .reg = (((request->engine)->mmio_base) + 0x30 ) }), (intel_ring_set_tail(request->ring, request->tail ))); |
461 | } |
462 | |
463 | static void __ring_context_fini(struct intel_context *ce) |
464 | { |
465 | i915_vma_put(ce->state); |
466 | } |
467 | |
468 | static void ring_context_destroy(struct kref *ref) |
469 | { |
470 | struct intel_context *ce = container_of(ref, typeof(*ce), ref)({ const __typeof( ((typeof(*ce) *)0)->ref ) *__mptr = (ref ); (typeof(*ce) *)( (char *)__mptr - __builtin_offsetof(typeof (*ce), ref) );}); |
471 | |
472 | GEM_BUG_ON(intel_context_is_pinned(ce))((void)0); |
473 | |
474 | if (ce->state) |
475 | __ring_context_fini(ce); |
476 | |
477 | intel_context_fini(ce); |
478 | intel_context_free(ce); |
479 | } |
480 | |
481 | static int ring_context_pre_pin(struct intel_context *ce, |
482 | struct i915_gem_ww_ctx *ww, |
483 | void **unused) |
484 | { |
485 | struct i915_address_space *vm; |
486 | int err = 0; |
487 | |
488 | vm = vm_alias(ce->vm); |
489 | if (vm) |
490 | err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww); |
491 | |
492 | return err; |
493 | } |
494 | |
495 | static void __context_unpin_ppgtt(struct intel_context *ce) |
496 | { |
497 | struct i915_address_space *vm; |
498 | |
499 | vm = vm_alias(ce->vm); |
500 | if (vm) |
501 | gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); |
502 | } |
503 | |
504 | static void ring_context_unpin(struct intel_context *ce) |
505 | { |
506 | } |
507 | |
508 | static void ring_context_post_unpin(struct intel_context *ce) |
509 | { |
510 | __context_unpin_ppgtt(ce); |
511 | } |
512 | |
513 | static struct i915_vma * |
514 | alloc_context_vma(struct intel_engine_cs *engine) |
515 | { |
516 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
517 | struct drm_i915_gem_object *obj; |
518 | struct i915_vma *vma; |
519 | int err; |
520 | |
521 | obj = i915_gem_object_create_shmem(i915, engine->context_size); |
522 | if (IS_ERR(obj)) |
523 | return ERR_CAST(obj); |
524 | |
525 | /* |
526 | * Try to make the context utilize L3 as well as LLC. |
527 | * |
528 | * On VLV we don't have L3 controls in the PTEs so we |
529 | * shouldn't touch the cache level, especially as that |
530 | * would make the object snooped which might have a |
531 | * negative performance impact. |
532 | * |
533 | * Snooping is required on non-llc platforms in execlist |
534 | * mode, but since all GGTT accesses use PAT entry 0 we |
535 | * get snooping anyway regardless of cache_level. |
536 | * |
537 | * This is only applicable for Ivy Bridge devices since |
538 | * later platforms don't have L3 control bits in the PTE. |
539 | */ |
540 | if (IS_IVYBRIDGE(i915)IS_PLATFORM(i915, INTEL_IVYBRIDGE)) |
541 | i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); |
542 | |
543 | if (engine->default_state) { |
544 | void *vaddr; |
545 | |
546 | vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); |
547 | if (IS_ERR(vaddr)) { |
548 | err = PTR_ERR(vaddr); |
549 | goto err_obj; |
550 | } |
551 | |
552 | #ifdef __linux__ |
553 | shmem_read(engine->default_state, 0, |
554 | vaddr, engine->context_size); |
555 | #else |
556 | uao_read(engine->default_state, 0, |
557 | vaddr, engine->context_size); |
558 | #endif |
559 | |
560 | i915_gem_object_flush_map(obj); |
561 | __i915_gem_object_release_map(obj); |
562 | } |
563 | |
564 | vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL((void *)0)); |
565 | if (IS_ERR(vma)) { |
566 | err = PTR_ERR(vma); |
567 | goto err_obj; |
568 | } |
569 | |
570 | return vma; |
571 | |
572 | err_obj: |
573 | i915_gem_object_put(obj); |
574 | return ERR_PTR(err); |
575 | } |
576 | |
577 | static int ring_context_alloc(struct intel_context *ce) |
578 | { |
579 | struct intel_engine_cs *engine = ce->engine; |
580 | |
581 | /* One ringbuffer to rule them all */ |
582 | GEM_BUG_ON(!engine->legacy.ring)((void)0); |
583 | ce->ring = engine->legacy.ring; |
584 | ce->timeline = intel_timeline_get(engine->legacy.timeline); |
585 | |
586 | GEM_BUG_ON(ce->state)((void)0); |
587 | if (engine->context_size) { |
588 | struct i915_vma *vma; |
589 | |
590 | vma = alloc_context_vma(engine); |
591 | if (IS_ERR(vma)) |
592 | return PTR_ERR(vma); |
593 | |
594 | ce->state = vma; |
595 | if (engine->default_state) |
596 | __set_bit(CONTEXT_VALID_BIT2, &ce->flags); |
597 | } |
598 | |
599 | return 0; |
600 | } |
601 | |
602 | static int ring_context_pin(struct intel_context *ce, void *unused) |
603 | { |
604 | return 0; |
605 | } |
606 | |
607 | static void ring_context_reset(struct intel_context *ce) |
608 | { |
609 | intel_ring_reset(ce->ring, ce->ring->emit); |
610 | } |
611 | |
612 | static const struct intel_context_ops ring_context_ops = { |
613 | .alloc = ring_context_alloc, |
614 | |
615 | .pre_pin = ring_context_pre_pin, |
616 | .pin = ring_context_pin, |
617 | .unpin = ring_context_unpin, |
618 | .post_unpin = ring_context_post_unpin, |
619 | |
620 | .enter = intel_context_enter_engine, |
621 | .exit = intel_context_exit_engine, |
622 | |
623 | .reset = ring_context_reset, |
624 | .destroy = ring_context_destroy, |
625 | }; |
626 | |
627 | static int load_pd_dir(struct i915_request *rq, |
628 | struct i915_address_space *vm, |
629 | u32 valid) |
630 | { |
631 | const struct intel_engine_cs * const engine = rq->engine; |
632 | u32 *cs; |
633 | |
634 | cs = intel_ring_begin(rq, 12); |
635 | if (IS_ERR(cs)) |
636 | return PTR_ERR(cs); |
637 | |
638 | *cs++ = MI_LOAD_REGISTER_IMM(1)(((0x22) << 23) | (2*(1)-1)); |
639 | *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x220) } )); |
640 | *cs++ = valid; |
641 | |
642 | *cs++ = MI_LOAD_REGISTER_IMM(1)(((0x22) << 23) | (2*(1)-1)); |
643 | *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x228) } )); |
644 | *cs++ = pp_dir(vm); |
645 | |
646 | /* Stall until the page table load is complete? */ |
647 | *cs++ = MI_STORE_REGISTER_MEM(((0x24) << 23) | (1)) | MI_SRM_LRM_GLOBAL_GTT(1<<22); |
648 | *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x228) } )); |
649 | *cs++ = intel_gt_scratch_offset(engine->gt, |
650 | INTEL_GT_SCRATCH_FIELD_DEFAULT); |
651 | |
652 | *cs++ = MI_LOAD_REGISTER_IMM(1)(((0x22) << 23) | (2*(1)-1)); |
653 | *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0xc0) } )); |
654 | *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE)({ typeof((1 << 9)) _a = ((1 << 9)); ({ if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }); |
655 | |
656 | intel_ring_advance(rq, cs); |
657 | |
658 | return rq->engine->emit_flush(rq, EMIT_FLUSH(1UL << (1))); |
659 | } |
660 | |
661 | static inline int mi_set_context(struct i915_request *rq, |
662 | struct intel_context *ce, |
663 | u32 flags) |
664 | { |
665 | struct intel_engine_cs *engine = rq->engine; |
666 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
667 | enum intel_engine_id id; |
668 | const int num_engines = |
669 | IS_HASWELL(i915)IS_PLATFORM(i915, INTEL_HASWELL) ? engine->gt->info.num_engines - 1 : 0; |
670 | bool_Bool force_restore = false0; |
671 | int len; |
672 | u32 *cs; |
673 | |
674 | len = 4; |
675 | if (IS_GEN(i915, 7)(0 + (&(i915)->__info)->gen == (7))) |
676 | len += 2 + (num_engines ? 4 * num_engines + 6 : 0); |
677 | else if (IS_GEN(i915, 5)(0 + (&(i915)->__info)->gen == (5))) |
678 | len += 2; |
679 | if (flags & MI_FORCE_RESTORE(1<<1)) { |
680 | GEM_BUG_ON(flags & MI_RESTORE_INHIBIT)((void)0); |
681 | flags &= ~MI_FORCE_RESTORE(1<<1); |
682 | force_restore = true1; |
683 | len += 2; |
684 | } |
685 | |
686 | cs = intel_ring_begin(rq, len); |
687 | if (IS_ERR(cs)) |
688 | return PTR_ERR(cs); |
689 | |
690 | /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ |
691 | if (IS_GEN(i915, 7)(0 + (&(i915)->__info)->gen == (7))) { |
692 | *cs++ = MI_ARB_ON_OFF(((0x08) << 23) | (0)) | MI_ARB_DISABLE(0<<0); |
693 | if (num_engines) { |
694 | struct intel_engine_cs *signaller; |
695 | |
696 | *cs++ = MI_LOAD_REGISTER_IMM(num_engines)(((0x22) << 23) | (2*(num_engines)-1)); |
697 | for_each_engine(signaller, engine->gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((signaller ) = (engine->gt)->engine[(id)])) {} else { |
698 | if (signaller == engine) |
699 | continue; |
700 | |
701 | *cs++ = i915_mmio_reg_offset( |
702 | RING_PSMI_CTL(signaller->mmio_base)((const i915_reg_t){ .reg = ((signaller->mmio_base) + 0x50 ) })); |
703 | *cs++ = _MASKED_BIT_ENABLE(({ typeof((1 << 0)) _a = ((1 << 0)); ({ if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }) |
704 | GEN6_PSMI_SLEEP_MSG_DISABLE)({ typeof((1 << 0)) _a = ((1 << 0)); ({ if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p (_a)) do { } while (0); ((_a) << 16 | (_a)); }); }); |
705 | } |
706 | } |
707 | } else if (IS_GEN(i915, 5)(0 + (&(i915)->__info)->gen == (5))) { |
708 | /* |
709 | * This w/a is only listed for pre-production ilk a/b steppings, |
710 | * but is also mentioned for programming the powerctx. To be |
711 | * safe, just apply the workaround; we do not use SyncFlush so |
712 | * this should never take effect and so be a no-op! |
713 | */ |
714 | *cs++ = MI_SUSPEND_FLUSH(((0x0b) << 23) | (0)) | MI_SUSPEND_FLUSH_EN(1<<0); |
715 | } |
716 | |
717 | if (force_restore) { |
718 | /* |
719 | * The HW doesn't handle being told to restore the current |
720 | * context very well. Quite often it likes goes to go off and |
721 | * sulk, especially when it is meant to be reloading PP_DIR. |
722 | * A very simple fix to force the reload is to simply switch |
723 | * away from the current context and back again. |
724 | * |
725 | * Note that the kernel_context will contain random state |
726 | * following the INHIBIT_RESTORE. We accept this since we |
727 | * never use the kernel_context state; it is merely a |
728 | * placeholder we use to flush other contexts. |
729 | */ |
730 | *cs++ = MI_SET_CONTEXT(((0x18) << 23) | (0)); |
731 | *cs++ = i915_ggtt_offset(engine->kernel_context->state) | |
732 | MI_MM_SPACE_GTT(1<<8) | |
733 | MI_RESTORE_INHIBIT(1<<0); |
734 | } |
735 | |
736 | *cs++ = MI_NOOP(((0) << 23) | (0)); |
737 | *cs++ = MI_SET_CONTEXT(((0x18) << 23) | (0)); |
738 | *cs++ = i915_ggtt_offset(ce->state) | flags; |
739 | /* |
740 | * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP |
741 | * WaMiSetContext_Hang:snb,ivb,vlv |
742 | */ |
743 | *cs++ = MI_NOOP(((0) << 23) | (0)); |
744 | |
745 | if (IS_GEN(i915, 7)(0 + (&(i915)->__info)->gen == (7))) { |
746 | if (num_engines) { |
747 | struct intel_engine_cs *signaller; |
748 | i915_reg_t last_reg = {}; /* keep gcc quiet */ |
749 | |
750 | *cs++ = MI_LOAD_REGISTER_IMM(num_engines)(((0x22) << 23) | (2*(num_engines)-1)); |
751 | for_each_engine(signaller, engine->gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((signaller ) = (engine->gt)->engine[(id)])) {} else { |
752 | if (signaller == engine) |
753 | continue; |
754 | |
755 | last_reg = RING_PSMI_CTL(signaller->mmio_base)((const i915_reg_t){ .reg = ((signaller->mmio_base) + 0x50 ) }); |
756 | *cs++ = i915_mmio_reg_offset(last_reg); |
757 | *cs++ = _MASKED_BIT_DISABLE((({ if (__builtin_constant_p(((1 << 0)))) do { } while ( 0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((1 << 0))) && __builtin_constant_p(0)) do { } while (0); ((((1 << 0))) << 16 | (0)); })) |
758 | GEN6_PSMI_SLEEP_MSG_DISABLE)(({ if (__builtin_constant_p(((1 << 0)))) do { } while ( 0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((1 << 0))) && __builtin_constant_p(0)) do { } while (0); ((((1 << 0))) << 16 | (0)); })); |
759 | } |
760 | |
761 | /* Insert a delay before the next switch! */ |
762 | *cs++ = MI_STORE_REGISTER_MEM(((0x24) << 23) | (1)) | MI_SRM_LRM_GLOBAL_GTT(1<<22); |
763 | *cs++ = i915_mmio_reg_offset(last_reg); |
764 | *cs++ = intel_gt_scratch_offset(engine->gt, |
765 | INTEL_GT_SCRATCH_FIELD_DEFAULT); |
766 | *cs++ = MI_NOOP(((0) << 23) | (0)); |
767 | } |
768 | *cs++ = MI_ARB_ON_OFF(((0x08) << 23) | (0)) | MI_ARB_ENABLE(1<<0); |
769 | } else if (IS_GEN(i915, 5)(0 + (&(i915)->__info)->gen == (5))) { |
770 | *cs++ = MI_SUSPEND_FLUSH(((0x0b) << 23) | (0)); |
771 | } |
772 | |
773 | intel_ring_advance(rq, cs); |
774 | |
775 | return 0; |
776 | } |
777 | |
778 | static int remap_l3_slice(struct i915_request *rq, int slice) |
779 | { |
780 | u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice]; |
781 | int i; |
782 | |
783 | if (!remap_info) |
784 | return 0; |
785 | |
786 | cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE0x80/4 * 2 + 2); |
787 | if (IS_ERR(cs)) |
788 | return PTR_ERR(cs); |
789 | |
790 | /* |
791 | * Note: We do not worry about the concurrent register cacheline hang |
792 | * here because no other code should access these registers other than |
793 | * at initialization time. |
794 | */ |
795 | *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4)(((0x22) << 23) | (2*(0x80/4)-1)); |
796 | for (i = 0; i < GEN7_L3LOG_SIZE0x80/4; i++) { |
797 | *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)((const i915_reg_t){ .reg = (0xB070 + (slice) * 0x200 + (i) * 4) })); |
798 | *cs++ = remap_info[i]; |
799 | } |
800 | *cs++ = MI_NOOP(((0) << 23) | (0)); |
801 | intel_ring_advance(rq, cs); |
802 | |
803 | return 0; |
804 | } |
805 | |
806 | static int remap_l3(struct i915_request *rq) |
807 | { |
808 | struct i915_gem_context *ctx = i915_request_gem_context(rq); |
809 | int i, err; |
810 | |
811 | if (!ctx || !ctx->remap_slice) |
812 | return 0; |
813 | |
814 | for (i = 0; i < MAX_L3_SLICES2; i++) { |
815 | if (!(ctx->remap_slice & BIT(i)(1UL << (i)))) |
816 | continue; |
817 | |
818 | err = remap_l3_slice(rq, i); |
819 | if (err) |
820 | return err; |
821 | } |
822 | |
823 | ctx->remap_slice = 0; |
824 | return 0; |
825 | } |
826 | |
827 | static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) |
828 | { |
829 | int ret; |
830 | |
831 | if (!vm) |
832 | return 0; |
833 | |
834 | ret = rq->engine->emit_flush(rq, EMIT_FLUSH(1UL << (1))); |
835 | if (ret) |
836 | return ret; |
837 | |
838 | /* |
839 | * Not only do we need a full barrier (post-sync write) after |
840 | * invalidating the TLBs, but we need to wait a little bit |
841 | * longer. Whether this is merely delaying us, or the |
842 | * subsequent flush is a key part of serialising with the |
843 | * post-sync op, this extra pass appears vital before a |
844 | * mm switch! |
845 | */ |
846 | ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G0xffffffff); |
847 | if (ret) |
848 | return ret; |
849 | |
850 | return rq->engine->emit_flush(rq, EMIT_INVALIDATE(1UL << (0))); |
851 | } |
852 | |
853 | static int clear_residuals(struct i915_request *rq) |
854 | { |
855 | struct intel_engine_cs *engine = rq->engine; |
856 | int ret; |
857 | |
858 | ret = switch_mm(rq, vm_alias(engine->kernel_context->vm)); |
859 | if (ret) |
860 | return ret; |
861 | |
862 | if (engine->kernel_context->state) { |
863 | ret = mi_set_context(rq, |
864 | engine->kernel_context, |
865 | MI_MM_SPACE_GTT(1<<8) | MI_RESTORE_INHIBIT(1<<0)); |
866 | if (ret) |
867 | return ret; |
868 | } |
869 | |
870 | ret = engine->emit_bb_start(rq, |
871 | engine->wa_ctx.vma->node.start, 0, |
872 | 0); |
873 | if (ret) |
874 | return ret; |
875 | |
876 | ret = engine->emit_flush(rq, EMIT_FLUSH(1UL << (1))); |
877 | if (ret) |
878 | return ret; |
879 | |
880 | /* Always invalidate before the next switch_mm() */ |
881 | return engine->emit_flush(rq, EMIT_INVALIDATE(1UL << (0))); |
882 | } |
883 | |
884 | static int switch_context(struct i915_request *rq) |
885 | { |
886 | struct intel_engine_cs *engine = rq->engine; |
887 | struct intel_context *ce = rq->context; |
888 | void **residuals = NULL((void *)0); |
889 | int ret; |
890 | |
891 | GEM_BUG_ON(HAS_EXECLISTS(engine->i915))((void)0); |
892 | |
893 | if (engine->wa_ctx.vma && ce != engine->kernel_context) { |
894 | if (engine->wa_ctx.vma->private != ce && |
895 | i915_mitigate_clear_residuals()) { |
896 | ret = clear_residuals(rq); |
897 | if (ret) |
898 | return ret; |
899 | |
900 | residuals = &engine->wa_ctx.vma->private; |
901 | } |
902 | } |
903 | |
904 | ret = switch_mm(rq, vm_alias(ce->vm)); |
905 | if (ret) |
906 | return ret; |
907 | |
908 | if (ce->state) { |
909 | u32 flags; |
910 | |
911 | GEM_BUG_ON(engine->id != RCS0)((void)0); |
912 | |
913 | /* For resource streamer on HSW+ and power context elsewhere */ |
914 | BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN)extern char _ctassert[(!((1<<3) != (1<<3))) ? 1 : -1 ] __attribute__((__unused__)); |
915 | BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN)extern char _ctassert[(!((1<<2) != (1<<2))) ? 1 : -1 ] __attribute__((__unused__)); |
916 | |
917 | flags = MI_SAVE_EXT_STATE_EN(1<<3) | MI_MM_SPACE_GTT(1<<8); |
918 | if (test_bit(CONTEXT_VALID_BIT2, &ce->flags)) |
919 | flags |= MI_RESTORE_EXT_STATE_EN(1<<2); |
920 | else |
921 | flags |= MI_RESTORE_INHIBIT(1<<0); |
922 | |
923 | ret = mi_set_context(rq, ce, flags); |
924 | if (ret) |
925 | return ret; |
926 | } |
927 | |
928 | ret = remap_l3(rq); |
929 | if (ret) |
930 | return ret; |
931 | |
932 | /* |
933 | * Now past the point of no return, this request _will_ be emitted. |
934 | * |
935 | * Or at least this preamble will be emitted, the request may be |
936 | * interrupted prior to submitting the user payload. If so, we |
937 | * still submit the "empty" request in order to preserve global |
938 | * state tracking such as this, our tracking of the current |
939 | * dirty context. |
940 | */ |
941 | if (residuals) { |
942 | intel_context_put(*residuals); |
943 | *residuals = intel_context_get(ce); |
944 | } |
945 | |
946 | return 0; |
947 | } |
948 | |
949 | static int ring_request_alloc(struct i915_request *request) |
950 | { |
951 | int ret; |
952 | |
953 | GEM_BUG_ON(!intel_context_is_pinned(request->context))((void)0); |
954 | GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb)((void)0); |
955 | |
956 | /* |
957 | * Flush enough space to reduce the likelihood of waiting after |
958 | * we start building the request - in which case we will just |
959 | * have to repeat work. |
960 | */ |
961 | request->reserved_space += LEGACY_REQUEST_SIZE200; |
962 | |
963 | /* Unconditionally invalidate GPU caches and TLBs. */ |
964 | ret = request->engine->emit_flush(request, EMIT_INVALIDATE(1UL << (0))); |
965 | if (ret) |
966 | return ret; |
967 | |
968 | ret = switch_context(request); |
969 | if (ret) |
970 | return ret; |
971 | |
972 | request->reserved_space -= LEGACY_REQUEST_SIZE200; |
973 | return 0; |
974 | } |
975 | |
976 | static void gen6_bsd_submit_request(struct i915_request *request) |
977 | { |
978 | struct intel_uncore *uncore = request->engine->uncore; |
979 | |
980 | intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); |
981 | |
982 | /* Every tail move must follow the sequence below */ |
983 | |
984 | /* Disable notification that the ring is IDLE. The GT |
985 | * will then assume that it is busy and bring it out of rc6. |
986 | */ |
987 | intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x12050 ) }), ({ typeof((1 << 0)) _a = ((1 << 0)); ({ if ( __builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p(_a)) do { } while (0); ((_a) << 16 | (_a)); }); })) |
988 | _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE))__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x12050 ) }), ({ typeof((1 << 0)) _a = ((1 << 0)); ({ if ( __builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p (_a)) do { } while (0); if (__builtin_constant_p(_a) && __builtin_constant_p(_a)) do { } while (0); ((_a) << 16 | (_a)); }); })); |
989 | |
990 | /* Clear the context id. Here be magic! */ |
991 | intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0)__raw_uncore_write64(uncore, ((const i915_reg_t){ .reg = (0x12198 ) }), 0x0); |
992 | |
993 | /* Wait for the ring not to be idle, i.e. for it to wake up. */ |
994 | if (__intel_wait_for_register_fw(uncore, |
995 | GEN6_BSD_SLEEP_PSMI_CONTROL((const i915_reg_t){ .reg = (0x12050) }), |
996 | GEN6_BSD_SLEEP_INDICATOR(1 << 3), |
997 | 0, |
998 | 1000, 0, NULL((void *)0))) |
999 | drm_err(&uncore->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "timed out waiting for the BSD ring to wake up\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__) |
1000 | "timed out waiting for the BSD ring to wake up\n")printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "timed out waiting for the BSD ring to wake up\n" , ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_p->ps_pid, __func__); |
1001 | |
1002 | /* Now that the ring is fully powered up, update the tail */ |
1003 | i9xx_submit_request(request); |
1004 | |
1005 | /* Let the ring send IDLE messages to the GT again, |
1006 | * and so let it sleep to conserve power when idle. |
1007 | */ |
1008 | intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x12050 ) }), (({ if (__builtin_constant_p(((1 << 0)))) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((1 << 0))) && __builtin_constant_p(0)) do { } while (0); ((((1 << 0))) << 16 | (0)); }))) |
1009 | _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE))__raw_uncore_write32(uncore, ((const i915_reg_t){ .reg = (0x12050 ) }), (({ if (__builtin_constant_p(((1 << 0)))) do { } while (0); if (__builtin_constant_p(0)) do { } while (0); if (__builtin_constant_p (((1 << 0))) && __builtin_constant_p(0)) do { } while (0); ((((1 << 0))) << 16 | (0)); }))); |
1010 | |
1011 | intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); |
1012 | } |
1013 | |
1014 | static void i9xx_set_default_submission(struct intel_engine_cs *engine) |
1015 | { |
1016 | engine->submit_request = i9xx_submit_request; |
1017 | |
1018 | engine->park = NULL((void *)0); |
1019 | engine->unpark = NULL((void *)0); |
1020 | } |
1021 | |
1022 | static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) |
1023 | { |
1024 | i9xx_set_default_submission(engine); |
1025 | engine->submit_request = gen6_bsd_submit_request; |
1026 | } |
1027 | |
1028 | static void ring_release(struct intel_engine_cs *engine) |
1029 | { |
1030 | struct drm_i915_privateinteldrm_softc *dev_priv = engine->i915; |
1031 | |
1032 | drm_WARN_ON(&dev_priv->drm, INTEL_GEN(dev_priv) > 2 &&({ int __ret = !!((((&(dev_priv)->__info)->gen) > 2 && (intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) })) & (1 << 9)) == 0)); if (__ret) printf("%s %s: " "%s", dev_driver_string (((&dev_priv->drm))->dev), "", "drm_WARN_ON(" "((&(dev_priv)->__info)->gen) > 2 && (intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) })) & (1 << 9)) == 0" ")"); __builtin_expect(!!(__ret), 0); }) |
1033 | (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0)({ int __ret = !!((((&(dev_priv)->__info)->gen) > 2 && (intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) })) & (1 << 9)) == 0)); if (__ret) printf("%s %s: " "%s", dev_driver_string (((&dev_priv->drm))->dev), "", "drm_WARN_ON(" "((&(dev_priv)->__info)->gen) > 2 && (intel_uncore_read(((engine))->uncore, ((const i915_reg_t){ .reg = (((engine)->mmio_base) + 0x9c) })) & (1 << 9)) == 0" ")"); __builtin_expect(!!(__ret), 0); }); |
1034 | |
1035 | intel_engine_cleanup_common(engine); |
1036 | |
1037 | if (engine->wa_ctx.vma) { |
1038 | intel_context_put(engine->wa_ctx.vma->private); |
1039 | i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); |
1040 | } |
1041 | |
1042 | intel_ring_unpin(engine->legacy.ring); |
1043 | intel_ring_put(engine->legacy.ring); |
1044 | |
1045 | intel_timeline_unpin(engine->legacy.timeline); |
1046 | intel_timeline_put(engine->legacy.timeline); |
1047 | } |
1048 | |
1049 | static void setup_irq(struct intel_engine_cs *engine) |
1050 | { |
1051 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
1052 | |
1053 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 6) { |
1054 | engine->irq_enable = gen6_irq_enable; |
1055 | engine->irq_disable = gen6_irq_disable; |
1056 | } else if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 5) { |
1057 | engine->irq_enable = gen5_irq_enable; |
1058 | engine->irq_disable = gen5_irq_disable; |
1059 | } else if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 3) { |
1060 | engine->irq_enable = gen3_irq_enable; |
1061 | engine->irq_disable = gen3_irq_disable; |
1062 | } else { |
1063 | engine->irq_enable = gen2_irq_enable; |
1064 | engine->irq_disable = gen2_irq_disable; |
1065 | } |
1066 | } |
1067 | |
1068 | static void setup_common(struct intel_engine_cs *engine) |
1069 | { |
1070 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
1071 | |
1072 | /* gen8+ are only supported with execlists */ |
1073 | GEM_BUG_ON(INTEL_GEN(i915) >= 8)((void)0); |
1074 | |
1075 | setup_irq(engine); |
1076 | |
1077 | engine->resume = xcs_resume; |
1078 | engine->reset.prepare = reset_prepare; |
1079 | engine->reset.rewind = reset_rewind; |
1080 | engine->reset.cancel = reset_cancel; |
1081 | engine->reset.finish = reset_finish; |
1082 | |
1083 | engine->cops = &ring_context_ops; |
1084 | engine->request_alloc = ring_request_alloc; |
1085 | |
1086 | /* |
1087 | * Using a global execution timeline; the previous final breadcrumb is |
1088 | * equivalent to our next initial bread so we can elide |
1089 | * engine->emit_init_breadcrumb(). |
1090 | */ |
1091 | engine->emit_fini_breadcrumb = gen3_emit_breadcrumb; |
1092 | if (IS_GEN(i915, 5)(0 + (&(i915)->__info)->gen == (5))) |
1093 | engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; |
1094 | |
1095 | engine->set_default_submission = i9xx_set_default_submission; |
1096 | |
1097 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 6) |
1098 | engine->emit_bb_start = gen6_emit_bb_start; |
1099 | else if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 4) |
1100 | engine->emit_bb_start = gen4_emit_bb_start; |
1101 | else if (IS_I830(i915)IS_PLATFORM(i915, INTEL_I830) || IS_I845G(i915)IS_PLATFORM(i915, INTEL_I845G)) |
1102 | engine->emit_bb_start = i830_emit_bb_start; |
1103 | else |
1104 | engine->emit_bb_start = gen3_emit_bb_start; |
1105 | } |
1106 | |
1107 | static void setup_rcs(struct intel_engine_cs *engine) |
1108 | { |
1109 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
1110 | |
1111 | if (HAS_L3_DPF(i915)((&(i915)->__info)->has_l3_dpf)) |
1112 | engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT(1 << 5); |
1113 | |
1114 | engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT(1 << 0); |
1115 | |
1116 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 7) { |
1117 | engine->emit_flush = gen7_emit_flush_rcs; |
1118 | engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs; |
1119 | } else if (IS_GEN(i915, 6)(0 + (&(i915)->__info)->gen == (6))) { |
1120 | engine->emit_flush = gen6_emit_flush_rcs; |
1121 | engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs; |
1122 | } else if (IS_GEN(i915, 5)(0 + (&(i915)->__info)->gen == (5))) { |
1123 | engine->emit_flush = gen4_emit_flush_rcs; |
1124 | } else { |
1125 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) < 4) |
1126 | engine->emit_flush = gen2_emit_flush; |
1127 | else |
1128 | engine->emit_flush = gen4_emit_flush_rcs; |
1129 | engine->irq_enable_mask = I915_USER_INTERRUPT(1 << 1); |
1130 | } |
1131 | |
1132 | if (IS_HASWELL(i915)IS_PLATFORM(i915, INTEL_HASWELL)) |
1133 | engine->emit_bb_start = hsw_emit_bb_start; |
1134 | } |
1135 | |
1136 | static void setup_vcs(struct intel_engine_cs *engine) |
1137 | { |
1138 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
1139 | |
1140 | if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 6) { |
1141 | /* gen6 bsd needs a special wa for tail updates */ |
1142 | if (IS_GEN(i915, 6)(0 + (&(i915)->__info)->gen == (6))) |
1143 | engine->set_default_submission = gen6_bsd_set_default_submission; |
1144 | engine->emit_flush = gen6_emit_flush_vcs; |
1145 | engine->irq_enable_mask = GT_BSD_USER_INTERRUPT(1 << 12); |
1146 | |
1147 | if (IS_GEN(i915, 6)(0 + (&(i915)->__info)->gen == (6))) |
1148 | engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; |
1149 | else |
1150 | engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; |
1151 | } else { |
1152 | engine->emit_flush = gen4_emit_flush_vcs; |
1153 | if (IS_GEN(i915, 5)(0 + (&(i915)->__info)->gen == (5))) |
1154 | engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT(1 << 5); |
1155 | else |
1156 | engine->irq_enable_mask = I915_BSD_USER_INTERRUPT(1 << 25); |
1157 | } |
1158 | } |
1159 | |
1160 | static void setup_bcs(struct intel_engine_cs *engine) |
1161 | { |
1162 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
1163 | |
1164 | engine->emit_flush = gen6_emit_flush_xcs; |
1165 | engine->irq_enable_mask = GT_BLT_USER_INTERRUPT(1 << 22); |
1166 | |
1167 | if (IS_GEN(i915, 6)(0 + (&(i915)->__info)->gen == (6))) |
1168 | engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; |
1169 | else |
1170 | engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; |
1171 | } |
1172 | |
1173 | static void setup_vecs(struct intel_engine_cs *engine) |
1174 | { |
1175 | struct drm_i915_privateinteldrm_softc *i915 = engine->i915; |
Value stored to 'i915' during its initialization is never read | |
1176 | |
1177 | GEM_BUG_ON(INTEL_GEN(i915) < 7)((void)0); |
1178 | |
1179 | engine->emit_flush = gen6_emit_flush_xcs; |
1180 | engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT(1 << 10); |
1181 | engine->irq_enable = hsw_irq_enable_vecs; |
1182 | engine->irq_disable = hsw_irq_disable_vecs; |
1183 | |
1184 | engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; |
1185 | } |
1186 | |
1187 | static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, |
1188 | struct i915_vma * const vma) |
1189 | { |
1190 | return gen7_setup_clear_gpr_bb(engine, vma); |
1191 | } |
1192 | |
1193 | static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine) |
1194 | { |
1195 | struct drm_i915_gem_object *obj; |
1196 | struct i915_vma *vma; |
1197 | int size; |
1198 | int err; |
1199 | |
1200 | size = gen7_ctx_switch_bb_setup(engine, NULL((void *)0) /* probe size */); |
1201 | if (size <= 0) |
1202 | return size; |
1203 | |
1204 | size = roundup2(size, PAGE_SIZE)(((size) + (((1 << 12)) - 1)) & (~((__typeof(size)) ((1 << 12)) - 1))); |
1205 | obj = i915_gem_object_create_internal(engine->i915, size); |
1206 | if (IS_ERR(obj)) |
1207 | return PTR_ERR(obj); |
1208 | |
1209 | vma = i915_vma_instance(obj, engine->gt->vm, NULL((void *)0)); |
1210 | if (IS_ERR(vma)) { |
1211 | err = PTR_ERR(vma); |
1212 | goto err_obj; |
1213 | } |
1214 | |
1215 | vma->private = intel_context_create(engine); /* dummy residuals */ |
1216 | if (IS_ERR(vma->private)) { |
1217 | err = PTR_ERR(vma->private); |
1218 | goto err_obj; |
1219 | } |
1220 | |
1221 | err = i915_vma_pin(vma, 0, 0, PIN_USER(1ULL << (11)) | PIN_HIGH(1ULL << (5))); |
1222 | if (err) |
1223 | goto err_private; |
1224 | |
1225 | err = i915_vma_sync(vma); |
1226 | if (err) |
1227 | goto err_unpin; |
1228 | |
1229 | err = gen7_ctx_switch_bb_setup(engine, vma); |
1230 | if (err) |
1231 | goto err_unpin; |
1232 | |
1233 | engine->wa_ctx.vma = vma; |
1234 | return 0; |
1235 | |
1236 | err_unpin: |
1237 | i915_vma_unpin(vma); |
1238 | err_private: |
1239 | intel_context_put(vma->private); |
1240 | err_obj: |
1241 | i915_gem_object_put(obj); |
1242 | return err; |
1243 | } |
1244 | |
1245 | int intel_ring_submission_setup(struct intel_engine_cs *engine) |
1246 | { |
1247 | struct intel_timeline *timeline; |
1248 | struct intel_ring *ring; |
1249 | int err; |
1250 | |
1251 | setup_common(engine); |
1252 | |
1253 | switch (engine->class) { |
1254 | case RENDER_CLASS0: |
1255 | setup_rcs(engine); |
1256 | break; |
1257 | case VIDEO_DECODE_CLASS1: |
1258 | setup_vcs(engine); |
1259 | break; |
1260 | case COPY_ENGINE_CLASS3: |
1261 | setup_bcs(engine); |
1262 | break; |
1263 | case VIDEO_ENHANCEMENT_CLASS2: |
1264 | setup_vecs(engine); |
1265 | break; |
1266 | default: |
1267 | MISSING_CASE(engine->class)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n" , "engine->class", (long)(engine->class)); __builtin_expect (!!(__ret), 0); }); |
1268 | return -ENODEV19; |
1269 | } |
1270 | |
1271 | timeline = intel_timeline_create_from_engine(engine, |
1272 | I915_GEM_HWS_SEQNO_ADDR(0x40 * sizeof(u32))); |
1273 | if (IS_ERR(timeline)) { |
1274 | err = PTR_ERR(timeline); |
1275 | goto err; |
1276 | } |
1277 | GEM_BUG_ON(timeline->has_initial_breadcrumb)((void)0); |
1278 | |
1279 | err = intel_timeline_pin(timeline, NULL((void *)0)); |
1280 | if (err) |
1281 | goto err_timeline; |
1282 | |
1283 | ring = intel_engine_create_ring(engine, SZ_16K(1024 * 16)); |
1284 | if (IS_ERR(ring)) { |
1285 | err = PTR_ERR(ring); |
1286 | goto err_timeline_unpin; |
1287 | } |
1288 | |
1289 | err = intel_ring_pin(ring, NULL((void *)0)); |
1290 | if (err) |
1291 | goto err_ring; |
1292 | |
1293 | GEM_BUG_ON(engine->legacy.ring)((void)0); |
1294 | engine->legacy.ring = ring; |
1295 | engine->legacy.timeline = timeline; |
1296 | |
1297 | GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma)((void)0); |
1298 | |
1299 | if (IS_GEN(engine->i915, 7)(0 + (&(engine->i915)->__info)->gen == (7)) && engine->class == RENDER_CLASS0) { |
1300 | err = gen7_ctx_switch_bb_init(engine); |
1301 | if (err) |
1302 | goto err_ring_unpin; |
1303 | } |
1304 | |
1305 | /* Finally, take ownership and responsibility for cleanup! */ |
1306 | engine->release = ring_release; |
1307 | |
1308 | return 0; |
1309 | |
1310 | err_ring_unpin: |
1311 | intel_ring_unpin(ring); |
1312 | err_ring: |
1313 | intel_ring_put(ring); |
1314 | err_timeline_unpin: |
1315 | intel_timeline_unpin(timeline); |
1316 | err_timeline: |
1317 | intel_timeline_put(timeline); |
1318 | err: |
1319 | intel_engine_cleanup_common(engine); |
1320 | return err; |
1321 | } |
1322 | |
1323 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0 |
1324 | #include "selftest_ring_submission.c" |
1325 | #endif |