/usr/src/sys/dev/pci/drm/i915/gt/uc/intel_guc

Bug Summary

File:	dev/pci/drm/i915/gt/uc/intel_guc_submission.c
Warning:	line 3808, column 33 Dereference of null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name intel_guc_submission.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/gt/uc/intel_guc_submission.c

/usr/src/sys/dev/pci/drm/i915/gt/uc/intel_guc_submission.c

→

1// SPDX-License-Identifier: MIT
2/*
* Copyright © 2014 Intel Corporation
*/

6#include <linux/circ_buf.h>

8#include "gem/i915_gem_context.h"
9#include "gt/gen8_engine_cs.h"
10#include "gt/intel_breadcrumbs.h"
11#include "gt/intel_context.h"
12#include "gt/intel_engine_heartbeat.h"
13#include "gt/intel_engine_pm.h"
14#include "gt/intel_engine_regs.h"
15#include "gt/intel_gpu_commands.h"
16#include "gt/intel_gt.h"
17#include "gt/intel_gt_clock_utils.h"
18#include "gt/intel_gt_irq.h"
19#include "gt/intel_gt_pm.h"
20#include "gt/intel_gt_regs.h"
21#include "gt/intel_gt_requests.h"
22#include "gt/intel_lrc.h"
23#include "gt/intel_lrc_reg.h"
24#include "gt/intel_mocs.h"
25#include "gt/intel_ring.h"

27#include "intel_guc_ads.h"
28#include "intel_guc_capture.h"
29#include "intel_guc_submission.h"

31#include "i915_drv.h"
32#include "i915_trace.h"

34/**
* DOC: GuC-based command submission
*
* The Scratch registers:
* There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
* a value to the action register (SOFT_SCRATCH_0) along with any data. It then
* triggers an interrupt on the GuC via another register write (0xC4C8).
* Firmware writes a success/fail code back to the action register after
* processes the request. The kernel driver polls waiting for this update and
* then proceeds.
*
* Command Transport buffers (CTBs):
* Covered in detail in other sections but CTBs (Host to GuC - H2G, GuC to Host
* - G2H) are a message interface between the i915 and GuC.
*
* Context registration:
* Before a context can be submitted it must be registered with the GuC via a
* H2G. A unique guc_id is associated with each context. The context is either
* registered at request creation time (normal operation) or at submission time
* (abnormal operation, e.g. after a reset).
*
* Context submission:
* The i915 updates the LRC tail value in memory. The i915 must enable the
* scheduling of the context within the GuC for the GuC to actually consider it.
* Therefore, the first time a disabled context is submitted we use a schedule
* enable H2G, while follow up submissions are done via the context submit H2G,
* which informs the GuC that a previously enabled context has new work
* available.
*
* Context unpin:
* To unpin a context a H2G is used to disable scheduling. When the
* corresponding G2H returns indicating the scheduling disable operation has
* completed it is safe to unpin the context. While a disable is in flight it
* isn't safe to resubmit the context so a fence is used to stall all future
* requests of that context until the G2H is returned.
*
* Context deregistration:
* Before a context can be destroyed or if we steal its guc_id we must
* deregister the context with the GuC via H2G. If stealing the guc_id it isn't
* safe to submit anything to this guc_id until the deregister completes so a
* fence is used to stall all requests associated with this guc_id until the
* corresponding G2H returns indicating the guc_id has been deregistered.
*
* submission_state.guc_ids:
* Unique number associated with private GuC context data passed in during
* context registration / submission / deregistration. 64k available. Simple ida
* is used for allocation.
*
* Stealing guc_ids:
* If no guc_ids are available they can be stolen from another context at
* request creation time if that context is unpinned. If a guc_id can't be found
* we punt this problem to the user as we believe this is near impossible to hit
* during normal use cases.
*
* Locking:
* In the GuC submission code we have 3 basic spin locks which protect
* everything. Details about each below.
*
* sched_engine->lock
* This is the submission lock for all contexts that share an i915 schedule
* engine (sched_engine), thus only one of the contexts which share a
* sched_engine can be submitting at a time. Currently only one sched_engine is
* used for all of GuC submission but that could change in the future.
*
* guc->submission_state.lock
* Global lock for GuC submission state. Protects guc_ids and destroyed contexts
* list.
*
* ce->guc_state.lock
* Protects everything under ce->guc_state. Ensures that a context is in the
* correct state before issuing a H2G. e.g. We don't issue a schedule disable
* on a disabled context (bad idea), we don't issue a schedule enable when a
* schedule disable is in flight, etc... Also protects list of inflight requests
* on the context and the priority management state. Lock is individual to each
* context.
*
* Lock ordering rules:
* sched_engine->lock -> ce->guc_state.lock
* guc->submission_state.lock -> ce->guc_state.lock
*
* Reset races:
* When a full GT reset is triggered it is assumed that some G2H responses to
* H2Gs can be lost as the GuC is also reset. Losing these G2H can prove to be
* fatal as we do certain operations upon receiving a G2H (e.g. destroy
* contexts, release guc_ids, etc...). When this occurs we can scrub the
* context state and cleanup appropriately, however this is quite racey.
* To avoid races, the reset code must disable submission before scrubbing for
* the missing G2H, while the submission code must check for submission being
* disabled and skip sending H2Gs and updating context states when it is. Both
* sides must also make sure to hold the relevant locks.
*/

126/* GuC Virtual Engine */
127struct guc_virtual_engine {
struct intel_engine_cs base;
struct intel_context context;
130};

132static struct intel_context *
133guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
   unsigned long flags);

136static struct intel_context *
137guc_create_parallel(struct intel_engine_cs **engines,
    unsigned int num_siblings,
    unsigned int width);

141#define GUC_REQUEST_SIZE64 64 /* bytes */

143/*
* We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous
* per the GuC submission interface. A different allocation algorithm is used
* (bitmap vs. ida) between multi-lrc and single-lrc hence the reason to
* partition the guc_id space. We believe the number of multi-lrc contexts in
* use should be low and 1/16 should be sufficient. Minimum of 32 guc_ids for
* multi-lrc.
*/
151#define NUMBER_MULTI_LRC_GUC_ID(guc)((guc)->submission_state.num_guc_ids / 16)	\
((guc)->submission_state.num_guc_ids / 16)

154/*
* Below is a set of functions which control the GuC scheduling state which
* require a lock.
*/
158#define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER(1UL << (0))	BIT(0)(1UL << (0))
159#define SCHED_STATE_DESTROYED(1UL << (1))				BIT(1)(1UL << (1))
160#define SCHED_STATE_PENDING_DISABLE(1UL << (2))			BIT(2)(1UL << (2))
161#define SCHED_STATE_BANNED(1UL << (3))				BIT(3)(1UL << (3))
162#define SCHED_STATE_ENABLED(1UL << (4))				BIT(4)(1UL << (4))
163#define SCHED_STATE_PENDING_ENABLE(1UL << (5))			BIT(5)(1UL << (5))
164#define SCHED_STATE_REGISTERED(1UL << (6))				BIT(6)(1UL << (6))
165#define SCHED_STATE_POLICY_REQUIRED(1UL << (7))			BIT(7)(1UL << (7))
166#define SCHED_STATE_BLOCKED_SHIFT8			8
167#define SCHED_STATE_BLOCKED(1UL << (8))		BIT(SCHED_STATE_BLOCKED_SHIFT)(1UL << (8))
168#define SCHED_STATE_BLOCKED_MASK(0xfff << 8)	(0xfff << SCHED_STATE_BLOCKED_SHIFT8)

170static inline void init_sched_state(struct intel_context *ce)
171{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK(0xfff << 8);
174}

176__maybe_unused__attribute__((__unused__))
177static bool_Bool sched_state_is_init(struct intel_context *ce)
178{
/* Kernel contexts can have SCHED_STATE_REGISTERED after suspend. */
return !(ce->guc_state.sched_state &
 ~(SCHED_STATE_BLOCKED_MASK(0xfff << 8) | SCHED_STATE_REGISTERED(1UL << (6))));
182}

184static inline bool_Bool
185context_wait_for_deregister_to_register(struct intel_context *ce)
186{
return ce->guc_state.sched_state &
SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER(1UL << (0));
189}

191static inline void
192set_context_wait_for_deregister_to_register(struct intel_context *ce)
193{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |=
SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER(1UL << (0));
197}

199static inline void
200clr_context_wait_for_deregister_to_register(struct intel_context *ce)
201{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &=
~SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER(1UL << (0));
205}

207static inline bool_Bool
208context_destroyed(struct intel_context *ce)
209{
return ce->guc_state.sched_state & SCHED_STATE_DESTROYED(1UL << (1));
211}

213static inline void
214set_context_destroyed(struct intel_context *ce)
215{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_DESTROYED(1UL << (1));
218}

220static inline bool_Bool context_pending_disable(struct intel_context *ce)
221{
return ce->guc_state.sched_state & SCHED_STATE_PENDING_DISABLE(1UL << (2));
223}

225static inline void set_context_pending_disable(struct intel_context *ce)
226{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_PENDING_DISABLE(1UL << (2));
229}

231static inline void clr_context_pending_disable(struct intel_context *ce)
232{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_DISABLE(1UL << (2));
235}

237static inline bool_Bool context_banned(struct intel_context *ce)
238{
return ce->guc_state.sched_state & SCHED_STATE_BANNED(1UL << (3));
240}

242static inline void set_context_banned(struct intel_context *ce)
243{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_BANNED(1UL << (3));
246}

248static inline void clr_context_banned(struct intel_context *ce)
249{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= ~SCHED_STATE_BANNED(1UL << (3));
252}

254static inline bool_Bool context_enabled(struct intel_context *ce)
255{
return ce->guc_state.sched_state & SCHED_STATE_ENABLED(1UL << (4));
257}

259static inline void set_context_enabled(struct intel_context *ce)
260{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_ENABLED(1UL << (4));
263}

265static inline void clr_context_enabled(struct intel_context *ce)
266{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= ~SCHED_STATE_ENABLED(1UL << (4));
269}

271static inline bool_Bool context_pending_enable(struct intel_context *ce)
272{
return ce->guc_state.sched_state & SCHED_STATE_PENDING_ENABLE(1UL << (5));
274}

276static inline void set_context_pending_enable(struct intel_context *ce)
277{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_PENDING_ENABLE(1UL << (5));
280}

282static inline void clr_context_pending_enable(struct intel_context *ce)
283{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_ENABLE(1UL << (5));
286}

288static inline bool_Bool context_registered(struct intel_context *ce)
289{
return ce->guc_state.sched_state & SCHED_STATE_REGISTERED(1UL << (6));
291}

293static inline void set_context_registered(struct intel_context *ce)
294{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_REGISTERED(1UL << (6));
297}

299static inline void clr_context_registered(struct intel_context *ce)
300{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED(1UL << (6));
303}

305static inline bool_Bool context_policy_required(struct intel_context *ce)
306{
return ce->guc_state.sched_state & SCHED_STATE_POLICY_REQUIRED(1UL << (7));
308}

310static inline void set_context_policy_required(struct intel_context *ce)
311{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state |= SCHED_STATE_POLICY_REQUIRED(1UL << (7));
314}

316static inline void clr_context_policy_required(struct intel_context *ce)
317{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
ce->guc_state.sched_state &= ~SCHED_STATE_POLICY_REQUIRED(1UL << (7));
320}

322static inline u32 context_blocked(struct intel_context *ce)
323{
return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK(0xfff << 8)) >>
SCHED_STATE_BLOCKED_SHIFT8;
326}

328static inline void incr_context_blocked(struct intel_context *ce)
329{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

ce->guc_state.sched_state += SCHED_STATE_BLOCKED(1UL << (8));

GEM_BUG_ON(!context_blocked(ce))((void)0);	/* Overflow check */
335}

337static inline void decr_context_blocked(struct intel_context *ce)
338{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

GEM_BUG_ON(!context_blocked(ce))((void)0);	/* Underflow check */

ce->guc_state.sched_state -= SCHED_STATE_BLOCKED(1UL << (8));
344}

346static inline bool_Bool context_has_committed_requests(struct intel_context *ce)
347{
return !!ce->guc_state.number_committed_requests;
349}

351static inline void incr_context_committed_requests(struct intel_context *ce)
352{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
++ce->guc_state.number_committed_requests;
GEM_BUG_ON(ce->guc_state.number_committed_requests < 0)((void)0);
356}

358static inline void decr_context_committed_requests(struct intel_context *ce)
359{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
--ce->guc_state.number_committed_requests;
GEM_BUG_ON(ce->guc_state.number_committed_requests < 0)((void)0);
363}

365static struct intel_context *
366request_to_scheduling_context(struct i915_request *rq)
367{
return intel_context_to_parent(rq->context);
369}

371static inline bool_Bool context_guc_id_invalid(struct intel_context *ce)
372{
return ce->guc_id.id == GUC_INVALID_CONTEXT_ID65535;
374}

376static inline void set_context_guc_id_invalid(struct intel_context *ce)
377{
ce->guc_id.id = GUC_INVALID_CONTEXT_ID65535;
379}

381static inline struct intel_guc *ce_to_guc(struct intel_context *ce)
382{
return &ce->engine->gt->uc.guc;
384}

386static inline struct i915_priolist *to_priolist(struct rb_node *rb)
387{
return rb_entry(rb, struct i915_priolist, node)({ const __typeof( ((struct i915_priolist *)0)->node ) *__mptr
 = (rb); (struct i915_priolist *)( (char *)__mptr - __builtin_offsetof
(struct i915_priolist, node) );});
389}

391/*
* When using multi-lrc submission a scratch memory area is reserved in the
* parent's context state for the process descriptor, work queue, and handshake
* between the parent + children contexts to insert safe preemption points
* between each of the BBs. Currently the scratch area is sized to a page.
*
* The layout of this scratch area is below:
* 0						guc_process_desc
* + sizeof(struct guc_process_desc)		child go
* + CACHELINE_BYTES				child join[0]
* ...
* + CACHELINE_BYTES				child join[n - 1]
* ...						unused
* PARENT_SCRATCH_SIZE / 2			work queue start
* ...						work queue
* PARENT_SCRATCH_SIZE - 1			work queue end
*/
408#define WQ_SIZE((1 << 12) / 2)			(PARENT_SCRATCH_SIZE(1 << 12) / 2)
409#define WQ_OFFSET((1 << 12) - ((1 << 12) / 2))		(PARENT_SCRATCH_SIZE(1 << 12) - WQ_SIZE((1 << 12) / 2))

411struct sync_semaphore {
u32 semaphore;
u8 unused[CACHELINE_BYTES64 - sizeof(u32)];
414};

416struct parent_scratch {
union guc_descs {
struct guc_sched_wq_desc wq_desc;
struct guc_process_desc_v69 pdesc;
} descs;

struct sync_semaphore go;
struct sync_semaphore join[MAX_ENGINE_INSTANCE8 + 1];

u8 unused[WQ_OFFSET((1 << 12) - ((1 << 12) / 2)) - sizeof(union guc_descs) -
sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE8 + 2)];

u32 wq[WQ_SIZE((1 << 12) / 2) / sizeof(u32)];
429};

431static u32 __get_parent_scratch_offset(struct intel_context *ce)
432{
GEM_BUG_ON(!ce->parallel.guc.parent_page)((void)0);

return ce->parallel.guc.parent_page * PAGE_SIZE(1 << 12);
436}

438static u32 __get_wq_offset(struct intel_context *ce)
439{
BUILD_BUG_ON(offsetof(struct parent_scratch, wq) != WQ_OFFSET)extern char _ctassert[(!(__builtin_offsetof(struct parent_scratch
, wq) != ((1 << 12) - ((1 << 12) / 2)))) ? 1 : -1
 ] __attribute__((__unused__));

return __get_parent_scratch_offset(ce) + WQ_OFFSET((1 << 12) - ((1 << 12) / 2));
443}

445static struct parent_scratch *
446__get_parent_scratch(struct intel_context *ce)
447{
BUILD_BUG_ON(sizeof(struct parent_scratch) != PARENT_SCRATCH_SIZE)extern char _ctassert[(!(sizeof(struct parent_scratch) != (1 <<
 12))) ? 1 : -1 ] __attribute__((__unused__));
BUILD_BUG_ON(sizeof(struct sync_semaphore) != CACHELINE_BYTES)extern char _ctassert[(!(sizeof(struct sync_semaphore) != 64)
) ? 1 : -1 ] __attribute__((__unused__));

/*
* Need to subtract LRC_STATE_OFFSET here as the
* parallel.guc.parent_page is the offset into ce->state while
* ce->lrc_reg_reg is ce->state + LRC_STATE_OFFSET.
*/
return (struct parent_scratch *)
(ce->lrc_reg_state +
 ((__get_parent_scratch_offset(ce) -
   LRC_STATE_OFFSET(((0) + (1)) * (1 << 12))) / sizeof(u32)));
460}

462static struct guc_process_desc_v69 *
463__get_process_desc_v69(struct intel_context *ce)
464{
struct parent_scratch *ps = __get_parent_scratch(ce);

return &ps->descs.pdesc;
468}

470static struct guc_sched_wq_desc *
471__get_wq_desc_v70(struct intel_context *ce)
472{
struct parent_scratch *ps = __get_parent_scratch(ce);

return &ps->descs.wq_desc;
476}

478static u32 *get_wq_pointer(struct intel_context *ce, u32 wqi_size)
479{
/*
* Check for space in work queue. Caching a value of head pointer in
* intel_context structure in order reduce the number accesses to shared
* GPU memory which may be across a PCIe bus.
*/
485#define AVAILABLE_SPACE	\
CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE)(((ce->parallel.guc.wqi_head) - ((ce->parallel.guc.wqi_tail
)+1)) & ((((1 << 12) / 2))-1))
if (wqi_size > AVAILABLE_SPACE) {
ce->parallel.guc.wqi_head = READ_ONCE(*ce->parallel.guc.wq_head)({ typeof(*ce->parallel.guc.wq_head) __tmp = *(volatile typeof
(*ce->parallel.guc.wq_head) *)&(*ce->parallel.guc.wq_head
); membar_datadep_consumer(); __tmp; });

if (wqi_size > AVAILABLE_SPACE)
	return NULL((void *)0);
}
493#undef AVAILABLE_SPACE

return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)];
496}

498static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id)
499{
struct intel_context *ce = xa_load(&guc->context_lookup, id);

GEM_BUG_ON(id >= GUC_MAX_CONTEXT_ID)((void)0);

return ce;
505}

507static struct guc_lrc_desc_v69 *__get_lrc_desc_v69(struct intel_guc *guc, u32 index)
508{
struct guc_lrc_desc_v69 *base = guc->lrc_desc_pool_vaddr_v69;

if (!base)
return NULL((void *)0);

GEM_BUG_ON(index >= GUC_MAX_CONTEXT_ID)((void)0);

return &base[index];
517}

519static int guc_lrc_desc_pool_create_v69(struct intel_guc *guc)
520{
u32 size;
int ret;

size = PAGE_ALIGN(sizeof(struct guc_lrc_desc_v69) *(((sizeof(struct guc_lrc_desc_v69) * 65535) + ((1 << 12
) - 1)) & ~((1 << 12) - 1))
	  GUC_MAX_CONTEXT_ID)(((sizeof(struct guc_lrc_desc_v69) * 65535) + ((1 << 12
) - 1)) & ~((1 << 12) - 1));
ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool_v69,
			     (void **)&guc->lrc_desc_pool_vaddr_v69);
if (ret)
return ret;

return 0;
532}

534static void guc_lrc_desc_pool_destroy_v69(struct intel_guc *guc)
535{
if (!guc->lrc_desc_pool_vaddr_v69)
return;

guc->lrc_desc_pool_vaddr_v69 = NULL((void *)0);
i915_vma_unpin_and_release(&guc->lrc_desc_pool_v69, I915_VMA_RELEASE_MAP(1UL << (0)));
541}

543static inline bool_Bool guc_submission_initialized(struct intel_guc *guc)
544{
return guc->submission_initialized;
546}

548static inline void _reset_lrc_desc_v69(struct intel_guc *guc, u32 id)
549{
struct guc_lrc_desc_v69 *desc = __get_lrc_desc_v69(guc, id);

if (desc)
memset(desc, 0, sizeof(*desc))__builtin_memset((desc), (0), (sizeof(*desc)));
554}

556static inline bool_Bool ctx_id_mapped(struct intel_guc *guc, u32 id)
557{
return __get_context(guc, id);
559}

561static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id,
		      struct intel_context *ce)
563{
unsigned long flags;

/*
* xarray API doesn't have xa_save_irqsave wrapper, so calling the
* lower level functions directly.
*/
xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
__xa_store(&guc->context_lookup, id, ce, GFP_ATOMIC0x0002);
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);
573}

575static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id)
576{
unsigned long flags;

if (unlikely(!guc_submission_initialized(guc))__builtin_expect(!!(!guc_submission_initialized(guc)), 0))
return;

_reset_lrc_desc_v69(guc, id);

/*
* xarray API doesn't have xa_erase_irqsave wrapper, so calling
* the lower level functions directly.
*/
xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
__xa_erase(&guc->context_lookup, id);
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);
591}

593static void decr_outstanding_submission_g2h(struct intel_guc *guc)
594{
if (atomic_dec_and_test(&guc->outstanding_submission_g2h)(__sync_sub_and_fetch((&guc->outstanding_submission_g2h
), 1) == 0))
wake_up_all(&guc->ct.wq)wake_up(&guc->ct.wq);
597}

599static int guc_submission_send_busy_loop(struct intel_guc *guc,
			 const u32 *action,
			 u32 len,
			 u32 g2h_len_dw,
			 bool_Bool loop)
604{
/*
* We always loop when a send requires a reply (i.e. g2h_len_dw > 0),
* so we don't handle the case where we don't get a reply because we
* aborted the send due to the channel being busy.
*/
GEM_BUG_ON(g2h_len_dw && !loop)((void)0);

if (g2h_len_dw)
atomic_inc(&guc->outstanding_submission_g2h)__sync_fetch_and_add(&guc->outstanding_submission_g2h,
 1);

return intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
616}

618int intel_guc_wait_for_pending_msg(struct intel_guc *guc,
		   atomic_t *wait_var,
		   bool_Bool interruptible,
		   long timeout)
622{
const int state = interruptible ?
TASK_INTERRUPTIBLE0x100 : TASK_UNINTERRUPTIBLE0;
DEFINE_WAIT(wait)struct wait_queue_entry wait = { .private = ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc, .func =
 autoremove_wake_function, .entry = { &((wait).entry), &
((wait).entry) }, };

might_sleep()assertwaitok();
GEM_BUG_ON(timeout < 0)((void)0);

if (!atomic_read(wait_var)({ typeof(*(wait_var)) __tmp = *(volatile typeof(*(wait_var))
 *)&(*(wait_var)); membar_datadep_consumer(); __tmp; }))
return 0;

if (!timeout)
return -ETIME60;

for (;;) {
prepare_to_wait(&guc->ct.wq, &wait, state);

if (!atomic_read(wait_var)({ typeof(*(wait_var)) __tmp = *(volatile typeof(*(wait_var))
 *)&(*(wait_var)); membar_datadep_consumer(); __tmp; }))
	break;

if (signal_pending_state(state, current)((state) & 0x100 ? (((({struct cpu_info *__ci; asm volatile
("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct
 cpu_info, ci_self))); __ci;})->ci_curproc)->p_siglist |
 (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc)->p_p->ps_siglist) & ~(({struct
 cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci
) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;
})->ci_curproc)->p_sigmask) : 0)) {
	timeout = -EINTR4;
	break;
}

if (!timeout) {
	timeout = -ETIME60;
	break;
}

timeout = io_schedule_timeout(timeout)schedule_timeout(timeout);
}
finish_wait(&guc->ct.wq, &wait);

return (timeout < 0) ? timeout : 0;
657}

659int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout)
660{
if (!intel_uc_uses_guc_submission(&guc_to_gt(guc)->uc))
return 0;

return intel_guc_wait_for_pending_msg(guc,
			      &guc->outstanding_submission_g2h,
			      true1, timeout);
667}

669static int guc_context_policy_init_v70(struct intel_context *ce, bool_Bool loop);
670static int try_context_registration(struct intel_context *ce, bool_Bool loop);

672static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)
673{
int err = 0;
struct intel_context *ce = request_to_scheduling_context(rq);
u32 action[3];
int len = 0;
u32 g2h_len_dw = 0;
bool_Bool enabled;

lockdep_assert_held(&rq->engine->sched_engine->lock)do { (void)(&rq->engine->sched_engine->lock); } while
(0);

/*
* Corner case where requests were sitting in the priority list or a
* request resubmitted after the context was banned.
*/
if (unlikely(!intel_context_is_schedulable(ce))__builtin_expect(!!(!intel_context_is_schedulable(ce)), 0)) {
i915_request_put(i915_request_mark_eio(rq));
intel_engine_signal_breadcrumbs(ce->engine);
return 0;
}

GEM_BUG_ON(!atomic_read(&ce->guc_id.ref))((void)0);
GEM_BUG_ON(context_guc_id_invalid(ce))((void)0);

if (context_policy_required(ce)) {
err = guc_context_policy_init_v70(ce, false0);
if (err)
	return err;
}

spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);

/*
* The request / context will be run on the hardware when scheduling
* gets enabled in the unblock. For multi-lrc we still submit the
* context to move the LRC tails.
*/
if (unlikely(context_blocked(ce) && !intel_context_is_parent(ce))__builtin_expect(!!(context_blocked(ce) && !intel_context_is_parent
(ce)), 0))
goto out;

enabled = context_enabled(ce) || context_blocked(ce);

if (!enabled) {
action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
action[len++] = ce->guc_id.id;
action[len++] = GUC_CONTEXT_ENABLE1;
set_context_pending_enable(ce);
intel_context_get(ce);
g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET2;
} else {
action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT;
action[len++] = ce->guc_id.id;
}

err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
if (!enabled && !err) {
trace_intel_context_sched_enable(ce);
atomic_inc(&guc->outstanding_submission_g2h)__sync_fetch_and_add(&guc->outstanding_submission_g2h,
 1);
set_context_enabled(ce);

/*
 * Without multi-lrc KMD does the submission step (moving the
 * lrc tail) so enabling scheduling is sufficient to submit the
 * context. This isn't the case in multi-lrc submission as the
 * GuC needs to move the tails, hence the need for another H2G
 * to submit a multi-lrc context after enabling scheduling.
 */
if (intel_context_is_parent(ce)) {
	action[0] = INTEL_GUC_ACTION_SCHED_CONTEXT;
	err = intel_guc_send_nb(guc, action, len - 1, 0);
}
} else if (!enabled) {
clr_context_pending_enable(ce);
intel_context_put(ce);
}
if (likely(!err)__builtin_expect(!!(!err), 1))
trace_i915_request_guc_submit(rq);

750out:
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);
return err;
753}

755static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)
756{
int ret = __guc_add_request(guc, rq);

if (unlikely(ret == -EBUSY)__builtin_expect(!!(ret == -16), 0)) {
guc->stalled_request = rq;
guc->submission_stall_reason = STALL_ADD_REQUEST;
}

return ret;
765}

767static inline void guc_set_lrc_tail(struct i915_request *rq)
768{
rq->context->lrc_reg_state[CTX_RING_TAIL(0x06 + 1)] =
intel_ring_set_tail(rq->ring, rq->tail);
771}

773static inline int rq_prio(const struct i915_request *rq)
774{
return rq->sched.attr.priority;
776}

778static bool_Bool is_multi_lrc_rq(struct i915_request *rq)
779{
return intel_context_is_parallel(rq->context);
781}

783static bool_Bool can_merge_rq(struct i915_request *rq,
	 struct i915_request *last)
785{
return request_to_scheduling_context(rq) ==
request_to_scheduling_context(last);
788}

790static u32 wq_space_until_wrap(struct intel_context *ce)
791{
return (WQ_SIZE((1 << 12) / 2) - ce->parallel.guc.wqi_tail);
793}

795static void write_wqi(struct intel_context *ce, u32 wqi_size)
796{
BUILD_BUG_ON(!is_power_of_2(WQ_SIZE))extern char _ctassert[(!(!(((((1 << 12) / 2)) != 0) &&
 (((((1 << 12) / 2)) - 1) & (((1 << 12) / 2))
) == 0))) ? 1 : -1 ] __attribute__((__unused__));

/*
* Ensure WQI are visible before updating tail
*/
intel_guc_write_barrier(ce_to_guc(ce));

ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) &
(WQ_SIZE((1 << 12) / 2) - 1);
WRITE_ONCE(*ce->parallel.guc.wq_tail, ce->parallel.guc.wqi_tail)({ typeof(*ce->parallel.guc.wq_tail) __tmp = (ce->parallel
.guc.wqi_tail); *(volatile typeof(*ce->parallel.guc.wq_tail
) *)&(*ce->parallel.guc.wq_tail) = __tmp; __tmp; });
807}

809static int guc_wq_noop_append(struct intel_context *ce)
810{
u32 *wqi = get_wq_pointer(ce, wq_space_until_wrap(ce));
u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1;

if (!wqi)
return -EBUSY16;

GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw))((void)0);

*wqi = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP)(((typeof((((~0UL) >> (64 - (7) - 1)) & ((~0UL) <<
 (0)))))(0x4) << (__builtin_ffsll((((~0UL) >> (64
 - (7) - 1)) & ((~0UL) << (0)))) - 1)) & ((((~0UL
) >> (64 - (7) - 1)) & ((~0UL) << (0))))) |
FIELD_PREP(WQ_LEN_MASK, len_dw)(((typeof((((~0UL) >> (64 - (26) - 1)) & ((~0UL) <<
 (16)))))(len_dw) << (__builtin_ffsll((((~0UL) >>
 (64 - (26) - 1)) & ((~0UL) << (16)))) - 1)) & (
(((~0UL) >> (64 - (26) - 1)) & ((~0UL) << (16
)))));
ce->parallel.guc.wqi_tail = 0;

return 0;
824}

826static int __guc_wq_item_append(struct i915_request *rq)
827{
struct intel_context *ce = request_to_scheduling_context(rq);
struct intel_context *child;
unsigned int wqi_size = (ce->parallel.number_children + 4) *
sizeof(u32);
u32 *wqi;
u32 len_dw = (wqi_size / sizeof(u32)) - 1;
int ret;

/* Ensure context is in correct state updating work queue */
GEM_BUG_ON(!atomic_read(&ce->guc_id.ref))((void)0);
GEM_BUG_ON(context_guc_id_invalid(ce))((void)0);
GEM_BUG_ON(context_wait_for_deregister_to_register(ce))((void)0);
GEM_BUG_ON(!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id))((void)0);

/* Insert NOOP if this work queue item will wrap the tail pointer. */
if (wqi_size > wq_space_until_wrap(ce)) {
ret = guc_wq_noop_append(ce);
if (ret)
	return ret;
}

wqi = get_wq_pointer(ce, wqi_size);
if (!wqi)
return -EBUSY16;

GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw))((void)0);

*wqi++ = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC)(((typeof((((~0UL) >> (64 - (7) - 1)) & ((~0UL) <<
 (0)))))(0x5) << (__builtin_ffsll((((~0UL) >> (64
 - (7) - 1)) & ((~0UL) << (0)))) - 1)) & ((((~0UL
) >> (64 - (7) - 1)) & ((~0UL) << (0))))) |
FIELD_PREP(WQ_LEN_MASK, len_dw)(((typeof((((~0UL) >> (64 - (26) - 1)) & ((~0UL) <<
 (16)))))(len_dw) << (__builtin_ffsll((((~0UL) >>
 (64 - (26) - 1)) & ((~0UL) << (16)))) - 1)) & (
(((~0UL) >> (64 - (26) - 1)) & ((~0UL) << (16
)))));
*wqi++ = ce->lrc.lrca;
*wqi++ = FIELD_PREP(WQ_GUC_ID_MASK, ce->guc_id.id)(((typeof((((~0UL) >> (64 - (15) - 1)) & ((~0UL) <<
 (0)))))(ce->guc_id.id) << (__builtin_ffsll((((~0UL)
 >> (64 - (15) - 1)) & ((~0UL) << (0)))) - 1)
) & ((((~0UL) >> (64 - (15) - 1)) & ((~0UL) <<
 (0))))) |
      FIELD_PREP(WQ_RING_TAIL_MASK, ce->ring->tail / sizeof(u64))(((typeof((((~0UL) >> (64 - (28) - 1)) & ((~0UL) <<
 (18)))))(ce->ring->tail / sizeof(u64)) << (__builtin_ffsll
((((~0UL) >> (64 - (28) - 1)) & ((~0UL) << (18
)))) - 1)) & ((((~0UL) >> (64 - (28) - 1)) & ((
~0UL) << (18)))));
*wqi++ = 0;	/* fence_id */
for_each_child(ce, child)for (child = ({ const __typeof( ((__typeof(*child) *)0)->parallel
.child_link ) *__mptr = ((&(ce)->parallel.child_list)->
next); (__typeof(*child) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*child), parallel.child_link) );}); &child->
parallel.child_link != (&(ce)->parallel.child_list); child
 = ({ const __typeof( ((__typeof(*child) *)0)->parallel.child_link
 ) *__mptr = (child->parallel.child_link.next); (__typeof(
*child) *)( (char *)__mptr - __builtin_offsetof(__typeof(*child
), parallel.child_link) );}))
*wqi++ = child->ring->tail / sizeof(u64);

write_wqi(ce, wqi_size);

return 0;
867}

869static int guc_wq_item_append(struct intel_guc *guc,
	      struct i915_request *rq)
871{
struct intel_context *ce = request_to_scheduling_context(rq);
int ret;

if (unlikely(!intel_context_is_schedulable(ce))__builtin_expect(!!(!intel_context_is_schedulable(ce)), 0))
return 0;

ret = __guc_wq_item_append(rq);
if (unlikely(ret == -EBUSY)__builtin_expect(!!(ret == -16), 0)) {
guc->stalled_request = rq;
guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
}

return ret;
885}

887static bool_Bool multi_lrc_submit(struct i915_request *rq)
888{
struct intel_context *ce = request_to_scheduling_context(rq);

intel_ring_set_tail(rq->ring, rq->tail);

/*
* We expect the front end (execbuf IOCTL) to set this flag on the last
* request generated from a multi-BB submission. This indicates to the
* backend (GuC interface) that we should submit this context thus
* submitting all the requests generated in parallel.
*/
return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) ||
      !intel_context_is_schedulable(ce);
901}

903static int guc_dequeue_one_context(struct intel_guc *guc)
904{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
struct i915_request *last = NULL((void *)0);
bool_Bool submit = false0;
struct rb_node *rb;
int ret;

lockdep_assert_held(&sched_engine->lock)do { (void)(&sched_engine->lock); } while(0);

if (guc->stalled_request) {
submit = true1;
last = guc->stalled_request;

switch (guc->submission_stall_reason) {
case STALL_REGISTER_CONTEXT:
	goto register_context;
case STALL_MOVE_LRC_TAIL:
	goto move_lrc_tail;
case STALL_ADD_REQUEST:
	goto add_request;
default:
	MISSING_CASE(guc->submission_stall_reason)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "guc->submission_stall_reason", (long)(guc->submission_stall_reason
)); __builtin_expect(!!(__ret), 0); });
}
}

while ((rb = rb_first_cached(&sched_engine->queue)linux_root_RB_MINMAX((struct linux_root *)(&(&sched_engine
->queue)->rb_root), -1))) {
struct i915_priolist *p = to_priolist(rb);
struct i915_request *rq, *rn;

priolist_for_each_request_consume(rq, rn, p)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&(p)->requests)->next); (__typeof(*rq
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = (rq->sched.link.next); (__typeof(*rq
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &rq->sched.link != (&(p)->requests
); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *)0)->
sched.link ) *__mptr = (rn->sched.link.next); (__typeof(*rn
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rn), sched
.link) );})) {
	if (last && !can_merge_rq(rq, last))
		goto register_context;

	list_del_init(&rq->sched.link);

	__i915_request_submit(rq);

	trace_i915_request_in(rq, 0);
	last = rq;

	if (is_multi_lrc_rq(rq)) {
		/*
		 * We need to coalesce all multi-lrc requests in
		 * a relationship into a single H2G. We are
		 * guaranteed that all of these requests will be
		 * submitted sequentially.
		 */
		if (multi_lrc_submit(rq)) {
			submit = true1;
			goto register_context;
		}
	} else {
		submit = true1;
	}
}

rb_erase_cached(&p->node, &sched_engine->queue)linux_root_RB_REMOVE((struct linux_root *)(&(&sched_engine
->queue)->rb_root), (&p->node));
i915_priolist_free(p);
}

964register_context:
if (submit) {
struct intel_context *ce = request_to_scheduling_context(last);

if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&__builtin_expect(!!(!ctx_id_mapped(guc, ce->guc_id.id) &&
 intel_context_is_schedulable(ce)), 0)
	     intel_context_is_schedulable(ce))__builtin_expect(!!(!ctx_id_mapped(guc, ce->guc_id.id) &&
 intel_context_is_schedulable(ce)), 0)) {
	ret = try_context_registration(ce, false0);
	if (unlikely(ret == -EPIPE)__builtin_expect(!!(ret == -32), 0)) {
		goto deadlk;
	} else if (ret == -EBUSY16) {
		guc->stalled_request = last;
		guc->submission_stall_reason =
			STALL_REGISTER_CONTEXT;
		goto schedule_tasklet;
	} else if (ret != 0) {
		GEM_WARN_ON(ret)({ __builtin_expect(!!(!!(ret)), 0); });	/* Unexpected */
		goto deadlk;
	}
}

984move_lrc_tail:
if (is_multi_lrc_rq(last)) {
	ret = guc_wq_item_append(guc, last);
	if (ret == -EBUSY16) {
		goto schedule_tasklet;
	} else if (ret != 0) {
		GEM_WARN_ON(ret)({ __builtin_expect(!!(!!(ret)), 0); });	/* Unexpected */
		goto deadlk;
	}
} else {
	guc_set_lrc_tail(last);
}

997add_request:
ret = guc_add_request(guc, last);
if (unlikely(ret == -EPIPE)__builtin_expect(!!(ret == -32), 0)) {
	goto deadlk;
} else if (ret == -EBUSY16) {
	goto schedule_tasklet;
} else if (ret != 0) {
	GEM_WARN_ON(ret)({ __builtin_expect(!!(!!(ret)), 0); });	/* Unexpected */
	goto deadlk;
}
}

guc->stalled_request = NULL((void *)0);
guc->submission_stall_reason = STALL_NONE;
return submit;

1013deadlk:
sched_engine->tasklet.callback = NULL((void *)0);
tasklet_disable_nosync(&sched_engine->tasklet);
return false0;

1018schedule_tasklet:
tasklet_schedule(&sched_engine->tasklet);
return false0;
1021}

1023static void guc_submission_tasklet(struct tasklet_struct *t)
1024{
struct i915_sched_engine *sched_engine =
from_tasklet(sched_engine, t, tasklet)({ const __typeof( ((typeof(*sched_engine) *)0)->tasklet )
 *__mptr = (t); (typeof(*sched_engine) *)( (char *)__mptr - __builtin_offsetof
(typeof(*sched_engine), tasklet) );});
unsigned long flags;
bool_Bool loop;

spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);

do {
loop = guc_dequeue_one_context(sched_engine->private_data);
} while (loop);

i915_sched_engine_reset_on_empty(sched_engine);

spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
1039}

1041static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir)
1042{
if (iir & GT_RENDER_USER_INTERRUPT(1 << 0))
intel_engine_signal_breadcrumbs(engine);
1045}

1047static void __guc_context_destroy(struct intel_context *ce);
1048static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);
1049static void guc_signal_context_fence(struct intel_context *ce);
1050static void guc_cancel_context_requests(struct intel_context *ce);
1051static void guc_blocked_fence_complete(struct intel_context *ce);

1053static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
1054{
struct intel_context *ce;
unsigned long index, flags;
bool_Bool pending_disable, pending_enable, deregister, destroyed, banned;

xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
xa_for_each(&guc->context_lookup, index, ce)for (index = 0; ((ce) = xa_get_next(&guc->context_lookup
, &(index))) != ((void *)0); index++) {
/*
 * Corner case where the ref count on the object is zero but and
 * deregister G2H was lost. In this case we don't touch the ref
 * count and finish the destroy of the context.
 */
bool_Bool do_put = kref_get_unless_zero(&ce->ref);

xa_unlock(&guc->context_lookup)do { mtx_leave(&(&guc->context_lookup)->xa_lock
); } while (0);

spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);

/*
 * Once we are at this point submission_disabled() is guaranteed
 * to be visible to all callers who set the below flags (see above
 * flush and flushes in reset_prepare). If submission_disabled()
 * is set, the caller shouldn't set these flags.
 */

destroyed = context_destroyed(ce);
pending_enable = context_pending_enable(ce);
pending_disable = context_pending_disable(ce);
deregister = context_wait_for_deregister_to_register(ce);
banned = context_banned(ce);
init_sched_state(ce);

spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);

if (pending_enable || destroyed || deregister) {
	decr_outstanding_submission_g2h(guc);
	if (deregister)
		guc_signal_context_fence(ce);
	if (destroyed) {
		intel_gt_pm_put_async(guc_to_gt(guc));
		release_guc_id(guc, ce);
		__guc_context_destroy(ce);
	}
	if (pending_enable || deregister)
		intel_context_put(ce);
}

/* Not mutualy exclusive with above if statement. */
if (pending_disable) {
	guc_signal_context_fence(ce);
	if (banned) {
		guc_cancel_context_requests(ce);
		intel_engine_signal_breadcrumbs(ce->engine);
	}
	intel_context_sched_disable_unpin(ce);
	decr_outstanding_submission_g2h(guc);

	spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
	guc_blocked_fence_complete(ce);
	spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);

	intel_context_put(ce);
}

if (do_put)
	intel_context_put(ce);
xa_lock(&guc->context_lookup)do { mtx_enter(&(&guc->context_lookup)->xa_lock
); } while (0);
}
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);
1123}

1125/*
* GuC stores busyness stats for each engine at context in/out boundaries. A
* context 'in' logs execution start time, 'out' adds in -> out delta to total.
* i915/kmd accesses 'start', 'total' and 'context id' from memory shared with
* GuC.
*
* __i915_pmu_event_read samples engine busyness. When sampling, if context id
* is valid (!= ~0) and start is non-zero, the engine is considered to be
* active. For an active engine total busyness = total + (now - start), where
* 'now' is the time at which the busyness is sampled. For inactive engine,
* total busyness = total.
*
* All times are captured from GUCPMTIMESTAMP reg and are in gt clock domain.
*
* The start and total values provided by GuC are 32 bits and wrap around in a
* few minutes. Since perf pmu provides busyness as 64 bit monotonically
* increasing ns values, there is a need for this implementation to account for
* overflows and extend the GuC provided values to 64 bits before returning
* busyness to the user. In order to do that, a worker runs periodically at
* frequency = 1/8th the time it takes for the timestamp to wrap (i.e. once in
* 27 seconds for a gt clock frequency of 19.2 MHz).
*/

1148#define WRAP_TIME_CLKS0xffffffffU U32_MAX0xffffffffU
1149#define POLL_TIME_CLKS(0xffffffffU >> 3) (WRAP_TIME_CLKS0xffffffffU >> 3)

1151static void
1152__extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start)
1153{
u32 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp)((u32)(((guc->timestamp.gt_stamp) >> 16) >> 16
));
u32 gt_stamp_last = lower_32_bits(guc->timestamp.gt_stamp)((u32)(guc->timestamp.gt_stamp));

if (new_start == lower_32_bits(*prev_start)((u32)(*prev_start)))
return;

/*
* When gt is unparked, we update the gt timestamp and start the ping
* worker that updates the gt_stamp every POLL_TIME_CLKS. As long as gt
* is unparked, all switched in contexts will have a start time that is
* within +/- POLL_TIME_CLKS of the most recent gt_stamp.
*
* If neither gt_stamp nor new_start has rolled over, then the
* gt_stamp_hi does not need to be adjusted, however if one of them has
* rolled over, we need to adjust gt_stamp_hi accordingly.
*
* The below conditions address the cases of new_start rollover and
* gt_stamp_last rollover respectively.
*/
if (new_start < gt_stamp_last &&
   (new_start - gt_stamp_last) <= POLL_TIME_CLKS(0xffffffffU >> 3))
gt_stamp_hi++;

if (new_start > gt_stamp_last &&
   (gt_stamp_last - new_start) <= POLL_TIME_CLKS(0xffffffffU >> 3) && gt_stamp_hi)
gt_stamp_hi--;

*prev_start = ((u64)gt_stamp_hi << 32) | new_start;
1182}

1184#define record_read(map_, field_)iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_
) \
iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_)

1187/*
* GuC updates shared memory and KMD reads it. Since this is not synchronized,
* we run into a race where the value read is inconsistent. Sometimes the
* inconsistency is in reading the upper MSB bytes of the last_in value when
* this race occurs. 2 types of cases are seen - upper 8 bits are zero and upper
* 24 bits are zero. Since these are non-zero values, it is non-trivial to
* determine validity of these values. Instead we read the values multiple times
* until they are consistent. In test runs, 3 attempts results in consistent
* values. The upper bound is set to 6 attempts and may need to be tuned as per
* any new occurences.
*/
1198static void __get_engine_usage_record(struct intel_engine_cs *engine,
		      u32 *last_in, u32 *id, u32 *total)
1200{
STUB()do { printf("%s: stub\n", __func__); } while(0);
1202#ifdef notyet
struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine);
int i = 0;

do {
*last_in = record_read(&rec_map, last_switch_in_stamp)iosys_map_rd_field(&rec_map, 0, struct guc_engine_usage_record
, last_switch_in_stamp);
*id = record_read(&rec_map, current_context_index)iosys_map_rd_field(&rec_map, 0, struct guc_engine_usage_record
, current_context_index);
*total = record_read(&rec_map, total_runtime)iosys_map_rd_field(&rec_map, 0, struct guc_engine_usage_record
, total_runtime);

if (record_read(&rec_map, last_switch_in_stamp)iosys_map_rd_field(&rec_map, 0, struct guc_engine_usage_record
, last_switch_in_stamp) == *last_in &&
    record_read(&rec_map, current_context_index)iosys_map_rd_field(&rec_map, 0, struct guc_engine_usage_record
, current_context_index) == *id &&
    record_read(&rec_map, total_runtime)iosys_map_rd_field(&rec_map, 0, struct guc_engine_usage_record
, total_runtime) == *total)
	break;
} while (++i < 6);
1216#endif
1217}

1219static void guc_update_engine_gt_clks(struct intel_engine_cs *engine)
1220{
struct intel_engine_guc_stats *stats = &engine->stats.guc;
struct intel_guc *guc = &engine->gt->uc.guc;
u32 last_switch, ctx_id, total;

lockdep_assert_held(&guc->timestamp.lock)do { (void)(&guc->timestamp.lock); } while(0);

__get_engine_usage_record(engine, &last_switch, &ctx_id, &total);

stats->running = ctx_id != ~0U && last_switch;
if (stats->running)
__extend_last_switch(guc, &stats->start_gt_clk, last_switch);

/*
* Instead of adjusting the total for overflow, just add the
* difference from previous sample stats->total_gt_clks
*/
if (total && total != ~0U) {
stats->total_gt_clks += (u32)(total - stats->prev_total);
stats->prev_total = total;
}
1241}

1243static u32 gpm_timestamp_shift(struct intel_gt *gt)
1244{
intel_wakeref_t wakeref;
u32 reg, shift;

with_intel_runtime_pm(gt->uncore->rpm, wakeref)for ((wakeref) = intel_runtime_pm_get(gt->uncore->rpm);
 (wakeref); intel_runtime_pm_put((gt->uncore->rpm), (wakeref
)), (wakeref) = 0)
reg = intel_uncore_read(gt->uncore, RPM_CONFIG0((const i915_reg_t){ .reg = (0xd00) }));

shift = (reg & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK(0x3 << 1)) >>
GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT1;

return 3 - shift;
1255}

1257static void guc_update_pm_timestamp(struct intel_guc *guc, ktime_t *now)
1258{
struct intel_gt *gt = guc_to_gt(guc);
u32 gt_stamp_lo, gt_stamp_hi;
u64 gpm_ts;

lockdep_assert_held(&guc->timestamp.lock)do { (void)(&guc->timestamp.lock); } while(0);

gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp)((u32)(((guc->timestamp.gt_stamp) >> 16) >> 16
));
gpm_ts = intel_uncore_read64_2x32(gt->uncore, MISC_STATUS0((const i915_reg_t){ .reg = (0xa500) }),
			  MISC_STATUS1((const i915_reg_t){ .reg = (0xa504) })) >> guc->timestamp.shift;
gt_stamp_lo = lower_32_bits(gpm_ts)((u32)(gpm_ts));
*now = ktime_get();

if (gt_stamp_lo < lower_32_bits(guc->timestamp.gt_stamp)((u32)(guc->timestamp.gt_stamp)))
gt_stamp_hi++;

guc->timestamp.gt_stamp = ((u64)gt_stamp_hi << 32) | gt_stamp_lo;
1275}

1277/*
* Unlike the execlist mode of submission total and active times are in terms of
* gt clocks. The *now parameter is retained to return the cpu time at which the
* busyness was sampled.
*/
1282static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)
1283{
struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;
struct i915_gpu_error *gpu_error = &engine->i915->gpu_error;
struct intel_gt *gt = engine->gt;
struct intel_guc *guc = &gt->uc.guc;
u64 total, gt_stamp_saved;
unsigned long flags;
u32 reset_count;
bool_Bool in_reset;

spin_lock_irqsave(&guc->timestamp.lock, flags)do { flags = 0; mtx_enter(&guc->timestamp.lock); } while
 (0);

/*
* If a reset happened, we risk reading partially updated engine
* busyness from GuC, so we just use the driver stored copy of busyness.
* Synchronize with gt reset using reset_count and the
* I915_RESET_BACKOFF flag. Note that reset flow updates the reset_count
* after I915_RESET_BACKOFF flag, so ensure that the reset_count is
* usable by checking the flag afterwards.
*/
reset_count = i915_reset_count(gpu_error);
in_reset = test_bit(I915_RESET_BACKOFF0, &gt->reset.flags);

*now = ktime_get();

/*
* The active busyness depends on start_gt_clk and gt_stamp.
* gt_stamp is updated by i915 only when gt is awake and the
* start_gt_clk is derived from GuC state. To get a consistent
* view of activity, we query the GuC state only if gt is awake.
*/
if (!in_reset && intel_gt_pm_get_if_awake(gt)) {
stats_saved = *stats;
gt_stamp_saved = guc->timestamp.gt_stamp;
/*
 * Update gt_clks, then gt timestamp to simplify the 'gt_stamp -
 * start_gt_clk' calculation below for active engines.
 */
guc_update_engine_gt_clks(engine);
guc_update_pm_timestamp(guc, now);
intel_gt_pm_put_async(gt);
if (i915_reset_count(gpu_error) != reset_count) {
	*stats = stats_saved;
	guc->timestamp.gt_stamp = gt_stamp_saved;
}
}

total = intel_gt_clock_interval_to_ns(gt, stats->total_gt_clks);
if (stats->running) {
u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;

total += intel_gt_clock_interval_to_ns(gt, clk);
}

spin_unlock_irqrestore(&guc->timestamp.lock, flags)do { (void)(flags); mtx_leave(&guc->timestamp.lock); }
 while (0);

return ns_to_ktime(total);
1340}

1342static void __reset_guc_busyness_stats(struct intel_guc *guc)
1343{
struct intel_gt *gt = guc_to_gt(guc);
struct intel_engine_cs *engine;
enum intel_engine_id id;
unsigned long flags;
ktime_t unused;

cancel_delayed_work_sync(&guc->timestamp.work);

spin_lock_irqsave(&guc->timestamp.lock, flags)do { flags = 0; mtx_enter(&guc->timestamp.lock); } while
 (0);

guc_update_pm_timestamp(guc, &unused);
for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
guc_update_engine_gt_clks(engine);
engine->stats.guc.prev_total = 0;
}

spin_unlock_irqrestore(&guc->timestamp.lock, flags)do { (void)(flags); mtx_leave(&guc->timestamp.lock); }
 while (0);
1361}

1363static void __update_guc_busyness_stats(struct intel_guc *guc)
1364{
struct intel_gt *gt = guc_to_gt(guc);
struct intel_engine_cs *engine;
enum intel_engine_id id;
unsigned long flags;
ktime_t unused;

guc->timestamp.last_stat_jiffies = jiffies;

spin_lock_irqsave(&guc->timestamp.lock, flags)do { flags = 0; mtx_enter(&guc->timestamp.lock); } while
 (0);

guc_update_pm_timestamp(guc, &unused);
for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else
guc_update_engine_gt_clks(engine);

spin_unlock_irqrestore(&guc->timestamp.lock, flags)do { (void)(flags); mtx_leave(&guc->timestamp.lock); }
 while (0);
1380}

1382static void guc_timestamp_ping(struct work_struct *wrk)
1383{
struct intel_guc *guc = container_of(wrk, typeof(*guc),({ const __typeof( ((typeof(*guc) *)0)->timestamp.work.work
 ) *__mptr = (wrk); (typeof(*guc) *)( (char *)__mptr - __builtin_offsetof
(typeof(*guc), timestamp.work.work) );})
			     timestamp.work.work)({ const __typeof( ((typeof(*guc) *)0)->timestamp.work.work
 ) *__mptr = (wrk); (typeof(*guc) *)( (char *)__mptr - __builtin_offsetof
(typeof(*guc), timestamp.work.work) );});
struct intel_uc *uc = container_of(guc, typeof(*uc), guc)({ const __typeof( ((typeof(*uc) *)0)->guc ) *__mptr = (guc
); (typeof(*uc) *)( (char *)__mptr - __builtin_offsetof(typeof
(*uc), guc) );});
struct intel_gt *gt = guc_to_gt(guc);
intel_wakeref_t wakeref;
int srcu, ret;

/*
* Synchronize with gt reset to make sure the worker does not
* corrupt the engine/guc stats.
*/
ret = intel_gt_reset_trylock(gt, &srcu);
if (ret)
return;

with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(&gt->i915->runtime_pm
); (wakeref); intel_runtime_pm_put((&gt->i915->runtime_pm
), (wakeref)), (wakeref) = 0)
__update_guc_busyness_stats(guc);

intel_gt_reset_unlock(gt, srcu);

mod_delayed_work(system_highpri_wq, &guc->timestamp.work,
	 guc->timestamp.ping_delay);
1406}

1408static int guc_action_enable_usage_stats(struct intel_guc *guc)
1409{
u32 offset = intel_guc_engine_usage_offset(guc);
u32 action[] = {
INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF,
offset,
0,
};

return intel_guc_send(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])));
1418}

1420static void guc_init_engine_stats(struct intel_guc *guc)
1421{
struct intel_gt *gt = guc_to_gt(guc);
intel_wakeref_t wakeref;

mod_delayed_work(system_highpri_wq, &guc->timestamp.work,
	 guc->timestamp.ping_delay);

with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(&gt->i915->runtime_pm
); (wakeref); intel_runtime_pm_put((&gt->i915->runtime_pm
), (wakeref)), (wakeref) = 0) {
int ret = guc_action_enable_usage_stats(guc);

if (ret)
	drm_err(&gt->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to enable usage stats: %d!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ret)
		"Failed to enable usage stats: %d!\n", ret)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Failed to enable usage stats: %d!\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ret);
}
1435}

1437void intel_guc_busyness_park(struct intel_gt *gt)
1438{
struct intel_guc *guc = &gt->uc.guc;

if (!guc_submission_initialized(guc))
return;

/*
* There is a race with suspend flow where the worker runs after suspend
* and causes an unclaimed register access warning. Cancel the worker
* synchronously here.
*/
cancel_delayed_work_sync(&guc->timestamp.work);

/*
* Before parking, we should sample engine busyness stats if we need to.
* We can skip it if we are less than half a ping from the last time we
* sampled the busyness stats.
*/
if (guc->timestamp.last_stat_jiffies &&
   !time_after(jiffies, guc->timestamp.last_stat_jiffies +
	(guc->timestamp.ping_delay / 2)))
return;

__update_guc_busyness_stats(guc);
1462}

1464void intel_guc_busyness_unpark(struct intel_gt *gt)
1465{
struct intel_guc *guc = &gt->uc.guc;
unsigned long flags;
ktime_t unused;

if (!guc_submission_initialized(guc))
return;

spin_lock_irqsave(&guc->timestamp.lock, flags)do { flags = 0; mtx_enter(&guc->timestamp.lock); } while
 (0);
guc_update_pm_timestamp(guc, &unused);
spin_unlock_irqrestore(&guc->timestamp.lock, flags)do { (void)(flags); mtx_leave(&guc->timestamp.lock); }
 while (0);
mod_delayed_work(system_highpri_wq, &guc->timestamp.work,
	 guc->timestamp.ping_delay);
1478}

1480static inline bool_Bool
1481submission_disabled(struct intel_guc *guc)
1482{
struct i915_sched_engine * const sched_engine = guc->sched_engine;

return unlikely(!sched_engine ||__builtin_expect(!!(!sched_engine || !__tasklet_is_enabled(&
sched_engine->tasklet) || intel_gt_is_wedged(guc_to_gt(guc
))), 0)
	!__tasklet_is_enabled(&sched_engine->tasklet) ||__builtin_expect(!!(!sched_engine || !__tasklet_is_enabled(&
sched_engine->tasklet) || intel_gt_is_wedged(guc_to_gt(guc
))), 0)
	intel_gt_is_wedged(guc_to_gt(guc)))__builtin_expect(!!(!sched_engine || !__tasklet_is_enabled(&
sched_engine->tasklet) || intel_gt_is_wedged(guc_to_gt(guc
))), 0);
1488}

1490static void disable_submission(struct intel_guc *guc)
1491{
struct i915_sched_engine * const sched_engine = guc->sched_engine;

if (__tasklet_is_enabled(&sched_engine->tasklet)) {
GEM_BUG_ON(!guc->ct.enabled)((void)0);
__tasklet_disable_sync_once(&sched_engine->tasklet);
sched_engine->tasklet.callback = NULL((void *)0);
}
1499}

1501static void enable_submission(struct intel_guc *guc)
1502{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
unsigned long flags;

spin_lock_irqsave(&guc->sched_engine->lock, flags)do { flags = 0; mtx_enter(&guc->sched_engine->lock)
; } while (0);
sched_engine->tasklet.callback = guc_submission_tasklet;
wmb()do { __asm volatile("sfence" ::: "memory"); } while (0);	/* Make sure callback visible */
if (!__tasklet_is_enabled(&sched_engine->tasklet) &&
   __tasklet_enable(&sched_engine->tasklet)) {
GEM_BUG_ON(!guc->ct.enabled)((void)0);

/* And kick in case we missed a new request submission. */
tasklet_hi_schedule(&sched_engine->tasklet);
}
spin_unlock_irqrestore(&guc->sched_engine->lock, flags)do { (void)(flags); mtx_leave(&guc->sched_engine->lock
); } while (0);
1517}

1519static void guc_flush_submissions(struct intel_guc *guc)
1520{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
unsigned long flags;

spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);
spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
1526}

1528static void guc_flush_destroyed_contexts(struct intel_guc *guc);

1530void intel_guc_submission_reset_prepare(struct intel_guc *guc)
1531{
if (unlikely(!guc_submission_initialized(guc))__builtin_expect(!!(!guc_submission_initialized(guc)), 0)) {
/* Reset called during driver load? GuC not yet initialised! */
return;
}

intel_gt_park_heartbeats(guc_to_gt(guc));
disable_submission(guc);
guc->interrupts.disable(guc);
__reset_guc_busyness_stats(guc);

/* Flush IRQ handler */
spin_lock_irq(guc_to_gt(guc)->irq_lock)mtx_enter(guc_to_gt(guc)->irq_lock);
spin_unlock_irq(guc_to_gt(guc)->irq_lock)mtx_leave(guc_to_gt(guc)->irq_lock);

guc_flush_submissions(guc);
guc_flush_destroyed_contexts(guc);
flush_work(&guc->ct.requests.worker);

scrub_guc_desc_for_outstanding_g2h(guc);
1551}

1553static struct intel_engine_cs *
1554guc_virtual_get_sibling(struct intel_engine_cs *ve, unsigned int sibling)
1555{
struct intel_engine_cs *engine;
intel_engine_mask_t tmp, mask = ve->mask;
unsigned int num_siblings = 0;

for_each_engine_masked(engine, ve->gt, mask, tmp)for ((tmp) = (mask) & (ve->gt)->info.engine_mask; (
tmp) ? ((engine) = (ve->gt)->engine[({ int __idx = ffs(
tmp) - 1; tmp &= ~(1UL << (__idx)); __idx; })]), 1 :
 0;)
if (num_siblings++ == sibling)
	return engine;

return NULL((void *)0);
1565}

1567static inline struct intel_engine_cs *
1568__context_to_physical_engine(struct intel_context *ce)
1569{
struct intel_engine_cs *engine = ce->engine;

if (intel_engine_is_virtual(engine))
engine = guc_virtual_get_sibling(engine, 0);

return engine;
1576}

1578static void guc_reset_state(struct intel_context *ce, u32 head, bool_Bool scrub)
1579{
struct intel_engine_cs *engine = __context_to_physical_engine(ce);

if (!intel_context_is_schedulable(ce))
return;

GEM_BUG_ON(!intel_context_is_pinned(ce))((void)0);

/*
* We want a simple context + ring to execute the breadcrumb update.
* We cannot rely on the context being intact across the GPU hang,
* so clear it and rebuild just what we need for the breadcrumb.
* All pending requests for this context will be zapped, and any
* future request will be after userspace has had the opportunity
* to recreate its own state.
*/
if (scrub)
lrc_init_regs(ce, engine, true1);

/* Rerun the request; its payload has been neutered (if guilty). */
lrc_update_regs(ce, engine, head);
1600}

1602static void guc_engine_reset_prepare(struct intel_engine_cs *engine)
1603{
if (!IS_GRAPHICS_VER(engine->i915, 11, 12)(((&(engine->i915)->__runtime)->graphics.ip.ver)
 >= (11) && ((&(engine->i915)->__runtime
)->graphics.ip.ver) <= (12)))
return;

intel_engine_stop_cs(engine);

/*
* Wa_22011802037:gen11/gen12: In addition to stopping the cs, we need
* to wait for any pending mi force wakeups
*/
intel_engine_wait_for_pending_mi_fw(engine);
1614}

1616static void guc_reset_nop(struct intel_engine_cs *engine)
1617{
1618}

1620static void guc_rewind_nop(struct intel_engine_cs *engine, bool_Bool stalled)
1621{
1622}

1624static void
1625__unwind_incomplete_requests(struct intel_context *ce)
1626{
struct i915_request *rq, *rn;
struct list_head *pl;
int prio = I915_PRIORITY_INVALID((-0x7fffffff-1));
struct i915_sched_engine * const sched_engine =
ce->engine->sched_engine;
unsigned long flags;

spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);
spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
list_for_each_entry_safe_reverse(rq, rn,for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&ce->guc_state.requests)->prev); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = ((rq)->sched.link.prev); (__typeof(
*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &(rq)->sched.link != (&ce->guc_state
.requests); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *
)0)->sched.link ) *__mptr = (rn->sched.link.prev); (__typeof
(*rn) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rn), sched
.link) );}))
			 &ce->guc_state.requests,for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&ce->guc_state.requests)->prev); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = ((rq)->sched.link.prev); (__typeof(
*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &(rq)->sched.link != (&ce->guc_state
.requests); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *
)0)->sched.link ) *__mptr = (rn->sched.link.prev); (__typeof
(*rn) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rn), sched
.link) );}))
			 sched.link)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&ce->guc_state.requests)->prev); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = ((rq)->sched.link.prev); (__typeof(
*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &(rq)->sched.link != (&ce->guc_state
.requests); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *
)0)->sched.link ) *__mptr = (rn->sched.link.prev); (__typeof
(*rn) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rn), sched
.link) );})) {
if (i915_request_completed(rq))
	continue;

list_del_init(&rq->sched.link);
__i915_request_unsubmit(rq);

/* Push the request back into the queue for later resubmission. */
GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID)((void)0);
if (rq_prio(rq) != prio) {
	prio = rq_prio(rq);
	pl = i915_sched_lookup_priolist(sched_engine, prio);
}
GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine))((void)0);

list_add(&rq->sched.link, pl);
set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
}
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);
spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
1658}

1660static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t stalled)
1661{
bool_Bool guilty;
struct i915_request *rq;
unsigned long flags;
u32 head;
int i, number_children = ce->parallel.number_children;
struct intel_context *parent = ce;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

intel_context_get(ce);

/*
* GuC will implicitly mark the context as non-schedulable when it sends
* the reset notification. Make sure our state reflects this change. The
* context will be marked enabled on resubmission.
*/
spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
clr_context_enabled(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

/*
* For each context in the relationship find the hanging request
* resetting each context / request as needed
*/
for (i = 0; i < number_children + 1; ++i) {
if (!intel_context_is_pinned(ce))
	goto next_context;

guilty = false0;
rq = intel_context_get_active_request(ce);
if (!rq) {
	head = ce->ring->tail;
	goto out_replay;
}

if (i915_request_started(rq))
	guilty = stalled & ce->engine->mask;

GEM_BUG_ON(i915_active_is_idle(&ce->active))((void)0);
head = intel_ring_wrap(ce->ring, rq->head);

__i915_request_reset(rq, guilty);
i915_request_put(rq);
1705out_replay:
guc_reset_state(ce, head, guilty);
1707next_context:
if (i != number_children)
	ce = list_next_entry(ce, parallel.child_link)({ const __typeof( ((typeof(*(ce)) *)0)->parallel.child_link
 ) *__mptr = (((ce)->parallel.child_link.next)); (typeof(*
(ce)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(ce)), parallel
.child_link) );});
}

__unwind_incomplete_requests(parent);
intel_context_put(parent);
1714}

1716void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
1717{
struct intel_context *ce;
unsigned long index;
unsigned long flags;

if (unlikely(!guc_submission_initialized(guc))__builtin_expect(!!(!guc_submission_initialized(guc)), 0)) {
/* Reset called during driver load? GuC not yet initialised! */
return;
}

xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
xa_for_each(&guc->context_lookup, index, ce)for (index = 0; ((ce) = xa_get_next(&guc->context_lookup
, &(index))) != ((void *)0); index++) {
if (!kref_get_unless_zero(&ce->ref))
	continue;

xa_unlock(&guc->context_lookup)do { mtx_leave(&(&guc->context_lookup)->xa_lock
); } while (0);

if (intel_context_is_pinned(ce) &&
    !intel_context_is_child(ce))
	__guc_reset_context(ce, stalled);

intel_context_put(ce);

xa_lock(&guc->context_lookup)do { mtx_enter(&(&guc->context_lookup)->xa_lock
); } while (0);
}
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);

/* GuC is blown away, drop all references to contexts */
xa_destroy(&guc->context_lookup);
1746}

1748static void guc_cancel_context_requests(struct intel_context *ce)
1749{
struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine;
struct i915_request *rq;
unsigned long flags;

/* Mark all executing requests as skipped. */
spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);
spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
list_for_each_entry(rq, &ce->guc_state.requests, sched.link)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&ce->guc_state.requests)->next); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &rq->sched.link != (&ce->guc_state
.requests); rq = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = (rq->sched.link.next); (__typeof(*rq
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}))
i915_request_put(i915_request_mark_eio(rq));
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);
spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
1761}

1763static void
1764guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine)
1765{
struct i915_request *rq, *rn;
struct rb_node *rb;
unsigned long flags;

/* Can be called during boot if GuC fails to load */
if (!sched_engine)
return;

/*
* Before we call engine->cancel_requests(), we should have exclusive
* access to the submission state. This is arranged for us by the
* caller disabling the interrupt generation, the tasklet and other
* threads that may then access the same state, giving us a free hand
* to reset state. However, we still need to let lockdep be aware that
* we know this state may be accessed in hardirq context, so we
* disable the irq around this manipulation and we want to keep
* the spinlock focused on its duties and not accidentally conflate
* coverage to the submission's irq state. (Similarly, although we
* shouldn't need to disable irq around the manipulation of the
* submission's irq state, we also wish to remind ourselves that
* it is irq state.)
*/
spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);

/* Flush the queued requests to the timeline list (for retiring). */
while ((rb = rb_first_cached(&sched_engine->queue)linux_root_RB_MINMAX((struct linux_root *)(&(&sched_engine
->queue)->rb_root), -1))) {
struct i915_priolist *p = to_priolist(rb);

priolist_for_each_request_consume(rq, rn, p)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&(p)->requests)->next); (__typeof(*rq
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = (rq->sched.link.next); (__typeof(*rq
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &rq->sched.link != (&(p)->requests
); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *)0)->
sched.link ) *__mptr = (rn->sched.link.next); (__typeof(*rn
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rn), sched
.link) );})) {
	list_del_init(&rq->sched.link);

	__i915_request_submit(rq);

	i915_request_put(i915_request_mark_eio(rq));
}

rb_erase_cached(&p->node, &sched_engine->queue)linux_root_RB_REMOVE((struct linux_root *)(&(&sched_engine
->queue)->rb_root), (&p->node));
i915_priolist_free(p);
}

/* Remaining _unready_ requests will be nop'ed when submitted */

sched_engine->queue_priority_hint = INT_MIN(-0x7fffffff-1);
sched_engine->queue = RB_ROOT_CACHED(struct rb_root_cached) { ((void *)0) };

spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
1812}

1814void intel_guc_submission_cancel_requests(struct intel_guc *guc)
1815{
struct intel_context *ce;
unsigned long index;
unsigned long flags;

xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
xa_for_each(&guc->context_lookup, index, ce)for (index = 0; ((ce) = xa_get_next(&guc->context_lookup
, &(index))) != ((void *)0); index++) {
if (!kref_get_unless_zero(&ce->ref))
	continue;

xa_unlock(&guc->context_lookup)do { mtx_leave(&(&guc->context_lookup)->xa_lock
); } while (0);

if (intel_context_is_pinned(ce) &&
    !intel_context_is_child(ce))
	guc_cancel_context_requests(ce);

intel_context_put(ce);

xa_lock(&guc->context_lookup)do { mtx_enter(&(&guc->context_lookup)->xa_lock
); } while (0);
}
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);

guc_cancel_sched_engine_requests(guc->sched_engine);

/* GuC is blown away, drop all references to contexts */
xa_destroy(&guc->context_lookup);
1841}

1843void intel_guc_submission_reset_finish(struct intel_guc *guc)
1844{
/* Reset called during driver load or during wedge? */
if (unlikely(!guc_submission_initialized(guc) ||__builtin_expect(!!(!guc_submission_initialized(guc) || intel_gt_is_wedged
(guc_to_gt(guc))), 0)
     intel_gt_is_wedged(guc_to_gt(guc)))__builtin_expect(!!(!guc_submission_initialized(guc) || intel_gt_is_wedged
(guc_to_gt(guc))), 0)) {
return;
}

/*
* Technically possible for either of these values to be non-zero here,
* but very unlikely + harmless. Regardless let's add a warn so we can
* see in CI if this happens frequently / a precursor to taking down the
* machine.
*/
GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h))({ __builtin_expect(!!(!!(({ typeof(*(&guc->outstanding_submission_g2h
)) __tmp = *(volatile typeof(*(&guc->outstanding_submission_g2h
)) *)&(*(&guc->outstanding_submission_g2h)); membar_datadep_consumer
(); __tmp; }))), 0); });
atomic_set(&guc->outstanding_submission_g2h, 0)({ typeof(*(&guc->outstanding_submission_g2h)) __tmp =
 ((0)); *(volatile typeof(*(&guc->outstanding_submission_g2h
)) *)&(*(&guc->outstanding_submission_g2h)) = __tmp
; __tmp; });

intel_guc_global_policies_update(guc);
enable_submission(guc);
intel_gt_unpark_heartbeats(guc_to_gt(guc));
1863}

1865static void destroyed_worker_func(struct work_struct *w);
1866static void reset_fail_worker_func(struct work_struct *w);

1868/*
* Set up the memory resources to be shared with the GuC (via the GGTT)
* at firmware loading time.
*/
1872int intel_guc_submission_init(struct intel_guc *guc)
1873{
struct intel_gt *gt = guc_to_gt(guc);
int ret;

if (guc->submission_initialized)
return 0;

if (GET_UC_VER(guc)((((guc)->fw.file_selected.patch_ver) | (((guc)->fw.file_selected
.minor_ver) << 8) | (((guc)->fw.file_selected.major_ver
) << 16))) < MAKE_UC_VER(70, 0, 0)((0) | ((0) << 8) | ((70) << 16))) {
ret = guc_lrc_desc_pool_create_v69(guc);
if (ret)
	return ret;
}

guc->submission_state.guc_ids_bitmap =
bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc)((guc)->submission_state.num_guc_ids / 16), GFP_KERNEL(0x0001 | 0x0004));
if (!guc->submission_state.guc_ids_bitmap) {
ret = -ENOMEM12;
goto destroy_pool;
}

guc->timestamp.ping_delay = (POLL_TIME_CLKS(0xffffffffU >> 3) / gt->clock_frequency + 1) * HZhz;
guc->timestamp.shift = gpm_timestamp_shift(gt);
guc->submission_initialized = true1;

return 0;

1899destroy_pool:
guc_lrc_desc_pool_destroy_v69(guc);

return ret;
1903}

1905void intel_guc_submission_fini(struct intel_guc *guc)
1906{
if (!guc->submission_initialized)
return;

guc_flush_destroyed_contexts(guc);
guc_lrc_desc_pool_destroy_v69(guc);
i915_sched_engine_put(guc->sched_engine);
bitmap_free(guc->submission_state.guc_ids_bitmap);
guc->submission_initialized = false0;
1915}

1917static inline void queue_request(struct i915_sched_engine *sched_engine,
		 struct i915_request *rq,
		 int prio)
1920{
GEM_BUG_ON(!list_empty(&rq->sched.link))((void)0);
list_add_tail(&rq->sched.link,
      i915_sched_lookup_priolist(sched_engine, prio));
set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
tasklet_hi_schedule(&sched_engine->tasklet);
1926}

1928static int guc_bypass_tasklet_submit(struct intel_guc *guc,
		     struct i915_request *rq)
1930{
int ret = 0;

__i915_request_submit(rq);

trace_i915_request_in(rq, 0);

if (is_multi_lrc_rq(rq)) {
if (multi_lrc_submit(rq)) {
	ret = guc_wq_item_append(guc, rq);
	if (!ret)
		ret = guc_add_request(guc, rq);
}
} else {
guc_set_lrc_tail(rq);
ret = guc_add_request(guc, rq);
}

if (unlikely(ret == -EPIPE)__builtin_expect(!!(ret == -32), 0))
disable_submission(guc);

return ret;
1952}

1954static bool_Bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)
1955{
struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
struct intel_context *ce = request_to_scheduling_context(rq);

return submission_disabled(guc) || guc->stalled_request ||
!i915_sched_engine_is_empty(sched_engine) ||
!ctx_id_mapped(guc, ce->guc_id.id);
1962}

1964static void guc_submit_request(struct i915_request *rq)
1965{
struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
struct intel_guc *guc = &rq->engine->gt->uc.guc;
unsigned long flags;

/* Will be called from irq-context when using foreign fences. */
spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);

if (need_tasklet(guc, rq))
queue_request(sched_engine, rq, rq_prio(rq));
else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY16)
tasklet_hi_schedule(&sched_engine->tasklet);

spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
1979}

1981static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
1982{
STUB()do { printf("%s: stub\n", __func__); } while(0);
return -ENOSYS78;
1985#ifdef notyet
int ret;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

if (intel_context_is_parent(ce))
ret = bitmap_find_free_region(guc->submission_state.guc_ids_bitmap,
			      NUMBER_MULTI_LRC_GUC_ID(guc)((guc)->submission_state.num_guc_ids / 16),
			      order_base_2(ce->parallel.number_childrendrm_order(ce->parallel.number_children + 1)
					   + 1)drm_order(ce->parallel.number_children + 1));
else
ret = ida_simple_get(&guc->submission_state.guc_ids,
		     NUMBER_MULTI_LRC_GUC_ID(guc)((guc)->submission_state.num_guc_ids / 16),
		     guc->submission_state.num_guc_ids,
		     GFP_KERNEL(0x0001 | 0x0004) | __GFP_RETRY_MAYFAIL0 |
		     __GFP_NOWARN0);
if (unlikely(ret < 0)__builtin_expect(!!(ret < 0), 0))
return ret;

ce->guc_id.id = ret;
return 0;
2006#endif
2007}

2009static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2010{
STUB()do { printf("%s: stub\n", __func__); } while(0);
2012#ifdef notyet
GEM_BUG_ON(intel_context_is_child(ce))((void)0);

if (!context_guc_id_invalid(ce)) {
if (intel_context_is_parent(ce))
	bitmap_release_region(guc->submission_state.guc_ids_bitmap,
			      ce->guc_id.id,
			      order_base_2(ce->parallel.number_childrendrm_order(ce->parallel.number_children + 1)
					   + 1)drm_order(ce->parallel.number_children + 1));
else
	ida_simple_remove(&guc->submission_state.guc_ids,
			  ce->guc_id.id);
clr_ctx_id_mapping(guc, ce->guc_id.id);
set_context_guc_id_invalid(ce);
}
if (!list_empty(&ce->guc_id.link))
list_del_init(&ce->guc_id.link);
2029#endif
2030}

2032static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2033{
unsigned long flags;

spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
__release_guc_id(guc, ce);
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);
2039}

2041static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce)
2042{
struct intel_context *cn;

lockdep_assert_held(&guc->submission_state.lock)do { (void)(&guc->submission_state.lock); } while(0);
GEM_BUG_ON(intel_context_is_child(ce))((void)0);
GEM_BUG_ON(intel_context_is_parent(ce))((void)0);

if (!list_empty(&guc->submission_state.guc_id_list)) {
cn = list_first_entry(&guc->submission_state.guc_id_list,({ const __typeof( ((struct intel_context *)0)->guc_id.link
 ) *__mptr = ((&guc->submission_state.guc_id_list)->
next); (struct intel_context *)( (char *)__mptr - __builtin_offsetof
(struct intel_context, guc_id.link) );})
		      struct intel_context,({ const __typeof( ((struct intel_context *)0)->guc_id.link
 ) *__mptr = ((&guc->submission_state.guc_id_list)->
next); (struct intel_context *)( (char *)__mptr - __builtin_offsetof
(struct intel_context, guc_id.link) );})
		      guc_id.link)({ const __typeof( ((struct intel_context *)0)->guc_id.link
 ) *__mptr = ((&guc->submission_state.guc_id_list)->
next); (struct intel_context *)( (char *)__mptr - __builtin_offsetof
(struct intel_context, guc_id.link) );});

GEM_BUG_ON(atomic_read(&cn->guc_id.ref))((void)0);
GEM_BUG_ON(context_guc_id_invalid(cn))((void)0);
GEM_BUG_ON(intel_context_is_child(cn))((void)0);
GEM_BUG_ON(intel_context_is_parent(cn))((void)0);

list_del_init(&cn->guc_id.link);
ce->guc_id.id = cn->guc_id.id;

spin_lock(&cn->guc_state.lock)mtx_enter(&cn->guc_state.lock);
clr_context_registered(cn);
spin_unlock(&cn->guc_state.lock)mtx_leave(&cn->guc_state.lock);

set_context_guc_id_invalid(cn);

2068#ifdef CONFIG_DRM_I915_SELFTEST
guc->number_guc_id_stolen++;
2070#endif

return 0;
} else {
return -EAGAIN35;
}
2076}

2078static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce)
2079{
int ret;

lockdep_assert_held(&guc->submission_state.lock)do { (void)(&guc->submission_state.lock); } while(0);
GEM_BUG_ON(intel_context_is_child(ce))((void)0);

ret = new_guc_id(guc, ce);
if (unlikely(ret < 0)__builtin_expect(!!(ret < 0), 0)) {
if (intel_context_is_parent(ce))
	return -ENOSPC28;

ret = steal_guc_id(guc, ce);
if (ret < 0)
	return ret;
}

if (intel_context_is_parent(ce)) {
struct intel_context *child;
int i = 1;

for_each_child(ce, child)for (child = ({ const __typeof( ((__typeof(*child) *)0)->parallel
.child_link ) *__mptr = ((&(ce)->parallel.child_list)->
next); (__typeof(*child) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*child), parallel.child_link) );}); &child->
parallel.child_link != (&(ce)->parallel.child_list); child
 = ({ const __typeof( ((__typeof(*child) *)0)->parallel.child_link
 ) *__mptr = (child->parallel.child_link.next); (__typeof(
*child) *)( (char *)__mptr - __builtin_offsetof(__typeof(*child
), parallel.child_link) );}))
	child->guc_id.id = ce->guc_id.id + i++;
}

return 0;
2104}

2106#define PIN_GUC_ID_TRIES4	4
2107static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2108{
int ret = 0;
unsigned long flags, tries = PIN_GUC_ID_TRIES4;

GEM_BUG_ON(atomic_read(&ce->guc_id.ref))((void)0);

2114try_again:
spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);

might_lock(&ce->guc_state.lock);

if (context_guc_id_invalid(ce)) {
ret = assign_guc_id(guc, ce);
if (ret)
	goto out_unlock;
ret = 1;	/* Indidcates newly assigned guc_id */
}
if (!list_empty(&ce->guc_id.link))
list_del_init(&ce->guc_id.link);
atomic_inc(&ce->guc_id.ref)__sync_fetch_and_add(&ce->guc_id.ref, 1);

2129out_unlock:
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);

/*
* -EAGAIN indicates no guc_id are available, let's retire any
* outstanding requests to see if that frees up a guc_id. If the first
* retire didn't help, insert a sleep with the timeslice duration before
* attempting to retire more requests. Double the sleep period each
* subsequent pass before finally giving up. The sleep period has max of
* 100ms and minimum of 1ms.
*/
if (ret == -EAGAIN35 && --tries) {
if (PIN_GUC_ID_TRIES4 - tries > 1) {
	unsigned int timeslice_shifted =
		ce->engine->props.timeslice_duration_ms <<
		(PIN_GUC_ID_TRIES4 - tries - 2);
	unsigned int max = min_t(unsigned int, 100,({ unsigned int __min_a = (100); unsigned int __min_b = (timeslice_shifted
); __min_a < __min_b ? __min_a : __min_b; })
				 timeslice_shifted)({ unsigned int __min_a = (100); unsigned int __min_b = (timeslice_shifted
); __min_a < __min_b ? __min_a : __min_b; });

	drm_msleep(max_t(unsigned int, max, 1))mdelay(({ unsigned int __max_a = (max); unsigned int __max_b =
 (1); __max_a > __max_b ? __max_a : __max_b; }));
}
intel_gt_retire_requests(guc_to_gt(guc));
goto try_again;
}

return ret;
2155}

2157static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2158{
unsigned long flags;

GEM_BUG_ON(atomic_read(&ce->guc_id.ref) < 0)((void)0);
GEM_BUG_ON(intel_context_is_child(ce))((void)0);

if (unlikely(context_guc_id_invalid(ce) ||__builtin_expect(!!(context_guc_id_invalid(ce) || intel_context_is_parent
(ce)), 0)
     intel_context_is_parent(ce))__builtin_expect(!!(context_guc_id_invalid(ce) || intel_context_is_parent
(ce)), 0))
return;

spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id.link) &&
   !atomic_read(&ce->guc_id.ref)({ typeof(*(&ce->guc_id.ref)) __tmp = *(volatile typeof
(*(&ce->guc_id.ref)) *)&(*(&ce->guc_id.ref)
); membar_datadep_consumer(); __tmp; }))
list_add_tail(&ce->guc_id.link,
	      &guc->submission_state.guc_id_list);
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);
2174}

2176static int __guc_action_register_multi_lrc_v69(struct intel_guc *guc,
			       struct intel_context *ce,
			       u32 guc_id,
			       u32 offset,
			       bool_Bool loop)
2181{
struct intel_context *child;
u32 action[4 + MAX_ENGINE_INSTANCE8];
int len = 0;

GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE)((void)0);

action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
action[len++] = guc_id;
action[len++] = ce->parallel.number_children + 1;
action[len++] = offset;
for_each_child(ce, child)for (child = ({ const __typeof( ((__typeof(*child) *)0)->parallel
.child_link ) *__mptr = ((&(ce)->parallel.child_list)->
next); (__typeof(*child) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*child), parallel.child_link) );}); &child->
parallel.child_link != (&(ce)->parallel.child_list); child
 = ({ const __typeof( ((__typeof(*child) *)0)->parallel.child_link
 ) *__mptr = (child->parallel.child_link.next); (__typeof(
*child) *)( (char *)__mptr - __builtin_offsetof(__typeof(*child
), parallel.child_link) );})) {
offset += sizeof(struct guc_lrc_desc_v69);
action[len++] = offset;
}

return guc_submission_send_busy_loop(guc, action, len, 0, loop);
2198}

2200static int __guc_action_register_multi_lrc_v70(struct intel_guc *guc,
			       struct intel_context *ce,
			       struct guc_ctxt_registration_info *info,
			       bool_Bool loop)
2204{
struct intel_context *child;
u32 action[13 + (MAX_ENGINE_INSTANCE8 * 2)];
int len = 0;
u32 next_id;

GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE)((void)0);

action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
action[len++] = info->flags;
action[len++] = info->context_idx;
action[len++] = info->engine_class;
action[len++] = info->engine_submit_mask;
action[len++] = info->wq_desc_lo;
action[len++] = info->wq_desc_hi;
action[len++] = info->wq_base_lo;
action[len++] = info->wq_base_hi;
action[len++] = info->wq_size;
action[len++] = ce->parallel.number_children + 1;
action[len++] = info->hwlrca_lo;
action[len++] = info->hwlrca_hi;

next_id = info->context_idx + 1;
for_each_child(ce, child)for (child = ({ const __typeof( ((__typeof(*child) *)0)->parallel
.child_link ) *__mptr = ((&(ce)->parallel.child_list)->
next); (__typeof(*child) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*child), parallel.child_link) );}); &child->
parallel.child_link != (&(ce)->parallel.child_list); child
 = ({ const __typeof( ((__typeof(*child) *)0)->parallel.child_link
 ) *__mptr = (child->parallel.child_link.next); (__typeof(
*child) *)( (char *)__mptr - __builtin_offsetof(__typeof(*child
), parallel.child_link) );})) {
GEM_BUG_ON(next_id++ != child->guc_id.id)((void)0);

/*
 * NB: GuC interface supports 64 bit LRCA even though i915/HW
 * only supports 32 bit currently.
 */
action[len++] = lower_32_bits(child->lrc.lrca)((u32)(child->lrc.lrca));
action[len++] = upper_32_bits(child->lrc.lrca)((u32)(((child->lrc.lrca) >> 16) >> 16));
}

GEM_BUG_ON(len > ARRAY_SIZE(action))((void)0);

return guc_submission_send_busy_loop(guc, action, len, 0, loop);
2241}

2243static int __guc_action_register_context_v69(struct intel_guc *guc,
			     u32 guc_id,
			     u32 offset,
			     bool_Bool loop)
2247{
u32 action[] = {
INTEL_GUC_ACTION_REGISTER_CONTEXT,
guc_id,
offset,
};

return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])),
			     0, loop);
2256}

2258static int __guc_action_register_context_v70(struct intel_guc *guc,
			     struct guc_ctxt_registration_info *info,
			     bool_Bool loop)
2261{
u32 action[] = {
INTEL_GUC_ACTION_REGISTER_CONTEXT,
info->flags,
info->context_idx,
info->engine_class,
info->engine_submit_mask,
info->wq_desc_lo,
info->wq_desc_hi,
info->wq_base_lo,
info->wq_base_hi,
info->wq_size,
info->hwlrca_lo,
info->hwlrca_hi,
};

return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])),
			     0, loop);
2279}

2281static void prepare_context_registration_info_v69(struct intel_context *ce);
2282static void prepare_context_registration_info_v70(struct intel_context *ce,
				  struct guc_ctxt_registration_info *info);

2285static int
2286register_context_v69(struct intel_guc *guc, struct intel_context *ce, bool_Bool loop)
2287{
u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool_v69) +
ce->guc_id.id * sizeof(struct guc_lrc_desc_v69);

prepare_context_registration_info_v69(ce);

if (intel_context_is_parent(ce))
return __guc_action_register_multi_lrc_v69(guc, ce, ce->guc_id.id,
					   offset, loop);
else
return __guc_action_register_context_v69(guc, ce->guc_id.id,
					 offset, loop);
2299}

2301static int
2302register_context_v70(struct intel_guc *guc, struct intel_context *ce, bool_Bool loop)
2303{
struct guc_ctxt_registration_info info;

prepare_context_registration_info_v70(ce, &info);

if (intel_context_is_parent(ce))
return __guc_action_register_multi_lrc_v70(guc, ce, &info, loop);
else
return __guc_action_register_context_v70(guc, &info, loop);
2312}

2314static int register_context(struct intel_context *ce, bool_Bool loop)
2315{
struct intel_guc *guc = ce_to_guc(ce);
int ret;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);
trace_intel_context_register(ce);

if (GET_UC_VER(guc)((((guc)->fw.file_selected.patch_ver) | (((guc)->fw.file_selected
.minor_ver) << 8) | (((guc)->fw.file_selected.major_ver
) << 16))) >= MAKE_UC_VER(70, 0, 0)((0) | ((0) << 8) | ((70) << 16)))
ret = register_context_v70(guc, ce, loop);
else
ret = register_context_v69(guc, ce, loop);

if (likely(!ret)__builtin_expect(!!(!ret), 1)) {
unsigned long flags;

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
set_context_registered(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

if (GET_UC_VER(guc)((((guc)->fw.file_selected.patch_ver) | (((guc)->fw.file_selected
.minor_ver) << 8) | (((guc)->fw.file_selected.major_ver
) << 16))) >= MAKE_UC_VER(70, 0, 0)((0) | ((0) << 8) | ((70) << 16)))
	guc_context_policy_init_v70(ce, loop);
}

return ret;
2339}

2341static int __guc_action_deregister_context(struct intel_guc *guc,
			   u32 guc_id)
2343{
u32 action[] = {
INTEL_GUC_ACTION_DEREGISTER_CONTEXT,
guc_id,
};

return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])),
			     G2H_LEN_DW_DEREGISTER_CONTEXT1,
			     true1);
2352}

2354static int deregister_context(struct intel_context *ce, u32 guc_id)
2355{
struct intel_guc *guc = ce_to_guc(ce);

GEM_BUG_ON(intel_context_is_child(ce))((void)0);
trace_intel_context_deregister(ce);

return __guc_action_deregister_context(guc, guc_id);
2362}

2364static inline void clear_children_join_go_memory(struct intel_context *ce)
2365{
struct parent_scratch *ps = __get_parent_scratch(ce);
int i;

ps->go.semaphore = 0;
for (i = 0; i < ce->parallel.number_children + 1; ++i)
ps->join[i].semaphore = 0;
2372}

2374static inline u32 get_children_go_value(struct intel_context *ce)
2375{
return __get_parent_scratch(ce)->go.semaphore;
2377}

2379static inline u32 get_children_join_value(struct intel_context *ce,
			  u8 child_index)
2381{
return __get_parent_scratch(ce)->join[child_index].semaphore;
2383}

2385struct context_policy {
u32 count;
struct guc_update_context_policy h2g;
2388};

2390static u32 __guc_context_policy_action_size(struct context_policy *policy)
2391{
size_t bytes = sizeof(policy->h2g.header) +
       (sizeof(policy->h2g.klv[0]) * policy->count);

return bytes / sizeof(u32);
2396}

2398static void __guc_context_policy_start_klv(struct context_policy *policy, u16 guc_id)
2399{
policy->h2g.header.action = INTEL_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
policy->h2g.header.ctx_id = guc_id;
policy->count = 0;
2403}

2405#define MAKE_CONTEXT_POLICY_ADD(func, id) \
2406static void __guc_context_policy_add_##func(struct context_policy *policy, u32 data) \
2407{ \
GEM_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS)((void)0); \
policy->h2g.klv[policy->count].kl = \
FIELD_PREP(GUC_KLV_0_KEY, GUC_CONTEXT_POLICIES_KLV_ID_##id)(((typeof((0xffff << 16)))(GUC_CONTEXT_POLICIES_KLV_ID_
##id) << (__builtin_ffsll((0xffff << 16)) - 1)) &
 ((0xffff << 16))) | \
FIELD_PREP(GUC_KLV_0_LEN, 1)(((typeof((0xffff << 0)))(1) << (__builtin_ffsll(
(0xffff << 0)) - 1)) & ((0xffff << 0))); \
policy->h2g.klv[policy->count].value = data; \
policy->count++; \
2414}

2416MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
2417MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
2418MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY)
2419MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY)

2421#undef MAKE_CONTEXT_POLICY_ADD

2423static int __guc_context_set_context_policies(struct intel_guc *guc,
			      struct context_policy *policy,
			      bool_Bool loop)
2426{
return guc_submission_send_busy_loop(guc, (u32 *)&policy->h2g,
			__guc_context_policy_action_size(policy),
			0, loop);
2430}

2432static int guc_context_policy_init_v70(struct intel_context *ce, bool_Bool loop)
2433{
struct intel_engine_cs *engine = ce->engine;
struct intel_guc *guc = &engine->gt->uc.guc;
struct context_policy policy;
u32 execution_quantum;
u32 preemption_timeout;
unsigned long flags;
int ret;

/* NB: For both of these, zero means disabled. */
GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,((void)0)
		  execution_quantum))((void)0);
GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,((void)0)
		  preemption_timeout))((void)0);
execution_quantum = engine->props.timeslice_duration_ms * 1000;
preemption_timeout = engine->props.preempt_timeout_ms * 1000;

__guc_context_policy_start_klv(&policy, ce->guc_id.id);

__guc_context_policy_add_priority(&policy, ce->guc_state.prio);
__guc_context_policy_add_execution_quantum(&policy, execution_quantum);
__guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);

if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION(1UL << (8)))
__guc_context_policy_add_preempt_to_idle(&policy, 1);

ret = __guc_context_set_context_policies(guc, &policy, loop);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
if (ret != 0)
set_context_policy_required(ce);
else
clr_context_policy_required(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

return ret;
2469}

2471static void guc_context_policy_init_v69(struct intel_engine_cs *engine,
			struct guc_lrc_desc_v69 *desc)
2473{
desc->policy_flags = 0;

if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION(1UL << (8)))
desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE_V69(1UL << (0));

/* NB: For both of these, zero means disabled. */
GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,((void)0)
		  desc->execution_quantum))((void)0);
GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,((void)0)
		  desc->preemption_timeout))((void)0);
desc->execution_quantum = engine->props.timeslice_duration_ms * 1000;
desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000;
2486}

2488static u32 map_guc_prio_to_lrc_desc_prio(u8 prio)
2489{
/*
* this matches the mapping we do in map_i915_prio_to_guc_prio()
* (e.g. prio < I915_PRIORITY_NORMAL maps to GUC_CLIENT_PRIORITY_NORMAL)
*/
switch (prio) {
default:
MISSING_CASE(prio)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "prio", (long)(prio)); __builtin_expect(!!(__ret), 0); });
fallthroughdo {} while (0);
case GUC_CLIENT_PRIORITY_KMD_NORMAL2:
return GEN12_CTX_PRIORITY_NORMAL(((typeof((((~0UL) >> (64 - (10) - 1)) & ((~0UL) <<
 (9)))))(1) << (__builtin_ffsll((((~0UL) >> (64 -
 (10) - 1)) & ((~0UL) << (9)))) - 1)) & ((((~0UL
) >> (64 - (10) - 1)) & ((~0UL) << (9)))));
case GUC_CLIENT_PRIORITY_NORMAL3:
return GEN12_CTX_PRIORITY_LOW(((typeof((((~0UL) >> (64 - (10) - 1)) & ((~0UL) <<
 (9)))))(0) << (__builtin_ffsll((((~0UL) >> (64 -
 (10) - 1)) & ((~0UL) << (9)))) - 1)) & ((((~0UL
) >> (64 - (10) - 1)) & ((~0UL) << (9)))));
case GUC_CLIENT_PRIORITY_HIGH1:
case GUC_CLIENT_PRIORITY_KMD_HIGH0:
return GEN12_CTX_PRIORITY_HIGH(((typeof((((~0UL) >> (64 - (10) - 1)) & ((~0UL) <<
 (9)))))(2) << (__builtin_ffsll((((~0UL) >> (64 -
 (10) - 1)) & ((~0UL) << (9)))) - 1)) & ((((~0UL
) >> (64 - (10) - 1)) & ((~0UL) << (9)))));
}
2506}

2508static void prepare_context_registration_info_v69(struct intel_context *ce)
2509{
struct intel_engine_cs *engine = ce->engine;
struct intel_guc *guc = &engine->gt->uc.guc;
u32 ctx_id = ce->guc_id.id;
struct guc_lrc_desc_v69 *desc;
struct intel_context *child;

GEM_BUG_ON(!engine->mask)((void)0);

/*
* Ensure LRC + CT vmas are is same region as write barrier is done
* based on CT vma region.
*/
GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=((void)0)
   i915_gem_object_is_lmem(ce->ring->vma->obj))((void)0);

desc = __get_lrc_desc_v69(guc, ctx_id);
desc->engine_class = engine_class_to_guc_class(engine->class);
desc->engine_submit_mask = engine->logical_mask;
desc->hw_context_desc = ce->lrc.lrca;
desc->priority = ce->guc_state.prio;
desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD(1UL << (0));
guc_context_policy_init_v69(engine, desc);

/*
* If context is a parent, we need to register a process descriptor
* describing a work queue and register all child contexts.
*/
if (intel_context_is_parent(ce)) {
struct guc_process_desc_v69 *pdesc;

ce->parallel.guc.wqi_tail = 0;
ce->parallel.guc.wqi_head = 0;

desc->process_desc = i915_ggtt_offset(ce->state) +
	__get_parent_scratch_offset(ce);
desc->wq_addr = i915_ggtt_offset(ce->state) +
	__get_wq_offset(ce);
desc->wq_size = WQ_SIZE((1 << 12) / 2);

pdesc = __get_process_desc_v69(ce);
memset(pdesc, 0, sizeof(*(pdesc)))__builtin_memset((pdesc), (0), (sizeof(*(pdesc))));
pdesc->stage_id = ce->guc_id.id;
pdesc->wq_base_addr = desc->wq_addr;
pdesc->wq_size_bytes = desc->wq_size;
pdesc->wq_status = WQ_STATUS_ACTIVE1;

ce->parallel.guc.wq_head = &pdesc->head;
ce->parallel.guc.wq_tail = &pdesc->tail;
ce->parallel.guc.wq_status = &pdesc->wq_status;

for_each_child(ce, child)for (child = ({ const __typeof( ((__typeof(*child) *)0)->parallel
.child_link ) *__mptr = ((&(ce)->parallel.child_list)->
next); (__typeof(*child) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*child), parallel.child_link) );}); &child->
parallel.child_link != (&(ce)->parallel.child_list); child
 = ({ const __typeof( ((__typeof(*child) *)0)->parallel.child_link
 ) *__mptr = (child->parallel.child_link.next); (__typeof(
*child) *)( (char *)__mptr - __builtin_offsetof(__typeof(*child
), parallel.child_link) );})) {
	desc = __get_lrc_desc_v69(guc, child->guc_id.id);

	desc->engine_class =
		engine_class_to_guc_class(engine->class);
	desc->hw_context_desc = child->lrc.lrca;
	desc->priority = ce->guc_state.prio;
	desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD(1UL << (0));
	guc_context_policy_init_v69(engine, desc);
}

clear_children_join_go_memory(ce);
}
2573}

2575static void prepare_context_registration_info_v70(struct intel_context *ce,
				  struct guc_ctxt_registration_info *info)
2577{
struct intel_engine_cs *engine = ce->engine;
struct intel_guc *guc = &engine->gt->uc.guc;
u32 ctx_id = ce->guc_id.id;

GEM_BUG_ON(!engine->mask)((void)0);

/*
* Ensure LRC + CT vmas are is same region as write barrier is done
* based on CT vma region.
*/
GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=((void)0)
   i915_gem_object_is_lmem(ce->ring->vma->obj))((void)0);

memset(info, 0, sizeof(*info))__builtin_memset((info), (0), (sizeof(*info)));
info->context_idx = ctx_id;
info->engine_class = engine_class_to_guc_class(engine->class);
info->engine_submit_mask = engine->logical_mask;
/*
* NB: GuC interface supports 64 bit LRCA even though i915/HW
* only supports 32 bit currently.
*/
info->hwlrca_lo = lower_32_bits(ce->lrc.lrca)((u32)(ce->lrc.lrca));
info->hwlrca_hi = upper_32_bits(ce->lrc.lrca)((u32)(((ce->lrc.lrca) >> 16) >> 16));
if (engine->flags & I915_ENGINE_HAS_EU_PRIORITY(1UL << (10)))
info->hwlrca_lo |= map_guc_prio_to_lrc_desc_prio(ce->guc_state.prio);
info->flags = CONTEXT_REGISTRATION_FLAG_KMD(1UL << (0));

/*
* If context is a parent, we need to register a process descriptor
* describing a work queue and register all child contexts.
*/
if (intel_context_is_parent(ce)) {
struct guc_sched_wq_desc *wq_desc;
u64 wq_desc_offset, wq_base_offset;

ce->parallel.guc.wqi_tail = 0;
ce->parallel.guc.wqi_head = 0;

wq_desc_offset = i915_ggtt_offset(ce->state) +
		 __get_parent_scratch_offset(ce);
wq_base_offset = i915_ggtt_offset(ce->state) +
		 __get_wq_offset(ce);
info->wq_desc_lo = lower_32_bits(wq_desc_offset)((u32)(wq_desc_offset));
info->wq_desc_hi = upper_32_bits(wq_desc_offset)((u32)(((wq_desc_offset) >> 16) >> 16));
info->wq_base_lo = lower_32_bits(wq_base_offset)((u32)(wq_base_offset));
info->wq_base_hi = upper_32_bits(wq_base_offset)((u32)(((wq_base_offset) >> 16) >> 16));
info->wq_size = WQ_SIZE((1 << 12) / 2);

wq_desc = __get_wq_desc_v70(ce);
memset(wq_desc, 0, sizeof(*wq_desc))__builtin_memset((wq_desc), (0), (sizeof(*wq_desc)));
wq_desc->wq_status = WQ_STATUS_ACTIVE1;

ce->parallel.guc.wq_head = &wq_desc->head;
ce->parallel.guc.wq_tail = &wq_desc->tail;
ce->parallel.guc.wq_status = &wq_desc->wq_status;

clear_children_join_go_memory(ce);
}
2636}

2638static int try_context_registration(struct intel_context *ce, bool_Bool loop)
2639{
struct intel_engine_cs *engine = ce->engine;
struct intel_runtime_pm *runtime_pm = engine->uncore->rpm;
struct intel_guc *guc = &engine->gt->uc.guc;
intel_wakeref_t wakeref;
u32 ctx_id = ce->guc_id.id;
bool_Bool context_registered;
int ret = 0;

GEM_BUG_ON(!sched_state_is_init(ce))((void)0);

context_registered = ctx_id_mapped(guc, ctx_id);

clr_ctx_id_mapping(guc, ctx_id);
set_ctx_id_mapping(guc, ctx_id, ce);

/*
* The context_lookup xarray is used to determine if the hardware
* context is currently registered. There are two cases in which it
* could be registered either the guc_id has been stolen from another
* context or the lrc descriptor address of this context has changed. In
* either case the context needs to be deregistered with the GuC before
* registering this context.
*/
if (context_registered) {
bool_Bool disabled;
unsigned long flags;

trace_intel_context_steal_guc_id(ce);
GEM_BUG_ON(!loop)((void)0);

/* Seal race with Reset */
spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
disabled = submission_disabled(guc);
if (likely(!disabled)__builtin_expect(!!(!disabled), 1)) {
	set_context_wait_for_deregister_to_register(ce);
	intel_context_get(ce);
}
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
if (unlikely(disabled)__builtin_expect(!!(disabled), 0)) {
	clr_ctx_id_mapping(guc, ctx_id);
	return 0;	/* Will get registered later */
}

/*
 * If stealing the guc_id, this ce has the same guc_id as the
 * context whose guc_id was stolen.
 */
with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
	ret = deregister_context(ce, ce->guc_id.id);
if (unlikely(ret == -ENODEV)__builtin_expect(!!(ret == -19), 0))
	ret = 0;	/* Will get registered later */
} else {
with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
	ret = register_context(ce, loop);
if (unlikely(ret == -EBUSY)__builtin_expect(!!(ret == -16), 0)) {
	clr_ctx_id_mapping(guc, ctx_id);
} else if (unlikely(ret == -ENODEV)__builtin_expect(!!(ret == -19), 0)) {
	clr_ctx_id_mapping(guc, ctx_id);
	ret = 0;	/* Will get registered later */
}
}

return ret;
2703}

2705static int __guc_context_pre_pin(struct intel_context *ce,
		 struct intel_engine_cs *engine,
		 struct i915_gem_ww_ctx *ww,
		 void **vaddr)
2709{
return lrc_pre_pin(ce, engine, ww, vaddr);
2711}

2713static int __guc_context_pin(struct intel_context *ce,
	     struct intel_engine_cs *engine,
	     void *vaddr)
2716{
if (i915_ggtt_offset(ce->state) !=
   (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK(((~0UL) >> (64 - (31) - 1)) & ((~0UL) << (12
)))))
set_bit(CONTEXT_LRCA_DIRTY9, &ce->flags);

/*
* GuC context gets pinned in guc_request_alloc. See that function for
* explaination of why.
*/

return lrc_pin(ce, engine, vaddr);
2727}

2729static int guc_context_pre_pin(struct intel_context *ce,
	       struct i915_gem_ww_ctx *ww,
	       void **vaddr)
2732{
return __guc_context_pre_pin(ce, ce->engine, ww, vaddr);
2734}

2736static int guc_context_pin(struct intel_context *ce, void *vaddr)
2737{
int ret = __guc_context_pin(ce, ce->engine, vaddr);

if (likely(!ret && !intel_context_is_barrier(ce))__builtin_expect(!!(!ret && !intel_context_is_barrier
(ce)), 1))
intel_engine_pm_get(ce->engine);

return ret;
2744}

2746static void guc_context_unpin(struct intel_context *ce)
2747{
struct intel_guc *guc = ce_to_guc(ce);

unpin_guc_id(guc, ce);
lrc_unpin(ce);

if (likely(!intel_context_is_barrier(ce))__builtin_expect(!!(!intel_context_is_barrier(ce)), 1))
intel_engine_pm_put_async(ce->engine);
2755}

2757static void guc_context_post_unpin(struct intel_context *ce)
2758{
lrc_post_unpin(ce);
2760}

2762static void __guc_context_sched_enable(struct intel_guc *guc,
		       struct intel_context *ce)
2764{
u32 action[] = {
INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
ce->guc_id.id,
GUC_CONTEXT_ENABLE1
};

trace_intel_context_sched_enable(ce);

guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])),
		      G2H_LEN_DW_SCHED_CONTEXT_MODE_SET2, true1);
2775}

2777static void __guc_context_sched_disable(struct intel_guc *guc,
			struct intel_context *ce,
			u16 guc_id)
2780{
u32 action[] = {
INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
guc_id,	/* ce->guc_id.id not stable */
GUC_CONTEXT_DISABLE0
};

GEM_BUG_ON(guc_id == GUC_INVALID_CONTEXT_ID)((void)0);

GEM_BUG_ON(intel_context_is_child(ce))((void)0);
trace_intel_context_sched_disable(ce);

guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])),
		      G2H_LEN_DW_SCHED_CONTEXT_MODE_SET2, true1);
2794}

2796static void guc_blocked_fence_complete(struct intel_context *ce)
2797{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

if (!i915_sw_fence_done(&ce->guc_state.blocked))
i915_sw_fence_complete(&ce->guc_state.blocked);
2802}

2804static void guc_blocked_fence_reinit(struct intel_context *ce)
2805{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_state.blocked))((void)0);

/*
* This fence is always complete unless a pending schedule disable is
* outstanding. We arm the fence here and complete it when we receive
* the pending schedule disable complete message.
*/
i915_sw_fence_fini(&ce->guc_state.blocked);
i915_sw_fence_reinit(&ce->guc_state.blocked);
i915_sw_fence_await(&ce->guc_state.blocked);
i915_sw_fence_commit(&ce->guc_state.blocked);
2818}

2820static u16 prep_context_pending_disable(struct intel_context *ce)
2821{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

set_context_pending_disable(ce);
clr_context_enabled(ce);
guc_blocked_fence_reinit(ce);
intel_context_get(ce);

return ce->guc_id.id;
2830}

2832static struct i915_sw_fence *guc_context_block(struct intel_context *ce)
2833{
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
intel_wakeref_t wakeref;
u16 guc_id;
bool_Bool enabled;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);

incr_context_blocked(ce);

enabled = context_enabled(ce);
if (unlikely(!enabled || submission_disabled(guc))__builtin_expect(!!(!enabled || submission_disabled(guc)), 0)) {
if (enabled)
	clr_context_enabled(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
return &ce->guc_state.blocked;
}

/*
* We add +2 here as the schedule disable complete CTB handler calls
* intel_context_sched_disable_unpin (-2 to pin_count).
*/
atomic_add(2, &ce->pin_count)__sync_fetch_and_add(&ce->pin_count, 2);

guc_id = prep_context_pending_disable(ce);

spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
__guc_context_sched_disable(guc, ce, guc_id);

return &ce->guc_state.blocked;
2869}

2871#define SCHED_STATE_MULTI_BLOCKED_MASK((0xfff << 8) & ~(1UL << (8))) \
(SCHED_STATE_BLOCKED_MASK(0xfff << 8) & ~SCHED_STATE_BLOCKED(1UL << (8)))
2873#define SCHED_STATE_NO_UNBLOCK(((0xfff << 8) & ~(1UL << (8))) | (1UL <<
 (2)) | (1UL << (3))) \
(SCHED_STATE_MULTI_BLOCKED_MASK((0xfff << 8) & ~(1UL << (8))) | \
SCHED_STATE_PENDING_DISABLE(1UL << (2)) | \
SCHED_STATE_BANNED(1UL << (3)))

2878static bool_Bool context_cant_unblock(struct intel_context *ce)
2879{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK(((0xfff << 8) & ~(1UL << (8))) | (1UL <<
 (2)) | (1UL << (3)))) ||
context_guc_id_invalid(ce) ||
!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id) ||
!intel_context_is_pinned(ce);
2886}

2888static void guc_context_unblock(struct intel_context *ce)
2889{
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
intel_wakeref_t wakeref;
bool_Bool enable;

GEM_BUG_ON(context_enabled(ce))((void)0);
GEM_BUG_ON(intel_context_is_child(ce))((void)0);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);

if (unlikely(submission_disabled(guc) ||__builtin_expect(!!(submission_disabled(guc) || context_cant_unblock
(ce)), 0)
     context_cant_unblock(ce))__builtin_expect(!!(submission_disabled(guc) || context_cant_unblock
(ce)), 0)) {
enable = false0;
} else {
enable = true1;
set_context_pending_enable(ce);
set_context_enabled(ce);
intel_context_get(ce);
}

decr_context_blocked(ce);

spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

if (enable) {
with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
	__guc_context_sched_enable(guc, ce);
}
2919}

2921static void guc_context_cancel_request(struct intel_context *ce,
		       struct i915_request *rq)
2923{
struct intel_context *block_context =
request_to_scheduling_context(rq);

if (i915_sw_fence_signaled(&rq->submit)) {
struct i915_sw_fence *fence;

intel_context_get(ce);
fence = guc_context_block(block_context);
i915_sw_fence_wait(fence);
if (!i915_request_completed(rq)) {
	__i915_request_skip(rq);
	guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head),
			true1);
}

guc_context_unblock(block_context);
intel_context_put(ce);
}
2942}

2944static void __guc_context_set_preemption_timeout(struct intel_guc *guc,
				 u16 guc_id,
				 u32 preemption_timeout)
2947{
if (GET_UC_VER(guc)((((guc)->fw.file_selected.patch_ver) | (((guc)->fw.file_selected
.minor_ver) << 8) | (((guc)->fw.file_selected.major_ver
) << 16))) >= MAKE_UC_VER(70, 0, 0)((0) | ((0) << 8) | ((70) << 16))) {
struct context_policy policy;

__guc_context_policy_start_klv(&policy, guc_id);
__guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
__guc_context_set_context_policies(guc, &policy, true1);
} else {
u32 action[] = {
	INTEL_GUC_ACTION_V69_SET_CONTEXT_PREEMPTION_TIMEOUT,
	guc_id,
	preemption_timeout
};

intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])), 0, true1);
}
2963}

2965static void
2966guc_context_revoke(struct intel_context *ce, struct i915_request *rq,
   unsigned int preempt_timeout_ms)
2968{
struct intel_guc *guc = ce_to_guc(ce);
struct intel_runtime_pm *runtime_pm =
&ce->engine->gt->i915->runtime_pm;
intel_wakeref_t wakeref;
unsigned long flags;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

guc_flush_submissions(guc);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
set_context_banned(ce);

if (submission_disabled(guc) ||
   (!context_enabled(ce) && !context_pending_disable(ce))) {
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

guc_cancel_context_requests(ce);
intel_engine_signal_breadcrumbs(ce->engine);
} else if (!context_pending_disable(ce)) {
u16 guc_id;

/*
 * We add +2 here as the schedule disable complete CTB handler
 * calls intel_context_sched_disable_unpin (-2 to pin_count).
 */
atomic_add(2, &ce->pin_count)__sync_fetch_and_add(&ce->pin_count, 2);

guc_id = prep_context_pending_disable(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

/*
 * In addition to disabling scheduling, set the preemption
 * timeout to the minimum value (1 us) so the banned context
 * gets kicked off the HW ASAP.
 */
with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
) {
	__guc_context_set_preemption_timeout(guc, guc_id,
					     preempt_timeout_ms);
	__guc_context_sched_disable(guc, ce, guc_id);
}
} else {
if (!context_guc_id_invalid(ce))
	with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
		__guc_context_set_preemption_timeout(guc,
						     ce->guc_id.id,
						     preempt_timeout_ms);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
}
3018}

3020static void guc_context_sched_disable(struct intel_context *ce)
3021{
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;
intel_wakeref_t wakeref;
u16 guc_id;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);

/*
* We have to check if the context has been disabled by another thread,
* check if submssion has been disabled to seal a race with reset and
* finally check if any more requests have been committed to the
* context ensursing that a request doesn't slip through the
* 'context_pending_disable' fence.
*/
if (unlikely(!context_enabled(ce) || submission_disabled(guc) ||__builtin_expect(!!(!context_enabled(ce) || submission_disabled
(guc) || context_has_committed_requests(ce)), 0)
     context_has_committed_requests(ce))__builtin_expect(!!(!context_enabled(ce) || submission_disabled
(guc) || context_has_committed_requests(ce)), 0)) {
clr_context_enabled(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
goto unpin;
}
guc_id = prep_context_pending_disable(ce);

spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
__guc_context_sched_disable(guc, ce, guc_id);

return;
3053unpin:
intel_context_sched_disable_unpin(ce);
3055}

3057static inline void guc_lrc_desc_unpin(struct intel_context *ce)
3058{
struct intel_guc *guc = ce_to_guc(ce);
struct intel_gt *gt = guc_to_gt(guc);
unsigned long flags;
bool_Bool disabled;

GEM_BUG_ON(!intel_gt_pm_is_awake(gt))((void)0);
GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id))((void)0);
GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id))((void)0);
GEM_BUG_ON(context_enabled(ce))((void)0);

/* Seal race with Reset */
spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
disabled = submission_disabled(guc);
if (likely(!disabled)__builtin_expect(!!(!disabled), 1)) {
__intel_gt_pm_get(gt);
set_context_destroyed(ce);
clr_context_registered(ce);
}
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
if (unlikely(disabled)__builtin_expect(!!(disabled), 0)) {
release_guc_id(guc, ce);
__guc_context_destroy(ce);
return;
}

deregister_context(ce, ce->guc_id.id);
3085}

3087static void __guc_context_destroy(struct intel_context *ce)
3088{
GEM_BUG_ON(ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] ||((void)0)
   ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_HIGH] ||((void)0)
   ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] ||((void)0)
   ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_NORMAL])((void)0);
GEM_BUG_ON(ce->guc_state.number_committed_requests)((void)0);

lrc_fini(ce);
intel_context_fini(ce);

if (intel_engine_is_virtual(ce->engine)) {
struct guc_virtual_engine *ve =
	container_of(ce, typeof(*ve), context)({ const __typeof( ((typeof(*ve) *)0)->context ) *__mptr =
 (ce); (typeof(*ve) *)( (char *)__mptr - __builtin_offsetof(typeof
(*ve), context) );});

if (ve->base.breadcrumbs)
	intel_breadcrumbs_put(ve->base.breadcrumbs);

kfree(ve);
} else {
intel_context_free(ce);
}
3109}

3111static void guc_flush_destroyed_contexts(struct intel_guc *guc)
3112{
struct intel_context *ce;
unsigned long flags;

GEM_BUG_ON(!submission_disabled(guc) &&((void)0)
   guc_submission_initialized(guc))((void)0);

while (!list_empty(&guc->submission_state.destroyed_contexts)) {
spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,(list_empty(&guc->submission_state.destroyed_contexts)
 ? ((void *)0) : ({ const __typeof( ((struct intel_context *)
0)->destroyed_link ) *__mptr = ((&guc->submission_state
.destroyed_contexts)->next); (struct intel_context *)( (char
 *)__mptr - __builtin_offsetof(struct intel_context, destroyed_link
) );}))
			      struct intel_context,(list_empty(&guc->submission_state.destroyed_contexts)
 ? ((void *)0) : ({ const __typeof( ((struct intel_context *)
0)->destroyed_link ) *__mptr = ((&guc->submission_state
.destroyed_contexts)->next); (struct intel_context *)( (char
 *)__mptr - __builtin_offsetof(struct intel_context, destroyed_link
) );}))
			      destroyed_link)(list_empty(&guc->submission_state.destroyed_contexts)
 ? ((void *)0) : ({ const __typeof( ((struct intel_context *)
0)->destroyed_link ) *__mptr = ((&guc->submission_state
.destroyed_contexts)->next); (struct intel_context *)( (char
 *)__mptr - __builtin_offsetof(struct intel_context, destroyed_link
) );}));
if (ce)
	list_del_init(&ce->destroyed_link);
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);

if (!ce)
	break;

release_guc_id(guc, ce);
__guc_context_destroy(ce);
}
3134}

3136static void deregister_destroyed_contexts(struct intel_guc *guc)
3137{
struct intel_context *ce;
unsigned long flags;

while (!list_empty(&guc->submission_state.destroyed_contexts)) {
spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,(list_empty(&guc->submission_state.destroyed_contexts)
 ? ((void *)0) : ({ const __typeof( ((struct intel_context *)
0)->destroyed_link ) *__mptr = ((&guc->submission_state
.destroyed_contexts)->next); (struct intel_context *)( (char
 *)__mptr - __builtin_offsetof(struct intel_context, destroyed_link
) );}))
			      struct intel_context,(list_empty(&guc->submission_state.destroyed_contexts)
 ? ((void *)0) : ({ const __typeof( ((struct intel_context *)
0)->destroyed_link ) *__mptr = ((&guc->submission_state
.destroyed_contexts)->next); (struct intel_context *)( (char
 *)__mptr - __builtin_offsetof(struct intel_context, destroyed_link
) );}))
			      destroyed_link)(list_empty(&guc->submission_state.destroyed_contexts)
 ? ((void *)0) : ({ const __typeof( ((struct intel_context *)
0)->destroyed_link ) *__mptr = ((&guc->submission_state
.destroyed_contexts)->next); (struct intel_context *)( (char
 *)__mptr - __builtin_offsetof(struct intel_context, destroyed_link
) );}));
if (ce)
	list_del_init(&ce->destroyed_link);
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);

if (!ce)
	break;

guc_lrc_desc_unpin(ce);
}
3155}

3157static void destroyed_worker_func(struct work_struct *w)
3158{
struct intel_guc *guc = container_of(w, struct intel_guc,({ const __typeof( ((struct intel_guc *)0)->submission_state
.destroyed_worker ) *__mptr = (w); (struct intel_guc *)( (char
 *)__mptr - __builtin_offsetof(struct intel_guc, submission_state
.destroyed_worker) );})
			     submission_state.destroyed_worker)({ const __typeof( ((struct intel_guc *)0)->submission_state
.destroyed_worker ) *__mptr = (w); (struct intel_guc *)( (char
 *)__mptr - __builtin_offsetof(struct intel_guc, submission_state
.destroyed_worker) );});
struct intel_gt *gt = guc_to_gt(guc);
int tmp;

with_intel_gt_pm(gt, tmp)for (tmp = 1, intel_gt_pm_get(gt); tmp; intel_gt_pm_put(gt), tmp
 = 0)
deregister_destroyed_contexts(guc);
3166}

3168static void guc_context_destroy(struct kref *kref)
3169{
struct intel_context *ce = container_of(kref, typeof(*ce), ref)({ const __typeof( ((typeof(*ce) *)0)->ref ) *__mptr = (kref
); (typeof(*ce) *)( (char *)__mptr - __builtin_offsetof(typeof
(*ce), ref) );});
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
bool_Bool destroy;

/*
* If the guc_id is invalid this context has been stolen and we can free
* it immediately. Also can be freed immediately if the context is not
* registered with the GuC or the GuC is in the middle of a reset.
*/
spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
destroy = submission_disabled(guc) || context_guc_id_invalid(ce) ||
!ctx_id_mapped(guc, ce->guc_id.id);
if (likely(!destroy)__builtin_expect(!!(!destroy), 1)) {
if (!list_empty(&ce->guc_id.link))
	list_del_init(&ce->guc_id.link);
list_add_tail(&ce->destroyed_link,
	      &guc->submission_state.destroyed_contexts);
} else {
__release_guc_id(guc, ce);
}
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);
if (unlikely(destroy)__builtin_expect(!!(destroy), 0)) {
__guc_context_destroy(ce);
return;
}

/*
* We use a worker to issue the H2G to deregister the context as we can
* take the GT PM for the first time which isn't allowed from an atomic
* context.
*/
queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker);
3203}

3205static int guc_context_alloc(struct intel_context *ce)
3206{
return lrc_alloc(ce, ce->engine);
3208}

3210static void __guc_context_set_prio(struct intel_guc *guc,
		   struct intel_context *ce)
3212{
if (GET_UC_VER(guc)((((guc)->fw.file_selected.patch_ver) | (((guc)->fw.file_selected
.minor_ver) << 8) | (((guc)->fw.file_selected.major_ver
) << 16))) >= MAKE_UC_VER(70, 0, 0)((0) | ((0) << 8) | ((70) << 16))) {
struct context_policy policy;

__guc_context_policy_start_klv(&policy, ce->guc_id.id);
__guc_context_policy_add_priority(&policy, ce->guc_state.prio);
__guc_context_set_context_policies(guc, &policy, true1);
} else {
u32 action[] = {
	INTEL_GUC_ACTION_V69_SET_CONTEXT_PRIORITY,
	ce->guc_id.id,
	ce->guc_state.prio,
};

guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action)(sizeof((action)) / sizeof((action)[0])), 0, true1);
}
3228}

3230static void guc_context_set_prio(struct intel_guc *guc,
		 struct intel_context *ce,
		 u8 prio)
3233{
GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH ||((void)0)
   prio > GUC_CLIENT_PRIORITY_NORMAL)((void)0);
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

if (ce->guc_state.prio == prio || submission_disabled(guc) ||
   !context_registered(ce)) {
ce->guc_state.prio = prio;
return;
}

ce->guc_state.prio = prio;
__guc_context_set_prio(guc, ce);

trace_intel_context_set_prio(ce);
3248}

3250static inline u8 map_i915_prio_to_guc_prio(int prio)
3251{
if (prio == I915_PRIORITY_NORMAL)
return GUC_CLIENT_PRIORITY_KMD_NORMAL2;
else if (prio < I915_PRIORITY_NORMAL)
return GUC_CLIENT_PRIORITY_NORMAL3;
else if (prio < I915_PRIORITY_DISPLAY)
return GUC_CLIENT_PRIORITY_HIGH1;
else
return GUC_CLIENT_PRIORITY_KMD_HIGH0;
3260}

3262static inline void add_context_inflight_prio(struct intel_context *ce,
			     u8 guc_prio)
3264{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count))((void)0);

++ce->guc_state.prio_count[guc_prio];

/* Overflow protection */
GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio])({ __builtin_expect(!!(!!(!ce->guc_state.prio_count[guc_prio
])), 0); });
3272}

3274static inline void sub_context_inflight_prio(struct intel_context *ce,
			     u8 guc_prio)
3276{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);
GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count))((void)0);

/* Underflow protection */
GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio])({ __builtin_expect(!!(!!(!ce->guc_state.prio_count[guc_prio
])), 0); });

--ce->guc_state.prio_count[guc_prio];
3284}

3286static inline void update_context_prio(struct intel_context *ce)
3287{
struct intel_guc *guc = &ce->engine->gt->uc.guc;
int i;

BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0)extern char _ctassert[(!(0 != 0)) ? 1 : -1 ] __attribute__((__unused__
));
BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL)extern char _ctassert[(!(0 > 3)) ? 1 : -1 ] __attribute__(
(__unused__));

lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

for (i = 0; i < ARRAY_SIZE(ce->guc_state.prio_count)(sizeof((ce->guc_state.prio_count)) / sizeof((ce->guc_state
.prio_count)[0])); ++i) {
if (ce->guc_state.prio_count[i]) {
	guc_context_set_prio(guc, ce, i);
	break;
}
}
3302}

3304static inline bool_Bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio)
3305{
/* Lower value is higher priority */
return new_guc_prio < old_guc_prio;
3308}

3310static void add_to_context(struct i915_request *rq)
3311{
struct intel_context *ce = request_to_scheduling_context(rq);
u8 new_guc_prio = map_i915_prio_to_guc_prio(rq_prio(rq));

GEM_BUG_ON(intel_context_is_child(ce))((void)0);
GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI)((void)0);

spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
list_move_tail(&rq->sched.link, &ce->guc_state.requests);

if (rq->guc_prio == GUC_PRIO_INIT0xff) {
rq->guc_prio = new_guc_prio;
add_context_inflight_prio(ce, rq->guc_prio);
} else if (new_guc_prio_higher(rq->guc_prio, new_guc_prio)) {
sub_context_inflight_prio(ce, rq->guc_prio);
rq->guc_prio = new_guc_prio;
add_context_inflight_prio(ce, rq->guc_prio);
}
update_context_prio(ce);

spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);
3332}

3334static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce)
3335{
lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

if (rq->guc_prio != GUC_PRIO_INIT0xff &&
   rq->guc_prio != GUC_PRIO_FINI0xfe) {
sub_context_inflight_prio(ce, rq->guc_prio);
update_context_prio(ce);
}
rq->guc_prio = GUC_PRIO_FINI0xfe;
3344}

3346static void remove_from_context(struct i915_request *rq)
3347{
struct intel_context *ce = request_to_scheduling_context(rq);

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

spin_lock_irq(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);

list_del_init(&rq->sched.link);
clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);

/* Prevent further __await_execution() registering a cb, then flush */
set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);

guc_prio_fini(rq, ce);

decr_context_committed_requests(ce);

spin_unlock_irq(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);

atomic_dec(&ce->guc_id.ref)__sync_fetch_and_sub(&ce->guc_id.ref, 1);
i915_request_notify_execute_cb_imm(rq);
3368}

3370static const struct intel_context_ops guc_context_ops = {
.alloc = guc_context_alloc,

.pre_pin = guc_context_pre_pin,
.pin = guc_context_pin,
.unpin = guc_context_unpin,
.post_unpin = guc_context_post_unpin,

.revoke = guc_context_revoke,

.cancel_request = guc_context_cancel_request,

.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,

.sched_disable = guc_context_sched_disable,

.reset = lrc_reset,
.destroy = guc_context_destroy,

.create_virtual = guc_create_virtual,
.create_parallel = guc_create_parallel,
3392};

3394static void submit_work_cb(struct irq_work *wrk)
3395{
struct i915_request *rq = container_of(wrk, typeof(*rq), submit_work)({ const __typeof( ((typeof(*rq) *)0)->submit_work ) *__mptr
 = (wrk); (typeof(*rq) *)( (char *)__mptr - __builtin_offsetof
(typeof(*rq), submit_work) );});

might_lock(&rq->engine->sched_engine->lock);
i915_sw_fence_complete(&rq->submit);
3400}

3402static void __guc_signal_context_fence(struct intel_context *ce)
3403{
struct i915_request *rq, *rn;

lockdep_assert_held(&ce->guc_state.lock)do { (void)(&ce->guc_state.lock); } while(0);

if (!list_empty(&ce->guc_state.fences))
trace_intel_context_fence_release(ce);

/*
* Use an IRQ to ensure locking order of sched_engine->lock ->
* ce->guc_state.lock is preserved.
*/
list_for_each_entry_safe(rq, rn, &ce->guc_state.fences,for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->guc_fence_link
 ) *__mptr = ((&ce->guc_state.fences)->next); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), guc_fence_link
) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->guc_fence_link
 ) *__mptr = (rq->guc_fence_link.next); (__typeof(*rq) *)(
 (char *)__mptr - __builtin_offsetof(__typeof(*rq), guc_fence_link
) );}); &rq->guc_fence_link != (&ce->guc_state.
fences); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *)0
)->guc_fence_link ) *__mptr = (rn->guc_fence_link.next)
; (__typeof(*rn) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*rn), guc_fence_link) );}))
		 guc_fence_link)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->guc_fence_link
 ) *__mptr = ((&ce->guc_state.fences)->next); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), guc_fence_link
) );}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->guc_fence_link
 ) *__mptr = (rq->guc_fence_link.next); (__typeof(*rq) *)(
 (char *)__mptr - __builtin_offsetof(__typeof(*rq), guc_fence_link
) );}); &rq->guc_fence_link != (&ce->guc_state.
fences); rq = rn, rn = ({ const __typeof( ((__typeof(*rn) *)0
)->guc_fence_link ) *__mptr = (rn->guc_fence_link.next)
; (__typeof(*rn) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*rn), guc_fence_link) );})) {
list_del(&rq->guc_fence_link);
irq_work_queue(&rq->submit_work);
}

INIT_LIST_HEAD(&ce->guc_state.fences);
3422}

3424static void guc_signal_context_fence(struct intel_context *ce)
3425{
unsigned long flags;

GEM_BUG_ON(intel_context_is_child(ce))((void)0);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
clr_context_wait_for_deregister_to_register(ce);
__guc_signal_context_fence(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
3434}

3436static bool_Bool context_needs_register(struct intel_context *ce, bool_Bool new_guc_id)
3437{
return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY9, &ce->flags) ||
!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id)) &&
!submission_disabled(ce_to_guc(ce));
3441}

3443static void guc_context_init(struct intel_context *ce)
3444{
const struct i915_gem_context *ctx;
int prio = I915_CONTEXT_DEFAULT_PRIORITY0;

rcu_read_lock();
ctx = rcu_dereference(ce->gem_context)(ce->gem_context);
if (ctx)
prio = ctx->sched.priority;
rcu_read_unlock();

ce->guc_state.prio = map_i915_prio_to_guc_prio(prio);
set_bit(CONTEXT_GUC_INIT10, &ce->flags);
3456}

3458static int guc_request_alloc(struct i915_request *rq)
3459{
struct intel_context *ce = request_to_scheduling_context(rq);
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
int ret;

GEM_BUG_ON(!intel_context_is_pinned(rq->context))((void)0);

/*
* Flush enough space to reduce the likelihood of waiting after
* we start building the request - in which case we will just
* have to repeat work.
*/
rq->reserved_space += GUC_REQUEST_SIZE64;

/*
* Note that after this point, we have committed to using
* this request as it is being used to both track the
* state of engine initialisation and liveness of the
* golden renderstate above. Think twice before you try
* to cancel/unwind this request now.
*/

/* Unconditionally invalidate GPU caches and TLBs. */
ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE(1UL << (0)));
if (ret)
return ret;

rq->reserved_space -= GUC_REQUEST_SIZE64;

if (unlikely(!test_bit(CONTEXT_GUC_INIT, &ce->flags))__builtin_expect(!!(!test_bit(10, &ce->flags)), 0))
guc_context_init(ce);

/*
* Call pin_guc_id here rather than in the pinning step as with
* dma_resv, contexts can be repeatedly pinned / unpinned trashing the
* guc_id and creating horrible race conditions. This is especially bad
* when guc_id are being stolen due to over subscription. By the time
* this function is reached, it is guaranteed that the guc_id will be
* persistent until the generated request is retired. Thus, sealing these
* race conditions. It is still safe to fail here if guc_id are
* exhausted and return -EAGAIN to the user indicating that they can try
* again in the future.
*
* There is no need for a lock here as the timeline mutex ensures at
* most one context can be executing this code path at once. The
* guc_id_ref is incremented once for every request in flight and
* decremented on each retire. When it is zero, a lock around the
* increment (in pin_guc_id) is needed to seal a race with unpin_guc_id.
*/
if (atomic_add_unless(&ce->guc_id.ref, 1, 0))
goto out;

ret = pin_guc_id(guc, ce);	/* returns 1 if new guc_id assigned */
if (unlikely(ret < 0)__builtin_expect(!!(ret < 0), 0))
return ret;
if (context_needs_register(ce, !!ret)) {
ret = try_context_registration(ce, true1);
if (unlikely(ret)__builtin_expect(!!(ret), 0)) {	/* unwind */
	if (ret == -EPIPE32) {
		disable_submission(guc);
		goto out;	/* GPU will be reset */
	}
	atomic_dec(&ce->guc_id.ref)__sync_fetch_and_sub(&ce->guc_id.ref, 1);
	unpin_guc_id(guc, ce);
	return ret;
}
}

clear_bit(CONTEXT_LRCA_DIRTY9, &ce->flags);

3530out:
/*
* We block all requests on this context if a G2H is pending for a
* schedule disable or context deregistration as the GuC will fail a
* schedule enable or context registration if either G2H is pending
* respectfully. Once a G2H returns, the fence is released that is
* blocking these requests (see guc_signal_context_fence).
*/
spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
if (context_wait_for_deregister_to_register(ce) ||
   context_pending_disable(ce)) {
init_irq_work(&rq->submit_work, submit_work_cb);
i915_sw_fence_await(&rq->submit);

list_add_tail(&rq->guc_fence_link, &ce->guc_state.fences);
}
incr_context_committed_requests(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

return 0;
3550}

3552static int guc_virtual_context_pre_pin(struct intel_context *ce,
		       struct i915_gem_ww_ctx *ww,
		       void **vaddr)
3555{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);

return __guc_context_pre_pin(ce, engine, ww, vaddr);
3559}

3561static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr)
3562{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
int ret = __guc_context_pin(ce, engine, vaddr);
intel_engine_mask_t tmp, mask = ce->engine->mask;

if (likely(!ret)__builtin_expect(!!(!ret), 1))
for_each_engine_masked(engine, ce->engine->gt, mask, tmp)for ((tmp) = (mask) & (ce->engine->gt)->info.engine_mask
; (tmp) ? ((engine) = (ce->engine->gt)->engine[({ int
 __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx
; })]), 1 : 0;)
	intel_engine_pm_get(engine);

return ret;
3572}

3574static void guc_virtual_context_unpin(struct intel_context *ce)
3575{
intel_engine_mask_t tmp, mask = ce->engine->mask;
struct intel_engine_cs *engine;
struct intel_guc *guc = ce_to_guc(ce);

GEM_BUG_ON(context_enabled(ce))((void)0);
GEM_BUG_ON(intel_context_is_barrier(ce))((void)0);

unpin_guc_id(guc, ce);
lrc_unpin(ce);

for_each_engine_masked(engine, ce->engine->gt, mask, tmp)for ((tmp) = (mask) & (ce->engine->gt)->info.engine_mask
; (tmp) ? ((engine) = (ce->engine->gt)->engine[({ int
 __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx
; })]), 1 : 0;)
intel_engine_pm_put_async(engine);
3588}

3590static void guc_virtual_context_enter(struct intel_context *ce)
3591{
intel_engine_mask_t tmp, mask = ce->engine->mask;
struct intel_engine_cs *engine;

for_each_engine_masked(engine, ce->engine->gt, mask, tmp)for ((tmp) = (mask) & (ce->engine->gt)->info.engine_mask
; (tmp) ? ((engine) = (ce->engine->gt)->engine[({ int
 __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx
; })]), 1 : 0;)
intel_engine_pm_get(engine);

intel_timeline_enter(ce->timeline);
3599}

3601static void guc_virtual_context_exit(struct intel_context *ce)
3602{
intel_engine_mask_t tmp, mask = ce->engine->mask;
struct intel_engine_cs *engine;

for_each_engine_masked(engine, ce->engine->gt, mask, tmp)for ((tmp) = (mask) & (ce->engine->gt)->info.engine_mask
; (tmp) ? ((engine) = (ce->engine->gt)->engine[({ int
 __idx = ffs(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx
; })]), 1 : 0;)
intel_engine_pm_put(engine);

intel_timeline_exit(ce->timeline);
3610}

3612static int guc_virtual_context_alloc(struct intel_context *ce)
3613{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);

return lrc_alloc(ce, engine);
3617}

3619static const struct intel_context_ops virtual_guc_context_ops = {
.alloc = guc_virtual_context_alloc,

.pre_pin = guc_virtual_context_pre_pin,
.pin = guc_virtual_context_pin,
.unpin = guc_virtual_context_unpin,
.post_unpin = guc_context_post_unpin,

.revoke = guc_context_revoke,

.cancel_request = guc_context_cancel_request,

.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,

.sched_disable = guc_context_sched_disable,

.destroy = guc_context_destroy,

.get_sibling = guc_virtual_get_sibling,
3639};

3641static int guc_parent_context_pin(struct intel_context *ce, void *vaddr)
3642{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
struct intel_guc *guc = ce_to_guc(ce);
int ret;

GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine))((void)0);

ret = pin_guc_id(guc, ce);
if (unlikely(ret < 0)__builtin_expect(!!(ret < 0), 0))
return ret;

return __guc_context_pin(ce, engine, vaddr);
3655}

3657static int guc_child_context_pin(struct intel_context *ce, void *vaddr)
3658{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);

GEM_BUG_ON(!intel_context_is_child(ce))((void)0);
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine))((void)0);

__intel_context_pin(ce->parallel.parent);
return __guc_context_pin(ce, engine, vaddr);
3666}

3668static void guc_parent_context_unpin(struct intel_context *ce)
3669{
struct intel_guc *guc = ce_to_guc(ce);

GEM_BUG_ON(context_enabled(ce))((void)0);
GEM_BUG_ON(intel_context_is_barrier(ce))((void)0);
GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine))((void)0);

unpin_guc_id(guc, ce);
lrc_unpin(ce);
3679}

3681static void guc_child_context_unpin(struct intel_context *ce)
3682{
GEM_BUG_ON(context_enabled(ce))((void)0);
GEM_BUG_ON(intel_context_is_barrier(ce))((void)0);
GEM_BUG_ON(!intel_context_is_child(ce))((void)0);
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine))((void)0);

lrc_unpin(ce);
3689}

3691static void guc_child_context_post_unpin(struct intel_context *ce)
3692{
GEM_BUG_ON(!intel_context_is_child(ce))((void)0);
GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent))((void)0);
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine))((void)0);

lrc_post_unpin(ce);
intel_context_unpin(ce->parallel.parent);
3699}

3701static void guc_child_context_destroy(struct kref *kref)
3702{
struct intel_context *ce = container_of(kref, typeof(*ce), ref)({ const __typeof( ((typeof(*ce) *)0)->ref ) *__mptr = (kref
); (typeof(*ce) *)( (char *)__mptr - __builtin_offsetof(typeof
(*ce), ref) );});

__guc_context_destroy(ce);
3706}

3708static const struct intel_context_ops virtual_parent_context_ops = {
.alloc = guc_virtual_context_alloc,

.pre_pin = guc_context_pre_pin,
.pin = guc_parent_context_pin,
.unpin = guc_parent_context_unpin,
.post_unpin = guc_context_post_unpin,

.revoke = guc_context_revoke,

.cancel_request = guc_context_cancel_request,

.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,

.sched_disable = guc_context_sched_disable,

.destroy = guc_context_destroy,

.get_sibling = guc_virtual_get_sibling,
3728};

3730static const struct intel_context_ops virtual_child_context_ops = {
.alloc = guc_virtual_context_alloc,

.pre_pin = guc_context_pre_pin,
.pin = guc_child_context_pin,
.unpin = guc_child_context_unpin,
.post_unpin = guc_child_context_post_unpin,

.cancel_request = guc_context_cancel_request,

.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,

.destroy = guc_child_context_destroy,

.get_sibling = guc_virtual_get_sibling,
3746};

3748/*
* The below override of the breadcrumbs is enabled when the user configures a
* context for parallel submission (multi-lrc, parent-child).
*
* The overridden breadcrumbs implements an algorithm which allows the GuC to
* safely preempt all the hw contexts configured for parallel submission
* between each BB. The contract between the i915 and GuC is if the parent
* context can be preempted, all the children can be preempted, and the GuC will
* always try to preempt the parent before the children. A handshake between the
* parent / children breadcrumbs ensures the i915 holds up its end of the deal
* creating a window to preempt between each set of BBs.
*/
3760static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
				     u64 offset, u32 len,
				     const unsigned int flags);
3763static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
				    u64 offset, u32 len,
				    const unsigned int flags);
3766static u32 *
3767emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
				 u32 *cs);
3769static u32 *
3770emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
				u32 *cs);

3773static struct intel_context *
3774guc_create_parallel(struct intel_engine_cs **engines,
    unsigned int num_siblings,
    unsigned int width)
3777{
struct intel_engine_cs **siblings = NULL((void *)0);
struct intel_context *parent = NULL((void *)0), *ce, *err;
1
'parent' initialized to a null pointer value→
int i, j;

siblings = kmalloc_array(num_siblings,
2
←
Calling 'kmalloc_array'→
5
←
Returning from 'kmalloc_array'→
		 sizeof(*siblings),
		 GFP_KERNEL(0x0001 | 0x0004));
if (!siblings)
6
←
Assuming 'siblings' is non-null→
7
←
Taking false branch→
return ERR_PTR(-ENOMEM12);

for (i = 0; i < width; ++i) {
8
←
Assuming 'i' is >= 'width'→
9
←
Loop condition is false. Execution continues on line 3808→
for (j = 0; j < num_siblings; ++j)
	siblings[j] = engines[i * num_siblings + j];

ce = intel_engine_create_virtual(siblings, num_siblings,
				 FORCE_VIRTUAL(1UL << (0)));
if (IS_ERR(ce)) {
	err = ERR_CAST(ce);
	goto unwind;
}

if (i == 0) {
	parent = ce;
	parent->ops = &virtual_parent_context_ops;
} else {
	ce->ops = &virtual_child_context_ops;
	intel_context_bind_parent_child(parent, ce);
}
}

parent->parallel.fence_context = dma_fence_context_alloc(1);
10
←
Dereference of null pointer

parent->engine->emit_bb_start =
emit_bb_start_parent_no_preempt_mid_batch;
parent->engine->emit_fini_breadcrumb =
emit_fini_breadcrumb_parent_no_preempt_mid_batch;
parent->engine->emit_fini_breadcrumb_dw =
12 + 4 * parent->parallel.number_children;
for_each_child(parent, ce)for (ce = ({ const __typeof( ((__typeof(*ce) *)0)->parallel
.child_link ) *__mptr = ((&(parent)->parallel.child_list
)->next); (__typeof(*ce) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*ce), parallel.child_link) );}); &ce->parallel
.child_link != (&(parent)->parallel.child_list); ce = (
{ const __typeof( ((__typeof(*ce) *)0)->parallel.child_link
 ) *__mptr = (ce->parallel.child_link.next); (__typeof(*ce
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*ce), parallel
.child_link) );})) {
ce->engine->emit_bb_start =
	emit_bb_start_child_no_preempt_mid_batch;
ce->engine->emit_fini_breadcrumb =
	emit_fini_breadcrumb_child_no_preempt_mid_batch;
ce->engine->emit_fini_breadcrumb_dw = 16;
}

kfree(siblings);
return parent;

3827unwind:
if (parent)
intel_context_put(parent);
kfree(siblings);
return err;
3832}

3834static bool_Bool
3835guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b)
3836{
struct intel_engine_cs *sibling;
intel_engine_mask_t tmp, mask = b->engine_mask;
bool_Bool result = false0;

for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)for ((tmp) = (mask) & (b->irq_engine->gt)->info.
engine_mask; (tmp) ? ((sibling) = (b->irq_engine->gt)->
engine[({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL <<
 (__idx)); __idx; })]), 1 : 0;)
result |= intel_engine_irq_enable(sibling);

return result;
3845}

3847static void
3848guc_irq_disable_breadcrumbs(struct intel_breadcrumbs *b)
3849{
struct intel_engine_cs *sibling;
intel_engine_mask_t tmp, mask = b->engine_mask;

for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)for ((tmp) = (mask) & (b->irq_engine->gt)->info.
engine_mask; (tmp) ? ((sibling) = (b->irq_engine->gt)->
engine[({ int __idx = ffs(tmp) - 1; tmp &= ~(1UL <<
 (__idx)); __idx; })]), 1 : 0;)
intel_engine_irq_disable(sibling);
3855}

3857static void guc_init_breadcrumbs(struct intel_engine_cs *engine)
3858{
int i;

/*
* In GuC submission mode we do not know which physical engine a request
* will be scheduled on, this creates a problem because the breadcrumb
* interrupt is per physical engine. To work around this we attach
* requests and direct all breadcrumb interrupts to the first instance
* of an engine per class. In addition all breadcrumb interrupts are
* enabled / disabled across an engine class in unison.
*/
for (i = 0; i < MAX_ENGINE_INSTANCE8; ++i) {
struct intel_engine_cs *sibling =
	engine->gt->engine_class[engine->class][i];

if (sibling) {
	if (engine->breadcrumbs != sibling->breadcrumbs) {
		intel_breadcrumbs_put(engine->breadcrumbs);
		engine->breadcrumbs =
			intel_breadcrumbs_get(sibling->breadcrumbs);
	}
	break;
}
}

if (engine->breadcrumbs) {
engine->breadcrumbs->engine_mask |= engine->mask;
engine->breadcrumbs->irq_enable = guc_irq_enable_breadcrumbs;
engine->breadcrumbs->irq_disable = guc_irq_disable_breadcrumbs;
}
3888}

3890static void guc_bump_inflight_request_prio(struct i915_request *rq,
			   int prio)
3892{
struct intel_context *ce = request_to_scheduling_context(rq);
u8 new_guc_prio = map_i915_prio_to_guc_prio(prio);

/* Short circuit function */
if (prio < I915_PRIORITY_NORMAL ||
   rq->guc_prio == GUC_PRIO_FINI0xfe ||
   (rq->guc_prio != GUC_PRIO_INIT0xff &&
    !new_guc_prio_higher(rq->guc_prio, new_guc_prio)))
return;

spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
if (rq->guc_prio != GUC_PRIO_FINI0xfe) {
if (rq->guc_prio != GUC_PRIO_INIT0xff)
	sub_context_inflight_prio(ce, rq->guc_prio);
rq->guc_prio = new_guc_prio;
add_context_inflight_prio(ce, rq->guc_prio);
update_context_prio(ce);
}
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);
3912}

3914static void guc_retire_inflight_request_prio(struct i915_request *rq)
3915{
struct intel_context *ce = request_to_scheduling_context(rq);

spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
guc_prio_fini(rq, ce);
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);
3921}

3923static void sanitize_hwsp(struct intel_engine_cs *engine)
3924{
struct intel_timeline *tl;

list_for_each_entry(tl, &engine->status_page.timelines, engine_link)for (tl = ({ const __typeof( ((__typeof(*tl) *)0)->engine_link
 ) *__mptr = ((&engine->status_page.timelines)->next
); (__typeof(*tl) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*tl), engine_link) );}); &tl->engine_link != (&engine
->status_page.timelines); tl = ({ const __typeof( ((__typeof
(*tl) *)0)->engine_link ) *__mptr = (tl->engine_link.next
); (__typeof(*tl) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*tl), engine_link) );}))
intel_timeline_reset_seqno(tl);
3929}

3931static void guc_sanitize(struct intel_engine_cs *engine)
3932{
/*
* Poison residual state on resume, in case the suspend didn't!
*
* We have to assume that across suspend/resume (or other loss
* of control) that the contents of our pinned buffers has been
* lost, replaced by garbage. Since this doesn't always happen,
* let's poison such state so that we more quickly spot when
* we falsely assume it has been preserved.
*/
if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)0)
memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE)__builtin_memset((engine->status_page.addr), (0xdb), ((1 <<
 12)));

/*
* The kernel_context HWSP is stored in the status_page. As above,
* that may be lost on resume/initialisation, and so we need to
* reset the value in the HWSP.
*/
sanitize_hwsp(engine);

/* And scrub the dirty cachelines for the HWSP */
drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE(1 << 12));

intel_engine_reset_pinned_contexts(engine);
3956}

3958static void setup_hwsp(struct intel_engine_cs *engine)
3959{
intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */

ENGINE_WRITE_FW(engine,__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x80) }), (i915_ggtt_offset
(engine->status_page.vma)))
	RING_HWS_PGA,__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x80) }), (i915_ggtt_offset
(engine->status_page.vma)))
	i915_ggtt_offset(engine->status_page.vma))__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x80) }), (i915_ggtt_offset
(engine->status_page.vma)));
3965}

3967static void start_engine(struct intel_engine_cs *engine)
3968{
ENGINE_WRITE_FW(engine,__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x29c) }), (({ typeof((
<< 3)) _a = ((1 << 3)); ({ if (__builtin_constant_p
(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while
 (0); if (__builtin_constant_p(_a) && __builtin_constant_p
(_a)) do { } while (0); ((_a) << 16 | (_a)); }); })))
	RING_MODE_GEN7,__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x29c) }), (({ typeof((
<< 3)) _a = ((1 << 3)); ({ if (__builtin_constant_p
(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while
 (0); if (__builtin_constant_p(_a) && __builtin_constant_p
(_a)) do { } while (0); ((_a) << 16 | (_a)); }); })))
	_MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE))__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x29c) }), (({ typeof((
<< 3)) _a = ((1 << 3)); ({ if (__builtin_constant_p
(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while
 (0); if (__builtin_constant_p(_a) && __builtin_constant_p
(_a)) do { } while (0); ((_a) << 16 | (_a)); }); })));

ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING))__raw_uncore_write32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x9c) }), ((({ if (__builtin_constant_p
((((u32)((1UL << (8)) + 0))))) do { } while (0); if (__builtin_constant_p
(0)) do { } while (0); if (__builtin_constant_p((((u32)((1UL <<
 (8)) + 0)))) && __builtin_constant_p(0)) do { } while
 (0); (((((u32)((1UL << (8)) + 0)))) << 16 | (0))
; }))));
ENGINE_POSTING_READ(engine, RING_MI_MODE)((void)__raw_uncore_read32(((engine))->uncore, ((const i915_reg_t
){ .reg = (((engine)->mmio_base) + 0x9c) })));
3975}

3977static int guc_resume(struct intel_engine_cs *engine)
3978{
assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);

intel_mocs_init_engine(engine);

intel_breadcrumbs_reset(engine->breadcrumbs);

setup_hwsp(engine);
start_engine(engine);

if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE(1UL << (11)))
xehp_enable_ccs_engines(engine);

return 0;
3992}

3994static bool_Bool guc_sched_engine_disabled(struct i915_sched_engine *sched_engine)
3995{
return !sched_engine->tasklet.callback;
3997}

3999static void guc_set_default_submission(struct intel_engine_cs *engine)
4000{
engine->submit_request = guc_submit_request;
4002}

4004static inline void guc_kernel_context_pin(struct intel_guc *guc,
			  struct intel_context *ce)
4006{
/*
* Note: we purposefully do not check the returns below because
* the registration can only fail if a reset is just starting.
* This is called at the end of reset so presumably another reset
* isn't happening and even it did this code would be run again.
*/

if (context_guc_id_invalid(ce))
pin_guc_id(guc, ce);

try_context_registration(ce, true1);
4018}

4020static inline void guc_init_lrc_mapping(struct intel_guc *guc)
4021{
struct intel_gt *gt = guc_to_gt(guc);
struct intel_engine_cs *engine;
enum intel_engine_id id;

/* make sure all descriptors are clean... */
xa_destroy(&guc->context_lookup);

/*
* A reset might have occurred while we had a pending stalled request,
* so make sure we clean that up.
*/
guc->stalled_request = NULL((void *)0);
guc->submission_stall_reason = STALL_NONE;

/*
* Some contexts might have been pinned before we enabled GuC
* submission, so we need to add them to the GuC bookeeping.
* Also, after a reset the of the GuC we want to make sure that the
* information shared with GuC is properly reset. The kernel LRCs are
* not attached to the gem_context, so they need to be added separately.
*/
for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
struct intel_context *ce;

list_for_each_entry(ce, &engine->pinned_contexts_list,for (ce = ({ const __typeof( ((__typeof(*ce) *)0)->pinned_contexts_link
 ) *__mptr = ((&engine->pinned_contexts_list)->next
); (__typeof(*ce) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*ce), pinned_contexts_link) );}); &ce->pinned_contexts_link
 != (&engine->pinned_contexts_list); ce = ({ const __typeof
( ((__typeof(*ce) *)0)->pinned_contexts_link ) *__mptr = (
ce->pinned_contexts_link.next); (__typeof(*ce) *)( (char *
)__mptr - __builtin_offsetof(__typeof(*ce), pinned_contexts_link
) );}))
		    pinned_contexts_link)for (ce = ({ const __typeof( ((__typeof(*ce) *)0)->pinned_contexts_link
 ) *__mptr = ((&engine->pinned_contexts_list)->next
); (__typeof(*ce) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*ce), pinned_contexts_link) );}); &ce->pinned_contexts_link
 != (&engine->pinned_contexts_list); ce = ({ const __typeof
( ((__typeof(*ce) *)0)->pinned_contexts_link ) *__mptr = (
ce->pinned_contexts_link.next); (__typeof(*ce) *)( (char *
)__mptr - __builtin_offsetof(__typeof(*ce), pinned_contexts_link
) );}))
	guc_kernel_context_pin(guc, ce);
}
4050}

4052static void guc_release(struct intel_engine_cs *engine)
4053{
engine->sanitize = NULL((void *)0); /* no longer in control, nothing to sanitize */

intel_engine_cleanup_common(engine);
lrc_fini_wa_ctx(engine);
4058}

4060static void virtual_guc_bump_serial(struct intel_engine_cs *engine)
4061{
struct intel_engine_cs *e;
intel_engine_mask_t tmp, mask = engine->mask;

for_each_engine_masked(e, engine->gt, mask, tmp)for ((tmp) = (mask) & (engine->gt)->info.engine_mask
; (tmp) ? ((e) = (engine->gt)->engine[({ int __idx = ffs
(tmp) - 1; tmp &= ~(1UL << (__idx)); __idx; })]), 1
 : 0;)
e->serial++;
4067}

4069static void guc_default_vfuncs(struct intel_engine_cs *engine)
4070{
/* Default vfuncs which can be overridden by each engine. */

engine->resume = guc_resume;

engine->cops = &guc_context_ops;
engine->request_alloc = guc_request_alloc;
engine->add_active_request = add_to_context;
engine->remove_active_request = remove_from_context;

engine->sched_engine->schedule = i915_schedule;

engine->reset.prepare = guc_engine_reset_prepare;
engine->reset.rewind = guc_rewind_nop;
engine->reset.cancel = guc_reset_nop;
engine->reset.finish = guc_reset_nop;

engine->emit_flush = gen8_emit_flush_xcs;
engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
if (GRAPHICS_VER(engine->i915)((&(engine->i915)->__runtime)->graphics.ip.ver) >= 12) {
engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
engine->emit_flush = gen12_emit_flush_xcs;
}
engine->set_default_submission = guc_set_default_submission;
engine->busyness = guc_engine_busyness;

engine->flags |= I915_ENGINE_SUPPORTS_STATS(1UL << (1));
engine->flags |= I915_ENGINE_HAS_PREEMPTION(1UL << (2));
engine->flags |= I915_ENGINE_HAS_TIMESLICES(1UL << (4));

/* Wa_14014475959:dg2 */
if (IS_DG2(engine->i915)IS_PLATFORM(engine->i915, INTEL_DG2) && engine->class == COMPUTE_CLASS5)
engine->flags |= I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT(1UL << (12));

/*
* TODO: GuC supports timeslicing and semaphores as well, but they're
* handled by the firmware so some minor tweaks are required before
* enabling.
*
* engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
*/

engine->emit_bb_start = gen8_emit_bb_start;
if (GRAPHICS_VER_FULL(engine->i915)(((&(engine->i915)->__runtime)->graphics.ip.ver)
 << 8 | ((&(engine->i915)->__runtime)->graphics
.ip.rel)) >= IP_VER(12, 50)((12) << 8 | (50)))
engine->emit_bb_start = gen125_emit_bb_start;
4116}

4118static void rcs_submission_override(struct intel_engine_cs *engine)
4119{
switch (GRAPHICS_VER(engine->i915)((&(engine->i915)->__runtime)->graphics.ip.ver)) {
case 12:
engine->emit_flush = gen12_emit_flush_rcs;
engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
break;
case 11:
engine->emit_flush = gen11_emit_flush_rcs;
engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
break;
default:
engine->emit_flush = gen8_emit_flush_rcs;
engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
break;
}
4134}

4136static inline void guc_default_irqs(struct intel_engine_cs *engine)
4137{
engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT(1 << 0);
intel_engine_set_irq_handler(engine, cs_irq_handler);
4140}

4142static void guc_sched_engine_destroy(struct kref *kref)
4143{
struct i915_sched_engine *sched_engine =
container_of(kref, typeof(*sched_engine), ref)({ const __typeof( ((typeof(*sched_engine) *)0)->ref ) *__mptr
 = (kref); (typeof(*sched_engine) *)( (char *)__mptr - __builtin_offsetof
(typeof(*sched_engine), ref) );});
struct intel_guc *guc = sched_engine->private_data;

guc->sched_engine = NULL((void *)0);
tasklet_kill(&sched_engine->tasklet); /* flush the callback */
kfree(sched_engine);
4151}

4153int intel_guc_submission_setup(struct intel_engine_cs *engine)
4154{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;
struct intel_guc *guc = &engine->gt->uc.guc;

/*
* The setup relies on several assumptions (e.g. irqs always enabled)
* that are only valid on gen11+
*/
GEM_BUG_ON(GRAPHICS_VER(i915) < 11)((void)0);

if (!guc->sched_engine) {
guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL2);
if (!guc->sched_engine)
	return -ENOMEM12;

guc->sched_engine->schedule = i915_schedule;
guc->sched_engine->disabled = guc_sched_engine_disabled;
guc->sched_engine->private_data = guc;
guc->sched_engine->destroy = guc_sched_engine_destroy;
guc->sched_engine->bump_inflight_request_prio =
	guc_bump_inflight_request_prio;
guc->sched_engine->retire_inflight_request_prio =
	guc_retire_inflight_request_prio;
tasklet_setup(&guc->sched_engine->tasklet,
	      guc_submission_tasklet);
}
i915_sched_engine_put(engine->sched_engine);
engine->sched_engine = i915_sched_engine_get(guc->sched_engine);

guc_default_vfuncs(engine);
guc_default_irqs(engine);
guc_init_breadcrumbs(engine);

if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE(1UL << (9)))
rcs_submission_override(engine);

lrc_init_wa_ctx(engine);

/* Finally, take ownership and responsibility for cleanup! */
engine->sanitize = guc_sanitize;
engine->release = guc_release;

return 0;
4197}

4199void intel_guc_submission_enable(struct intel_guc *guc)
4200{
struct intel_gt *gt = guc_to_gt(guc);

/* Enable and route to GuC */
if (GRAPHICS_VER(gt->i915)((&(gt->i915)->__runtime)->graphics.ip.ver) >= 12)
intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES((const i915_reg_t){ .reg = (0xc71c) }),
		   GUC_SEM_INTR_ROUTE_TO_GUC(1UL << (31)) |
		   GUC_SEM_INTR_ENABLE_ALL(0xff));

guc_init_lrc_mapping(guc);
guc_init_engine_stats(guc);
4211}

4213void intel_guc_submission_disable(struct intel_guc *guc)
4214{
struct intel_gt *gt = guc_to_gt(guc);

/* Note: By the time we're here, GuC may have already been reset */

/* Disable and route to host */
if (GRAPHICS_VER(gt->i915)((&(gt->i915)->__runtime)->graphics.ip.ver) >= 12)
intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES((const i915_reg_t){ .reg = (0xc71c) }), 0x0);
4222}

4224static bool_Bool __guc_submission_supported(struct intel_guc *guc)
4225{
/* GuC submission is unavailable for pre-Gen11 */
return intel_guc_is_supported(guc) &&
      GRAPHICS_VER(guc_to_gt(guc)->i915)((&(guc_to_gt(guc)->i915)->__runtime)->graphics.
ip.ver) >= 11;
4229}

4231static bool_Bool __guc_submission_selected(struct intel_guc *guc)
4232{
struct drm_i915_privateinteldrm_softc *i915 = guc_to_gt(guc)->i915;

if (!intel_guc_submission_is_supported(guc))
return false0;

return i915->params.enable_guc & ENABLE_GUC_SUBMISSION(1UL << (0));
4239}

4241void intel_guc_submission_init_early(struct intel_guc *guc)
4242{
xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ4);

mtx_init(&guc->submission_state.lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&guc->
submission_state.lock), ((((0x9)) > 0x0 && ((0x9))
 < 0x9) ? 0x9 : ((0x9)))); } while (0);
INIT_LIST_HEAD(&guc->submission_state.guc_id_list);
ida_init(&guc->submission_state.guc_ids);
INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts);
INIT_WORK(&guc->submission_state.destroyed_worker,
  destroyed_worker_func);
INIT_WORK(&guc->submission_state.reset_fail_worker,
  reset_fail_worker_func);

mtx_init(&guc->timestamp.lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&guc->
timestamp.lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9
) ? 0x9 : ((0x9)))); } while (0);
INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping);

guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID65535;
guc->submission_supported = __guc_submission_supported(guc);
guc->submission_selected = __guc_submission_selected(guc);
4260}

4262static inline struct intel_context *
4263g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)
4264{
struct intel_context *ce;

if (unlikely(ctx_id >= GUC_MAX_CONTEXT_ID)__builtin_expect(!!(ctx_id >= 65535), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx_id
)
	"Invalid ctx_id %u\n", ctx_id)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx_id
);
return NULL((void *)0);
}

ce = __get_context(guc, ctx_id);
if (unlikely(!ce)__builtin_expect(!!(!ce), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Context is NULL, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx_id
)
	"Context is NULL, ctx_id %u\n", ctx_id)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Context is NULL, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx_id
);
return NULL((void *)0);
}

if (unlikely(intel_context_is_child(ce))__builtin_expect(!!(intel_context_is_child(ce)), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Context is child, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx_id
)
	"Context is child, ctx_id %u\n", ctx_id)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Context is child, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ctx_id
);
return NULL((void *)0);
}

return ce;
4287}

4289int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
			  const u32 *msg,
			  u32 len)
4292{
struct intel_context *ce;
u32 ctx_id;

if (unlikely(len < 1)__builtin_expect(!!(len < 1), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u\n", len)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid length %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , len);
return -EPROTO95;
}
ctx_id = msg[0];

ce = g2h_context_lookup(guc, ctx_id);
if (unlikely(!ce)__builtin_expect(!!(!ce), 0))
return -EPROTO95;

trace_intel_context_deregister_done(ce);

4308#ifdef CONFIG_DRM_I915_SELFTEST
if (unlikely(ce->drop_deregister)__builtin_expect(!!(ce->drop_deregister), 0)) {
ce->drop_deregister = false0;
return 0;
}
4313#endif

if (context_wait_for_deregister_to_register(ce)) {
struct intel_runtime_pm *runtime_pm =
	&ce->engine->gt->i915->runtime_pm;
intel_wakeref_t wakeref;

/*
 * Previous owner of this guc_id has been deregistered, now safe
 * register this context.
 */
with_intel_runtime_pm(runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(runtime_pm); (wakeref);
 intel_runtime_pm_put((runtime_pm), (wakeref)), (wakeref) = 0
)
	register_context(ce, true1);
guc_signal_context_fence(ce);
intel_context_put(ce);
} else if (context_destroyed(ce)) {
/* Context has been destroyed */
intel_gt_pm_put_async(guc_to_gt(guc));
release_guc_id(guc, ce);
__guc_context_destroy(ce);
}

decr_outstanding_submission_g2h(guc);

return 0;
4338}

4340int intel_guc_sched_done_process_msg(struct intel_guc *guc,
		     const u32 *msg,
		     u32 len)
4343{
struct intel_context *ce;
unsigned long flags;
u32 ctx_id;

if (unlikely(len < 2)__builtin_expect(!!(len < 2), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u\n", len)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid length %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , len);
return -EPROTO95;
}
ctx_id = msg[0];

ce = g2h_context_lookup(guc, ctx_id);
if (unlikely(!ce)__builtin_expect(!!(!ce), 0))
return -EPROTO95;

if (unlikely(context_destroyed(ce) ||__builtin_expect(!!(context_destroyed(ce) || (!context_pending_enable
(ce) && !context_pending_disable(ce))), 0)
     (!context_pending_enable(ce) &&__builtin_expect(!!(context_destroyed(ce) || (!context_pending_enable
(ce) && !context_pending_disable(ce))), 0)
     !context_pending_disable(ce)))__builtin_expect(!!(context_destroyed(ce) || (!context_pending_enable
(ce) && !context_pending_disable(ce))), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Bad context sched_state 0x%x, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ce->
guc_state.sched_state, ctx_id)
	"Bad context sched_state 0x%x, ctx_id %u\n",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Bad context sched_state 0x%x, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ce->
guc_state.sched_state, ctx_id)
	ce->guc_state.sched_state, ctx_id)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Bad context sched_state 0x%x, ctx_id %u\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , ce->
guc_state.sched_state, ctx_id);
return -EPROTO95;
}

trace_intel_context_sched_done(ce);

if (context_pending_enable(ce)) {
4370#ifdef CONFIG_DRM_I915_SELFTEST
if (unlikely(ce->drop_schedule_enable)__builtin_expect(!!(ce->drop_schedule_enable), 0)) {
	ce->drop_schedule_enable = false0;
	return 0;
}
4375#endif

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
clr_context_pending_enable(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);
} else if (context_pending_disable(ce)) {
bool_Bool banned;

4383#ifdef CONFIG_DRM_I915_SELFTEST
if (unlikely(ce->drop_schedule_disable)__builtin_expect(!!(ce->drop_schedule_disable), 0)) {
	ce->drop_schedule_disable = false0;
	return 0;
}
4388#endif

/*
 * Unpin must be done before __guc_signal_context_fence,
 * otherwise a race exists between the requests getting
 * submitted + retired before this unpin completes resulting in
 * the pin_count going to zero and the context still being
 * enabled.
 */
intel_context_sched_disable_unpin(ce);

spin_lock_irqsave(&ce->guc_state.lock, flags)do { flags = 0; mtx_enter(&ce->guc_state.lock); } while
 (0);
banned = context_banned(ce);
clr_context_banned(ce);
clr_context_pending_disable(ce);
__guc_signal_context_fence(ce);
guc_blocked_fence_complete(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags)do { (void)(flags); mtx_leave(&ce->guc_state.lock); } while
 (0);

if (banned) {
	guc_cancel_context_requests(ce);
	intel_engine_signal_breadcrumbs(ce->engine);
}
}

decr_outstanding_submission_g2h(guc);
intel_context_put(ce);

return 0;
4417}

4419static void capture_error_state(struct intel_guc *guc,
		struct intel_context *ce)
4421{
struct intel_gt *gt = guc_to_gt(guc);
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;
struct intel_engine_cs *engine = __context_to_physical_engine(ce);
intel_wakeref_t wakeref;

intel_engine_set_hung_context(engine, ce);
with_intel_runtime_pm(&i915->runtime_pm, wakeref)for ((wakeref) = intel_runtime_pm_get(&i915->runtime_pm
); (wakeref); intel_runtime_pm_put((&i915->runtime_pm)
, (wakeref)), (wakeref) = 0)
i915_capture_error_state(gt, engine->mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE(1UL << (0)));
atomic_inc(&i915->gpu_error.reset_engine_count[engine->uabi_class])__sync_fetch_and_add(&i915->gpu_error.reset_engine_count
[engine->uabi_class], 1);
4431}

4433static void guc_context_replay(struct intel_context *ce)
4434{
struct i915_sched_engine *sched_engine = ce->engine->sched_engine;

__guc_reset_context(ce, ce->engine->mask);
tasklet_hi_schedule(&sched_engine->tasklet);
4439}

4441static void guc_handle_context_reset(struct intel_guc *guc,
		     struct intel_context *ce)
4443{
trace_intel_context_reset(ce);

if (likely(intel_context_is_schedulable(ce))__builtin_expect(!!(intel_context_is_schedulable(ce)), 1)) {
capture_error_state(guc, ce);
guc_context_replay(ce);
} else {
drm_info(&guc_to_gt(guc)->i915->drm,do { } while(0)
	 "Ignoring context reset notification of exiting context 0x%04X on %s",do { } while(0)
	 ce->guc_id.id, ce->engine->name)do { } while(0);
}
4454}

4456int intel_guc_context_reset_process_msg(struct intel_guc *guc,
			const u32 *msg, u32 len)
4458{
struct intel_context *ce;
unsigned long flags;
int ctx_id;

if (unlikely(len != 1)__builtin_expect(!!(len != 1), 0)) {
drm_err(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid length %u"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , len);
return -EPROTO95;
}

ctx_id = msg[0];

/*
* The context lookup uses the xarray but lookups only require an RCU lock
* not the full spinlock. So take the lock explicitly and keep it until the
* context has been reference count locked to ensure it can't be destroyed
* asynchronously until the reset is done.
*/
xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
ce = g2h_context_lookup(guc, ctx_id);
if (ce)
intel_context_get(ce);
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);

if (unlikely(!ce)__builtin_expect(!!(!ce), 0))
return -EPROTO95;

guc_handle_context_reset(guc, ce);
intel_context_put(ce);

return 0;
4489}

4491int intel_guc_error_capture_process_msg(struct intel_guc *guc,
			const u32 *msg, u32 len)
4493{
u32 status;

if (unlikely(len != 1)__builtin_expect(!!(len != 1), 0)) {
drm_dbg(&guc_to_gt(guc)->i915->drm, "Invalid length %u", len)__drm_dev_dbg(((void *)0), (&guc_to_gt(guc)->i915->
drm) ? (&guc_to_gt(guc)->i915->drm)->dev : ((void
 *)0), DRM_UT_DRIVER, "Invalid length %u", len);
return -EPROTO95;
}

status = msg[0] & INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK0x000000FF;
if (status == INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)
drm_warn(&guc_to_gt(guc)->i915->drm, "G2H-Error capture no space")printf("drm:pid%d:%s *WARNING* " "[drm] " "G2H-Error capture no space"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__);

intel_guc_capture_process(guc);

return 0;
4508}

4510struct intel_engine_cs *
4511intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance)
4512{
struct intel_gt *gt = guc_to_gt(guc);
u8 engine_class = guc_class_to_engine_class(guc_class);

/* Class index is checked in class converter */
GEM_BUG_ON(instance > MAX_ENGINE_INSTANCE)((void)0);

return gt->engine_class[engine_class][instance];
4520}

4522static void reset_fail_worker_func(struct work_struct *w)
4523{
struct intel_guc *guc = container_of(w, struct intel_guc,({ const __typeof( ((struct intel_guc *)0)->submission_state
.reset_fail_worker ) *__mptr = (w); (struct intel_guc *)( (char
 *)__mptr - __builtin_offsetof(struct intel_guc, submission_state
.reset_fail_worker) );})
			     submission_state.reset_fail_worker)({ const __typeof( ((struct intel_guc *)0)->submission_state
.reset_fail_worker ) *__mptr = (w); (struct intel_guc *)( (char
 *)__mptr - __builtin_offsetof(struct intel_guc, submission_state
.reset_fail_worker) );});
struct intel_gt *gt = guc_to_gt(guc);
intel_engine_mask_t reset_fail_mask;
unsigned long flags;

spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
reset_fail_mask = guc->submission_state.reset_fail_mask;
guc->submission_state.reset_fail_mask = 0;
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);

if (likely(reset_fail_mask)__builtin_expect(!!(reset_fail_mask), 1))
intel_gt_handle_error(gt, reset_fail_mask,
		      I915_ERROR_CAPTURE(1UL << (0)),
		      "GuC failed to reset engine mask=0x%x\n",
		      reset_fail_mask);
4540}

4542int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
			 const u32 *msg, u32 len)
4544{
struct intel_engine_cs *engine;
struct intel_gt *gt = guc_to_gt(guc);
u8 guc_class, instance;
u32 reason;
unsigned long flags;

if (unlikely(len != 3)__builtin_expect(!!(len != 3), 0)) {
drm_err(&gt->i915->drm, "Invalid length %u", len)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid length %u"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , len);
return -EPROTO95;
}

guc_class = msg[0];
instance = msg[1];
reason = msg[2];

engine = intel_guc_lookup_engine(guc, guc_class, instance);
if (unlikely(!engine)__builtin_expect(!!(!engine), 0)) {
drm_err(&gt->i915->drm,printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid engine %d:%d"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , guc_class
, instance)
	"Invalid engine %d:%d", guc_class, instance)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "Invalid engine %d:%d"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , guc_class
, instance);
return -EPROTO95;
}

/*
* This is an unexpected failure of a hardware feature. So, log a real
* error message not just the informational that comes with the reset.
*/
drm_err(&gt->i915->drm, "GuC engine reset request failed on %d:%d (%s) because 0x%08X",printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "GuC engine reset request failed on %d:%d (%s) because 0x%08X"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , guc_class
, instance, engine->name, reason)
guc_class, instance, engine->name, reason)printf("drm:pid%d:%s *ERROR* " "[drm] " "*ERROR* " "GuC engine reset request failed on %d:%d (%s) because 0x%08X"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_p->ps_pid, __func__ , guc_class
, instance, engine->name, reason);

spin_lock_irqsave(&guc->submission_state.lock, flags)do { flags = 0; mtx_enter(&guc->submission_state.lock)
; } while (0);
guc->submission_state.reset_fail_mask |= engine->mask;
spin_unlock_irqrestore(&guc->submission_state.lock, flags)do { (void)(flags); mtx_leave(&guc->submission_state.lock
); } while (0);

/*
* A GT reset flushes this worker queue (G2H handler) so we must use
* another worker to trigger a GT reset.
*/
queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker);

return 0;
4585}

4587void intel_guc_find_hung_context(struct intel_engine_cs *engine)
4588{
struct intel_guc *guc = &engine->gt->uc.guc;
struct intel_context *ce;
struct i915_request *rq;
unsigned long index;
unsigned long flags;

/* Reset called during driver load? GuC not yet initialised! */
if (unlikely(!guc_submission_initialized(guc))__builtin_expect(!!(!guc_submission_initialized(guc)), 0))
return;

xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
xa_for_each(&guc->context_lookup, index, ce)for (index = 0; ((ce) = xa_get_next(&guc->context_lookup
, &(index))) != ((void *)0); index++) {
bool_Bool found;

if (!kref_get_unless_zero(&ce->ref))
	continue;

xa_unlock(&guc->context_lookup)do { mtx_leave(&(&guc->context_lookup)->xa_lock
); } while (0);

if (!intel_context_is_pinned(ce))
	goto next;

if (intel_engine_is_virtual(ce->engine)) {
	if (!(ce->engine->mask & engine->mask))
		goto next;
} else {
	if (ce->engine != engine)
		goto next;
}

found = false0;
spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
list_for_each_entry(rq, &ce->guc_state.requests, sched.link)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&ce->guc_state.requests)->next); (__typeof
(*rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &rq->sched.link != (&ce->guc_state
.requests); rq = ({ const __typeof( ((__typeof(*rq) *)0)->
sched.link ) *__mptr = (rq->sched.link.next); (__typeof(*rq
) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );})) {
	if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE)
		continue;

	found = true1;
	break;
}
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);

if (found) {
	intel_engine_set_hung_context(engine, ce);

	/* Can only cope with one hang at a time... */
	intel_context_put(ce);
	xa_lock(&guc->context_lookup)do { mtx_enter(&(&guc->context_lookup)->xa_lock
); } while (0);
	goto done;
}

4639next:
intel_context_put(ce);
xa_lock(&guc->context_lookup)do { mtx_enter(&(&guc->context_lookup)->xa_lock
); } while (0);
}
4643done:
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);
4645}

4647void intel_guc_dump_active_requests(struct intel_engine_cs *engine,
		    struct i915_request *hung_rq,
		    struct drm_printer *m)
4650{
struct intel_guc *guc = &engine->gt->uc.guc;
struct intel_context *ce;
unsigned long index;
unsigned long flags;

/* Reset called during driver load? GuC not yet initialised! */
if (unlikely(!guc_submission_initialized(guc))__builtin_expect(!!(!guc_submission_initialized(guc)), 0))
return;

xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
xa_for_each(&guc->context_lookup, index, ce)for (index = 0; ((ce) = xa_get_next(&guc->context_lookup
, &(index))) != ((void *)0); index++) {
if (!kref_get_unless_zero(&ce->ref))
	continue;

xa_unlock(&guc->context_lookup)do { mtx_leave(&(&guc->context_lookup)->xa_lock
); } while (0);

if (!intel_context_is_pinned(ce))
	goto next;

if (intel_engine_is_virtual(ce->engine)) {
	if (!(ce->engine->mask & engine->mask))
		goto next;
} else {
	if (ce->engine != engine)
		goto next;
}

spin_lock(&ce->guc_state.lock)mtx_enter(&ce->guc_state.lock);
intel_engine_dump_active_requests(&ce->guc_state.requests,
				  hung_rq, m);
spin_unlock(&ce->guc_state.lock)mtx_leave(&ce->guc_state.lock);

4683next:
intel_context_put(ce);
xa_lock(&guc->context_lookup)do { mtx_enter(&(&guc->context_lookup)->xa_lock
); } while (0);
}
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);
4688}

4690void intel_guc_submission_print_info(struct intel_guc *guc,
		     struct drm_printer *p)
4692{
struct i915_sched_engine *sched_engine = guc->sched_engine;
struct rb_node *rb;
unsigned long flags;

if (!sched_engine)
return;

drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n",
   atomic_read(&guc->outstanding_submission_g2h)({ typeof(*(&guc->outstanding_submission_g2h)) __tmp =
 *(volatile typeof(*(&guc->outstanding_submission_g2h)
) *)&(*(&guc->outstanding_submission_g2h)); membar_datadep_consumer
(); __tmp; }));
drm_printf(p, "GuC tasklet count: %u\n\n",
   atomic_read(&sched_engine->tasklet.count)({ typeof(*(&sched_engine->tasklet.count)) __tmp = *(volatile
 typeof(*(&sched_engine->tasklet.count)) *)&(*(&
sched_engine->tasklet.count)); membar_datadep_consumer(); __tmp
; }));

spin_lock_irqsave(&sched_engine->lock, flags)do { flags = 0; mtx_enter(&sched_engine->lock); } while
 (0);
drm_printf(p, "Requests in GuC submit tasklet:\n");
for (rb = rb_first_cached(&sched_engine->queue)linux_root_RB_MINMAX((struct linux_root *)(&(&sched_engine
->queue)->rb_root), -1); rb; rb = rb_next(rb)linux_root_RB_NEXT((rb))) {
struct i915_priolist *pl = to_priolist(rb);
struct i915_request *rq;

priolist_for_each_request(rq, pl)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = ((&(pl)->requests)->next); (__typeof(*
rq) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rq), sched
.link) );}); &rq->sched.link != (&(pl)->requests
); rq = ({ const __typeof( ((__typeof(*rq) *)0)->sched.link
 ) *__mptr = (rq->sched.link.next); (__typeof(*rq) *)( (char
 *)__mptr - __builtin_offsetof(__typeof(*rq), sched.link) );}
))
	drm_printf(p, "guc_id=%u, seqno=%llu\n",
		   rq->context->guc_id.id,
		   rq->fence.seqno);
}
spin_unlock_irqrestore(&sched_engine->lock, flags)do { (void)(flags); mtx_leave(&sched_engine->lock); } while
 (0);
drm_printf(p, "\n");
4718}

4720static inline void guc_log_context_priority(struct drm_printer *p,
			    struct intel_context *ce)
4722{
int i;

drm_printf(p, "\t\tPriority: %d\n", ce->guc_state.prio);
drm_printf(p, "\t\tNumber Requests (lower index == higher priority)\n");
for (i = GUC_CLIENT_PRIORITY_KMD_HIGH0;
    i < GUC_CLIENT_PRIORITY_NUM4; ++i) {
drm_printf(p, "\t\tNumber requests in priority band[%d]: %d\n",
	   i, ce->guc_state.prio_count[i]);
}
drm_printf(p, "\n");
4733}

4735static inline void guc_log_context(struct drm_printer *p,
		   struct intel_context *ce)
4737{
drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id.id);
drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca);
drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n",
   ce->ring->head,
   ce->lrc_reg_state[CTX_RING_HEAD(0x04 + 1)]);
drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n",
   ce->ring->tail,
   ce->lrc_reg_state[CTX_RING_TAIL(0x06 + 1)]);
drm_printf(p, "\t\tContext Pin Count: %u\n",
   atomic_read(&ce->pin_count)({ typeof(*(&ce->pin_count)) __tmp = *(volatile typeof
(*(&ce->pin_count)) *)&(*(&ce->pin_count));
 membar_datadep_consumer(); __tmp; }));
drm_printf(p, "\t\tGuC ID Ref Count: %u\n",
   atomic_read(&ce->guc_id.ref)({ typeof(*(&ce->guc_id.ref)) __tmp = *(volatile typeof
(*(&ce->guc_id.ref)) *)&(*(&ce->guc_id.ref)
); membar_datadep_consumer(); __tmp; }));
drm_printf(p, "\t\tSchedule State: 0x%x\n\n",
   ce->guc_state.sched_state);
4752}

4754void intel_guc_submission_print_context_info(struct intel_guc *guc,
			     struct drm_printer *p)
4756{
struct intel_context *ce;
unsigned long index;
unsigned long flags;

xa_lock_irqsave(&guc->context_lookup, flags)do { flags = 0; mtx_enter(&(&guc->context_lookup)->
xa_lock); } while (0);
xa_for_each(&guc->context_lookup, index, ce)for (index = 0; ((ce) = xa_get_next(&guc->context_lookup
, &(index))) != ((void *)0); index++) {
GEM_BUG_ON(intel_context_is_child(ce))((void)0);

guc_log_context(p, ce);
guc_log_context_priority(p, ce);

if (intel_context_is_parent(ce)) {
	struct intel_context *child;

	drm_printf(p, "\t\tNumber children: %u\n",
		   ce->parallel.number_children);

	if (ce->parallel.guc.wq_status) {
		drm_printf(p, "\t\tWQI Head: %u\n",
			   READ_ONCE(*ce->parallel.guc.wq_head)({ typeof(*ce->parallel.guc.wq_head) __tmp = *(volatile typeof
(*ce->parallel.guc.wq_head) *)&(*ce->parallel.guc.wq_head
); membar_datadep_consumer(); __tmp; }));
		drm_printf(p, "\t\tWQI Tail: %u\n",
			   READ_ONCE(*ce->parallel.guc.wq_tail)({ typeof(*ce->parallel.guc.wq_tail) __tmp = *(volatile typeof
(*ce->parallel.guc.wq_tail) *)&(*ce->parallel.guc.wq_tail
); membar_datadep_consumer(); __tmp; }));
		drm_printf(p, "\t\tWQI Status: %u\n\n",
			   READ_ONCE(*ce->parallel.guc.wq_status)({ typeof(*ce->parallel.guc.wq_status) __tmp = *(volatile typeof
(*ce->parallel.guc.wq_status) *)&(*ce->parallel.guc
.wq_status); membar_datadep_consumer(); __tmp; }));
	}

	if (ce->engine->emit_bb_start ==
	    emit_bb_start_parent_no_preempt_mid_batch) {
		u8 i;

		drm_printf(p, "\t\tChildren Go: %u\n\n",
			   get_children_go_value(ce));
		for (i = 0; i < ce->parallel.number_children; ++i)
			drm_printf(p, "\t\tChildren Join: %u\n",
				   get_children_join_value(ce, i));
	}

	for_each_child(ce, child)for (child = ({ const __typeof( ((__typeof(*child) *)0)->parallel
.child_link ) *__mptr = ((&(ce)->parallel.child_list)->
next); (__typeof(*child) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*child), parallel.child_link) );}); &child->
parallel.child_link != (&(ce)->parallel.child_list); child
 = ({ const __typeof( ((__typeof(*child) *)0)->parallel.child_link
 ) *__mptr = (child->parallel.child_link.next); (__typeof(
*child) *)( (char *)__mptr - __builtin_offsetof(__typeof(*child
), parallel.child_link) );}))
		guc_log_context(p, child);
}
}
xa_unlock_irqrestore(&guc->context_lookup, flags)do { (void)(flags); mtx_leave(&(&guc->context_lookup
)->xa_lock); } while (0);
4799}

4801static inline u32 get_children_go_addr(struct intel_context *ce)
4802{
GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);

return i915_ggtt_offset(ce->state) +
__get_parent_scratch_offset(ce) +
offsetof(struct parent_scratch, go.semaphore)__builtin_offsetof(struct parent_scratch, go.semaphore);
4808}

4810static inline u32 get_children_join_addr(struct intel_context *ce,
			 u8 child_index)
4812{
GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);

return i915_ggtt_offset(ce->state) +
__get_parent_scratch_offset(ce) +
offsetof(struct parent_scratch, join[child_index].semaphore)__builtin_offsetof(struct parent_scratch, join[child_index].semaphore
);
4818}

4820#define PARENT_GO_BB1			1
4821#define PARENT_GO_FINI_BREADCRUMB0	0
4822#define CHILD_GO_BB1			1
4823#define CHILD_GO_FINI_BREADCRUMB0	0
4824static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
				     u64 offset, u32 len,
				     const unsigned int flags)
4827{
struct intel_context *ce = rq->context;
u32 *cs;
u8 i;

GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);

cs = intel_ring_begin(rq, 10 + 4 * ce->parallel.number_children);
if (IS_ERR(cs))
return PTR_ERR(cs);

/* Wait on children */
for (i = 0; i < ce->parallel.number_children; ++i) {
*cs++ = (MI_SEMAPHORE_WAIT(((0x0) << 29) | (0x1c) << 23 | (2)) |
	 MI_SEMAPHORE_GLOBAL_GTT(1<<22) |
	 MI_SEMAPHORE_POLL(1 << 15) |
	 MI_SEMAPHORE_SAD_EQ_SDD(4 << 12));
*cs++ = PARENT_GO_BB1;
*cs++ = get_children_join_addr(ce, i);
*cs++ = 0;
}

/* Turn off preemption */
*cs++ = MI_ARB_ON_OFF(((0x0) << 29) | (0x08) << 23 | (0)) | MI_ARB_DISABLE(0<<0);
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

/* Tell children go */
cs = gen8_emit_ggtt_write(cs,
		  CHILD_GO_BB1,
		  get_children_go_addr(ce),
		  0);

/* Jump to batch */
*cs++ = MI_BATCH_BUFFER_START_GEN8(((0x0) << 29) | (0x31) << 23 | (1)) |
(flags & I915_DISPATCH_SECURE(1UL << (0)) ? 0 : BIT(8)(1UL << (8)));
*cs++ = lower_32_bits(offset)((u32)(offset));
*cs++ = upper_32_bits(offset)((u32)(((offset) >> 16) >> 16));
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

intel_ring_advance(rq, cs);

return 0;
4869}

4871static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
				    u64 offset, u32 len,
				    const unsigned int flags)
4874{
struct intel_context *ce = rq->context;
struct intel_context *parent = intel_context_to_parent(ce);
u32 *cs;

GEM_BUG_ON(!intel_context_is_child(ce))((void)0);

cs = intel_ring_begin(rq, 12);
if (IS_ERR(cs))
return PTR_ERR(cs);

/* Signal parent */
cs = gen8_emit_ggtt_write(cs,
		  PARENT_GO_BB1,
		  get_children_join_addr(parent,
					 ce->parallel.child_index),
		  0);

/* Wait on parent for go */
*cs++ = (MI_SEMAPHORE_WAIT(((0x0) << 29) | (0x1c) << 23 | (2)) |
 MI_SEMAPHORE_GLOBAL_GTT(1<<22) |
 MI_SEMAPHORE_POLL(1 << 15) |
 MI_SEMAPHORE_SAD_EQ_SDD(4 << 12));
*cs++ = CHILD_GO_BB1;
*cs++ = get_children_go_addr(parent);
*cs++ = 0;

/* Turn off preemption */
*cs++ = MI_ARB_ON_OFF(((0x0) << 29) | (0x08) << 23 | (0)) | MI_ARB_DISABLE(0<<0);

/* Jump to batch */
*cs++ = MI_BATCH_BUFFER_START_GEN8(((0x0) << 29) | (0x31) << 23 | (1)) |
(flags & I915_DISPATCH_SECURE(1UL << (0)) ? 0 : BIT(8)(1UL << (8)));
*cs++ = lower_32_bits(offset)((u32)(offset));
*cs++ = upper_32_bits(offset)((u32)(((offset) >> 16) >> 16));

intel_ring_advance(rq, cs);

return 0;
4913}

4915static u32 *
4916__emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
				   u32 *cs)
4918{
struct intel_context *ce = rq->context;
u8 i;

GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);

/* Wait on children */
for (i = 0; i < ce->parallel.number_children; ++i) {
*cs++ = (MI_SEMAPHORE_WAIT(((0x0) << 29) | (0x1c) << 23 | (2)) |
	 MI_SEMAPHORE_GLOBAL_GTT(1<<22) |
	 MI_SEMAPHORE_POLL(1 << 15) |
	 MI_SEMAPHORE_SAD_EQ_SDD(4 << 12));
*cs++ = PARENT_GO_FINI_BREADCRUMB0;
*cs++ = get_children_join_addr(ce, i);
*cs++ = 0;
}

/* Turn on preemption */
*cs++ = MI_ARB_ON_OFF(((0x0) << 29) | (0x08) << 23 | (0)) | MI_ARB_ENABLE(1<<0);
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

/* Tell children go */
cs = gen8_emit_ggtt_write(cs,
		  CHILD_GO_FINI_BREADCRUMB0,
		  get_children_go_addr(ce),
		  0);

return cs;
4946}

4948/*
* If this true, a submission of multi-lrc requests had an error and the
* requests need to be skipped. The front end (execuf IOCTL) should've called
* i915_request_skip which squashes the BB but we still need to emit the fini
* breadrcrumbs seqno write. At this point we don't know how many of the
* requests in the multi-lrc submission were generated so we can't do the
* handshake between the parent and children (e.g. if 4 requests should be
* generated but 2nd hit an error only 1 would be seen by the GuC backend).
* Simply skip the handshake, but still emit the breadcrumbd seqno, if an error
* has occurred on any of the requests in submission / relationship.
*/
4959static inline bool_Bool skip_handshake(struct i915_request *rq)
4960{
return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags);
4962}

4964#define NON_SKIP_LEN	6
4965static u32 *
4966emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
				 u32 *cs)
4968{
struct intel_context *ce = rq->context;
__maybe_unused__attribute__((__unused__)) u32 *before_fini_breadcrumb_user_interrupt_cs;
__maybe_unused__attribute__((__unused__)) u32 *start_fini_breadcrumb_cs = cs;

GEM_BUG_ON(!intel_context_is_parent(ce))((void)0);

if (unlikely(skip_handshake(rq))__builtin_expect(!!(skip_handshake(rq)), 0)) {
/*
 * NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch,
 * the NON_SKIP_LEN comes from the length of the emits below.
 */
memset(cs, 0, sizeof(u32) *__builtin_memset((cs), (0), (sizeof(u32) * (ce->engine->
emit_fini_breadcrumb_dw - NON_SKIP_LEN)))
       (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN))__builtin_memset((cs), (0), (sizeof(u32) * (ce->engine->
emit_fini_breadcrumb_dw - NON_SKIP_LEN)));
cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
} else {
cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs);
}

/* Emit fini breadcrumb */
before_fini_breadcrumb_user_interrupt_cs = cs;
cs = gen8_emit_ggtt_write(cs,
		  rq->fence.seqno,
		  i915_request_active_timeline(rq)->hwsp_offset,
		  0);

/* User interrupt */
*cs++ = MI_USER_INTERRUPT(((0x0) << 29) | (0x02) << 23 | (0));
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

/* Ensure our math for skip + emit is correct */
GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=((void)0)
   cs)((void)0);
GEM_BUG_ON(start_fini_breadcrumb_cs +((void)0)
   ce->engine->emit_fini_breadcrumb_dw != cs)((void)0);

rq->tail = intel_ring_offset(rq, cs);

return cs;
5007}

5009static u32 *
5010__emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
				  u32 *cs)
5012{
struct intel_context *ce = rq->context;
struct intel_context *parent = intel_context_to_parent(ce);

GEM_BUG_ON(!intel_context_is_child(ce))((void)0);

/* Turn on preemption */
*cs++ = MI_ARB_ON_OFF(((0x0) << 29) | (0x08) << 23 | (0)) | MI_ARB_ENABLE(1<<0);
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

/* Signal parent */
cs = gen8_emit_ggtt_write(cs,
		  PARENT_GO_FINI_BREADCRUMB0,
		  get_children_join_addr(parent,
					 ce->parallel.child_index),
		  0);

/* Wait parent on for go */
*cs++ = (MI_SEMAPHORE_WAIT(((0x0) << 29) | (0x1c) << 23 | (2)) |
 MI_SEMAPHORE_GLOBAL_GTT(1<<22) |
 MI_SEMAPHORE_POLL(1 << 15) |
 MI_SEMAPHORE_SAD_EQ_SDD(4 << 12));
*cs++ = CHILD_GO_FINI_BREADCRUMB0;
*cs++ = get_children_go_addr(parent);
*cs++ = 0;

return cs;
5039}

5041static u32 *
5042emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
				u32 *cs)
5044{
struct intel_context *ce = rq->context;
__maybe_unused__attribute__((__unused__)) u32 *before_fini_breadcrumb_user_interrupt_cs;
__maybe_unused__attribute__((__unused__)) u32 *start_fini_breadcrumb_cs = cs;

GEM_BUG_ON(!intel_context_is_child(ce))((void)0);

if (unlikely(skip_handshake(rq))__builtin_expect(!!(skip_handshake(rq)), 0)) {
/*
 * NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch,
 * the NON_SKIP_LEN comes from the length of the emits below.
 */
memset(cs, 0, sizeof(u32) *__builtin_memset((cs), (0), (sizeof(u32) * (ce->engine->
emit_fini_breadcrumb_dw - NON_SKIP_LEN)))
       (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN))__builtin_memset((cs), (0), (sizeof(u32) * (ce->engine->
emit_fini_breadcrumb_dw - NON_SKIP_LEN)));
cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
} else {
cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs);
}

/* Emit fini breadcrumb */
before_fini_breadcrumb_user_interrupt_cs = cs;
cs = gen8_emit_ggtt_write(cs,
		  rq->fence.seqno,
		  i915_request_active_timeline(rq)->hwsp_offset,
		  0);

/* User interrupt */
*cs++ = MI_USER_INTERRUPT(((0x0) << 29) | (0x02) << 23 | (0));
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

/* Ensure our math for skip + emit is correct */
GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=((void)0)
   cs)((void)0);
GEM_BUG_ON(start_fini_breadcrumb_cs +((void)0)
   ce->engine->emit_fini_breadcrumb_dw != cs)((void)0);

rq->tail = intel_ring_offset(rq, cs);

return cs;
5083}

5085#undef NON_SKIP_LEN

5087static struct intel_context *
5088guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
   unsigned long flags)
5090{
struct guc_virtual_engine *ve;
struct intel_guc *guc;
unsigned int n;
int err;

ve = kzalloc(sizeof(*ve), GFP_KERNEL(0x0001 | 0x0004));
if (!ve)
return ERR_PTR(-ENOMEM12);

guc = &siblings[0]->gt->uc.guc;

ve->base.i915 = siblings[0]->i915;
ve->base.gt = siblings[0]->gt;
ve->base.uncore = siblings[0]->uncore;
ve->base.id = -1;

ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL-2;
ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL-2;
ve->base.saturated = ALL_ENGINES((intel_engine_mask_t)~0ul);

snprintf(ve->base.name, sizeof(ve->base.name), "virtual");

ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine);

ve->base.cops = &virtual_guc_context_ops;
ve->base.request_alloc = guc_request_alloc;
ve->base.bump_serial = virtual_guc_bump_serial;

ve->base.submit_request = guc_submit_request;

ve->base.flags = I915_ENGINE_IS_VIRTUAL(1UL << (5));

5124#ifdef notyet
BUILD_BUG_ON(ilog2(VIRTUAL_ENGINES) < I915_NUM_ENGINES)extern char _ctassert[(!(((sizeof((1UL << ((8 * sizeof(
intel_engine_mask_t)) - 1))) <= 4) ? (fls((1UL << ((
* sizeof(intel_engine_mask_t)) - 1))) - 1) : (flsl((1UL <<
 ((8 * sizeof(intel_engine_mask_t)) - 1))) - 1)) < I915_NUM_ENGINES
)) ? 1 : -1 ] __attribute__((__unused__));
5126#endif
ve->base.mask = VIRTUAL_ENGINES(1UL << ((8 * sizeof(intel_engine_mask_t)) - 1));

intel_context_init(&ve->context, &ve->base);

for (n = 0; n < count; n++) {
struct intel_engine_cs *sibling = siblings[n];

GEM_BUG_ON(!is_power_of_2(sibling->mask))((void)0);
if (sibling->mask & ve->base.mask) {
	DRM_DEBUG("duplicate %s entry in load balancer\n",___drm_dbg(((void *)0), DRM_UT_CORE, "duplicate %s entry in load balancer\n"
, sibling->name)
		  sibling->name)___drm_dbg(((void *)0), DRM_UT_CORE, "duplicate %s entry in load balancer\n"
, sibling->name);
	err = -EINVAL22;
	goto err_put;
}

ve->base.mask |= sibling->mask;
ve->base.logical_mask |= sibling->logical_mask;

if (n != 0 && ve->base.class != sibling->class) {
	DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n",___drm_dbg(((void *)0), DRM_UT_CORE, "invalid mixing of engine class, sibling %d, already %d\n"
, sibling->class, ve->base.class)
		  sibling->class, ve->base.class)___drm_dbg(((void *)0), DRM_UT_CORE, "invalid mixing of engine class, sibling %d, already %d\n"
, sibling->class, ve->base.class);
	err = -EINVAL22;
	goto err_put;
} else if (n == 0) {
	ve->base.class = sibling->class;
	ve->base.uabi_class = sibling->uabi_class;
	snprintf(ve->base.name, sizeof(ve->base.name),
		 "v%dx%d", ve->base.class, count);
	ve->base.context_size = sibling->context_size;

	ve->base.add_active_request =
		sibling->add_active_request;
	ve->base.remove_active_request =
		sibling->remove_active_request;
	ve->base.emit_bb_start = sibling->emit_bb_start;
	ve->base.emit_flush = sibling->emit_flush;
	ve->base.emit_init_breadcrumb =
		sibling->emit_init_breadcrumb;
	ve->base.emit_fini_breadcrumb =
		sibling->emit_fini_breadcrumb;
	ve->base.emit_fini_breadcrumb_dw =
		sibling->emit_fini_breadcrumb_dw;
	ve->base.breadcrumbs =
		intel_breadcrumbs_get(sibling->breadcrumbs);

	ve->base.flags |= sibling->flags;

	ve->base.props.timeslice_duration_ms =
		sibling->props.timeslice_duration_ms;
	ve->base.props.preempt_timeout_ms =
		sibling->props.preempt_timeout_ms;
}
}

return &ve->context;

5183err_put:
intel_context_put(&ve->context);
return ERR_PTR(err);
5186}

5188bool_Bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve)
5189{
struct intel_engine_cs *engine;
intel_engine_mask_t tmp, mask = ve->mask;

for_each_engine_masked(engine, ve->gt, mask, tmp)for ((tmp) = (mask) & (ve->gt)->info.engine_mask; (
tmp) ? ((engine) = (ve->gt)->engine[({ int __idx = ffs(
tmp) - 1; tmp &= ~(1UL << (__idx)); __idx; })]), 1 :
 0;)
if (READ_ONCE(engine->props.heartbeat_interval_ms)({ typeof(engine->props.heartbeat_interval_ms) __tmp = *(volatile
 typeof(engine->props.heartbeat_interval_ms) *)&(engine
->props.heartbeat_interval_ms); membar_datadep_consumer();
 __tmp; }))
	return true1;

return false0;
5198}

5200#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
5201#include "selftest_guc.c"
5202#include "selftest_guc_multi_lrc.c"
5203#include "selftest_guc_hangcheck.c"
5204#endif

←

/usr/src/sys/dev/pci/drm/include/linux/slab.h

1/* Public domain. */
2 
3#ifndef _LINUX_SLAB_H
4#define _LINUX_SLAB_H
5 
6#include <sys/types.h>
7#include <sys/malloc.h>
8 
9#include <linux/types.h>
10#include <linux/workqueue.h>
11#include <linux/gfp.h>
12 
13#include <linux/processor.h>	/* for CACHELINESIZE */
14 
15#define ARCH_KMALLOC_MINALIGN64 CACHELINESIZE64
16 
17#define ZERO_SIZE_PTR((void *)0) NULL((void *)0)
18 
19static inline void *
20kmalloc(size_t size, int flags)
21{
22	return malloc(size, M_DRM145, flags);
23}
24 
25static inline void *
26kmalloc_array(size_t n, size_t size, int flags)
27{
28	if (n != 0 && SIZE_MAX0xffffffffffffffffUL / n < size)
3
←
Assuming 'n' is equal to 0→
29		return NULL((void *)0);
30	return malloc(n * size, M_DRM145, flags);
4
←
Returning pointer, which participates in a condition later→
31}
32 
33static inline void *
34kcalloc(size_t n, size_t size, int flags)
35{
36	if (n != 0 && SIZE_MAX0xffffffffffffffffUL / n < size)
37		return NULL((void *)0);
38	return malloc(n * size, M_DRM145, flags | M_ZERO0x0008);
39}
40 
41static inline void *
42kzalloc(size_t size, int flags)
43{
44	return malloc(size, M_DRM145, flags | M_ZERO0x0008);
45}
46 
47static inline void
48kfree(const void *objp)
49{
50	free((void *)objp, M_DRM145, 0);
51}
52 
53#endif