/usr/src/sys/dev/pci/drm/i915/i915_reg

Bug Summary

File:	dev/pci/drm/i915/i915_reg_defs.h
Warning:	line 111, column 2 Undefined or garbage value returned to caller

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name intel_workarounds.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/gt/intel_workarounds.c

/usr/src/sys/dev/pci/drm/i915/gt/intel_workarounds.c

→

1// SPDX-License-Identifier: MIT
2/*
* Copyright © 2014-2018 Intel Corporation
*/

6#include "i915_drv.h"
7#include "intel_context.h"
8#include "intel_engine_pm.h"
9#include "intel_engine_regs.h"
10#include "intel_gpu_commands.h"
11#include "intel_gt.h"
12#include "intel_gt_mcr.h"
13#include "intel_gt_regs.h"
14#include "intel_ring.h"
15#include "intel_workarounds.h"

17/**
* DOC: Hardware workarounds
*
* This file is intended as a central place to implement most [1]_ of the
* required workarounds for hardware to work as originally intended. They fall
* in five basic categories depending on how/when they are applied:
*
* - Workarounds that touch registers that are saved/restored to/from the HW
*   context image. The list is emitted (via Load Register Immediate commands)
*   everytime a new context is created.
* - GT workarounds. The list of these WAs is applied whenever these registers
*   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
* - Display workarounds. The list is applied during display clock-gating
*   initialization.
* - Workarounds that whitelist a privileged register, so that UMDs can manage
*   them directly. This is just a special case of a MMMIO workaround (as we
*   write the list of these to/be-whitelisted registers to some special HW
*   registers).
* - Workaround batchbuffers, that get executed automatically by the hardware
*   on every HW context restore.
*
* .. [1] Please notice that there are other WAs that, due to their nature,
*    cannot be applied from a central place. Those are peppered around the rest
*    of the code, as needed.
*
* .. [2] Technically, some registers are powercontext saved & restored, so they
*    survive a suspend/resume. In practice, writing them again is not too
*    costly and simplifies things. We can revisit this in the future.
*
* Layout
* ~~~~~~
*
* Keep things in this file ordered by WA type, as per the above (context, GT,
* display, register whitelist, batchbuffer). Then, inside each type, keep the
* following order:
*
* - Infrastructure functions and macros
* - WAs per platform in standard gen/chrono order
* - Public functions to init or apply the given workaround type.
*/

58static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
59{
wal->name = name;
wal->engine_name = engine_name;
62}

64#define WA_LIST_CHUNK(1 << 4) (1 << 4)

66static void wa_init_finish(struct i915_wa_list *wal)
67{
/* Trim unused entries. */
if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)(((wal->count) & (((1 << 4)) - 1)) == 0)) {
struct i915_wa *list = kmemdup(wal->list,
			       wal->count * sizeof(*list),
			       GFP_KERNEL(0x0001 | 0x0004));

if (list) {
	kfree(wal->list);
	wal->list = list;
}
}

if (!wal->count)
return;

DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",___drm_dbg(((void *)0), DRM_UT_DRIVER, "Initialized %u %s workarounds on %s\n"
, wal->wa_count, wal->name, wal->engine_name)
	 wal->wa_count, wal->name, wal->engine_name)___drm_dbg(((void *)0), DRM_UT_DRIVER, "Initialized %u %s workarounds on %s\n"
, wal->wa_count, wal->name, wal->engine_name);
85}

87static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
88{
unsigned int addr = i915_mmio_reg_offset(wa->reg);
unsigned int start = 0, end = wal->count;
const unsigned int grow = WA_LIST_CHUNK(1 << 4);
struct i915_wa *wa_;

GEM_BUG_ON(!is_power_of_2(grow))((void)0);

if (IS_ALIGNED(wal->count, grow)(((wal->count) & ((grow) - 1)) == 0)) { /* Either uninitialized or full. */
21
←
Assuming the condition is true→
22
←
Taking true branch→
struct i915_wa *list;

list = kmalloc_array(roundup2(wal->count + 1, grow)(((wal->count + 1) + ((grow) - 1)) & (~((__typeof(wal->
count + 1))(grow) - 1))), sizeof(*wa),
		     GFP_KERNEL(0x0001 | 0x0004));
if (!list) {
23
←
Assuming 'list' is non-null→
24
←
Taking false branch→
	DRM_ERROR("No space for workaround init!\n")__drm_err("No space for workaround init!\n");
	return;
}

if (wal->list) {
25
←
Assuming field 'list' is null→
26
←
Taking false branch→
	memcpy(list, wal->list, sizeof(*wa) * wal->count)__builtin_memcpy((list), (wal->list), (sizeof(*wa) * wal->
count));
	kfree(wal->list);
}

wal->list = list;
}

while (start < end) {
27
←
Assuming 'start' is < 'end'→
28
←
Loop condition is true.  Entering loop body→
unsigned int mid = start + (end - start) / 2;

if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
29
←
Uninitialized value stored to 'reg.reg'→
30
←
Calling 'i915_mmio_reg_offset'→
	start = mid + 1;
} else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
	end = mid;
} else {
	wa_ = &wal->list[mid];

	if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
		DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",__drm_err("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n"
, i915_mmio_reg_offset(wa_->reg), wa_->clr, wa_->set
)
			  i915_mmio_reg_offset(wa_->reg),__drm_err("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n"
, i915_mmio_reg_offset(wa_->reg), wa_->clr, wa_->set
)
			  wa_->clr, wa_->set)__drm_err("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n"
, i915_mmio_reg_offset(wa_->reg), wa_->clr, wa_->set
);

		wa_->set &= ~wa->clr;
	}

	wal->wa_count++;
	wa_->set |= wa->set;
	wa_->clr |= wa->clr;
	wa_->read |= wa->read;
	return;
}
}

wal->wa_count++;
wa_ = &wal->list[wal->count++];
*wa_ = *wa;

while (wa_-- > wal->list) {
GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==((void)0)
	   i915_mmio_reg_offset(wa_[1].reg))((void)0);
if (i915_mmio_reg_offset(wa_[1].reg) >
    i915_mmio_reg_offset(wa_[0].reg))
	break;

swap(wa_[1], wa_[0])do { __typeof(wa_[1]) __tmp = (wa_[1]); (wa_[1]) = (wa_[0]); (
wa_[0]) = __tmp; } while(0);
}
153}

155static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
   u32 clear, u32 set, u32 read_mask, bool_Bool masked_reg)
157{
struct i915_wa wa = {
.reg  = reg,
.clr  = clear,
.set  = set,
.read = read_mask,
.masked_reg = masked_reg,
};

_wa_add(wal, &wa);
20
←
Calling '_wa_add'→
167}

169static void
170wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
171{
wa_add(wal, reg, clear, set, clear, false0);
19
←
Calling 'wa_add'→
173}

175static void
176wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
177{
wa_write_clr_set(wal, reg, ~0, set);
18
←
Calling 'wa_write_clr_set'→
179}

181static void
182wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
183{
wa_write_clr_set(wal, reg, set, set);
185}

187static void
188wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
189{
wa_write_clr_set(wal, reg, clr, 0);
191}

193/*
* WA operations on "masked register". A masked register has the upper 16 bits
* documented as "masked" in b-spec. Its purpose is to allow writing to just a
* portion of the register without a rmw: you simply write in the upper 16 bits
* the mask of bits you are going to modify.
*
* The wa_masked_* family of functions already does the necessary operations to
* calculate the mask based on the parameters passed, so user only has to
* provide the lower 16 bits of that register.
*/

204static void
205wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
206{
wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val)({ typeof(val) _a = (val); ({ if (__builtin_constant_p(_a)) do
 { } while (0); if (__builtin_constant_p(_a)) do { } while (0
); if (__builtin_constant_p(_a) && __builtin_constant_p
(_a)) do { } while (0); ((_a) << 16 | (_a)); }); }), val, true1);
208}

210static void
211wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
212{
wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val)(({ if (__builtin_constant_p((val))) do { } while (0); if (__builtin_constant_p
(0)) do { } while (0); if (__builtin_constant_p((val)) &&
 __builtin_constant_p(0)) do { } while (0); (((val)) <<
| (0)); })), val, true1);
214}

216static void
217wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
    u32 mask, u32 val)
219{
wa_add(wal, reg, 0, _MASKED_FIELD(mask, val)({ if (__builtin_constant_p(mask)) do { } while (0); if (__builtin_constant_p
(val)) do { } while (0); if (__builtin_constant_p(mask) &&
 __builtin_constant_p(val)) do { } while (0); ((mask) <<
| (val)); }), mask, true1);
221}

223static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
		      struct i915_wa_list *wal)
225{
wa_masked_en(wal, INSTPM((const i915_reg_t){ .reg = (0x20c0) }), INSTPM_FORCE_ORDERING(1 << 7));
227}

229static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
		      struct i915_wa_list *wal)
231{
wa_masked_en(wal, INSTPM((const i915_reg_t){ .reg = (0x20c0) }), INSTPM_FORCE_ORDERING(1 << 7));
233}

235static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
		      struct i915_wa_list *wal)
237{
wa_masked_en(wal, INSTPM((const i915_reg_t){ .reg = (0x20c0) }), INSTPM_FORCE_ORDERING(1 << 7));

/* WaDisableAsyncFlipPerfMode:bdw,chv */
wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x9c) }), ASYNC_FLIP_PERF_DISABLE((u32)((1UL << (14)) + 0)));

/* WaDisablePartialInstShootdown:bdw,chv */
wa_masked_en(wal, GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }),
     PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE((u32)((1UL << (8)) + 0)));

/* Use Force Non-Coherent whenever executing a 3D context. This is a
* workaround for a possible hang in the unlikely event a TLB
* invalidation occurs during a PSD flush.
*/
/* WaForceEnableNonCoherent:bdw,chv */
/* WaHdcDisableFetchWhenMasked:bdw,chv */
wa_masked_en(wal, HDC_CHICKEN0((const i915_reg_t){ .reg = (0x7300) }),
     HDC_DONOT_FETCH_MEM_WHEN_MASKED(1 << 11) |
     HDC_FORCE_NON_COHERENT(1 << 4));

/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
* "The Hierarchical Z RAW Stall Optimization allows non-overlapping
*  polygons in the same 8x4 pixel/sample area to be processed without
*  stalling waiting for the earlier ones to write to Hierarchical Z
*  buffer."
*
* This optimization is off by default for BDW and CHV; turn it on.
*/
wa_masked_dis(wal, CACHE_MODE_0_GEN7((const i915_reg_t){ .reg = (0x7000) }), HIZ_RAW_STALL_OPT_DISABLE(1 << 2));

/* Wa4x4STCOptimizationDisable:bdw,chv */
wa_masked_en(wal, CACHE_MODE_1((const i915_reg_t){ .reg = (0x7004) }), GEN8_4x4_STC_OPTIMIZATION_DISABLE(1 << 6));

/*
* BSpec recommends 8x4 when MSAA is used,
* however in practice 16x4 seems fastest.
*
* Note that PS/WM thread counts depend on the WIZ hashing
* disable bit, which we don't touch here, but it's good
* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
*/
wa_masked_field_set(wal, GEN7_GT_MODE((const i915_reg_t){ .reg = (0x7008) }),
	    GEN6_WIZ_HASHING_MASK(((1) << 9) | ((1) << 7)),
	    GEN6_WIZ_HASHING_16x4(((1) << 9) | ((0) << 7)));
281}

283static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
285{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

gen8_ctx_workarounds_init(engine, wal);

/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
wa_masked_en(wal, GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }), STALL_DOP_GATING_DISABLE((u32)((1UL << (5)) + 0)));

/* WaDisableDopClockGating:bdw
*
* Also see the related UCGTCL1 write in bdw_init_clock_gating()
* to disable EUTC clock gating.
*/
wa_masked_en(wal, GEN7_ROW_CHICKEN2((const i915_reg_t){ .reg = (0xe4f4) }),
     DOP_CLOCK_GATING_DISABLE(1 << 0));

wa_masked_en(wal, HALF_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0xe184) }),
     GEN8_SAMPLER_POWER_BYPASS_DIS(1 << 1));

wa_masked_en(wal, HDC_CHICKEN0((const i915_reg_t){ .reg = (0x7300) }),
     /* WaForceContextSaveRestoreNonCoherent:bdw */
     HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT(1 << 5) |
     /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
     (IS_BDW_GT3(i915)(IS_PLATFORM(i915, INTEL_BROADWELL) && (&(i915)->
__info)->gt == 3) ? HDC_FENCE_DEST_SLM_DISABLE(1 << 14) : 0));
309}

311static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
313{
gen8_ctx_workarounds_init(engine, wal);

/* WaDisableThreadStallDopClockGating:chv */
wa_masked_en(wal, GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }), STALL_DOP_GATING_DISABLE((u32)((1UL << (5)) + 0)));

/* Improve HiZ throughput on CHV. */
wa_masked_en(wal, HIZ_CHICKEN((const i915_reg_t){ .reg = (0x7018) }), CHV_HZ_8X8_MODE_IN_1X((u32)((1UL << (15)) + 0)));
321}

323static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
		      struct i915_wa_list *wal)
325{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

if (HAS_LLC(i915)((&(i915)->__info)->has_llc)) {
/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 *
 * Must match Display Engine. See
 * WaCompressedResourceDisplayNewHashMode.
 */
wa_masked_en(wal, COMMON_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0x7014) }),
	     GEN9_PBE_COMPRESSED_HASH_SELECTION(1 << 13));
wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7((const i915_reg_t){ .reg = (0xe194) }),
	     GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR((u32)((1UL << (8)) + 0)));
}

/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
wa_masked_en(wal, GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }),
     FLOW_CONTROL_ENABLE((u32)((1UL << (15)) + 0)) |
     PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE((u32)((1UL << (8)) + 0)));

/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7((const i915_reg_t){ .reg = (0xe194) }),
     GEN9_ENABLE_YV12_BUGFIX((u32)((1UL << (4)) + 0)) |
     GEN9_ENABLE_GPGPU_PREEMPTION((u32)((1UL << (2)) + 0)));

/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
wa_masked_en(wal, CACHE_MODE_1((const i915_reg_t){ .reg = (0x7004) }),
     GEN8_4x4_STC_OPTIMIZATION_DISABLE(1 << 6) |
     GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE(1 << 1));

/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5((const i915_reg_t){ .reg = (0xe188) }),
      GEN9_CCS_TLB_PREFETCH_ENABLE(1 << 3));

/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
wa_masked_en(wal, HDC_CHICKEN0((const i915_reg_t){ .reg = (0x7300) }),
     HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT(1 << 5) |
     HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE(1 << 15));

/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
* both tied to WaForceContextSaveRestoreNonCoherent
* in some hsds for skl. We keep the tie for all gen9. The
* documentation is a bit hazy and so we want to get common behaviour,
* even though there is no clear evidence we would need both on kbl/bxt.
* This area has been source of system hangs so we play it safe
* and mimic the skl regardless of what bspec says.
*
* Use Force Non-Coherent whenever executing a 3D context. This
* is a workaround for a possible hang in the unlikely event
* a TLB invalidation occurs during a PSD flush.
*/

/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
wa_masked_en(wal, HDC_CHICKEN0((const i915_reg_t){ .reg = (0x7300) }),
     HDC_FORCE_NON_COHERENT(1 << 4));

/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
if (IS_SKYLAKE(i915)IS_PLATFORM(i915, INTEL_SKYLAKE) ||
   IS_KABYLAKE(i915)IS_PLATFORM(i915, INTEL_KABYLAKE) ||
   IS_COFFEELAKE(i915)IS_PLATFORM(i915, INTEL_COFFEELAKE) ||
   IS_COMETLAKE(i915)IS_PLATFORM(i915, INTEL_COMETLAKE))
wa_masked_en(wal, HALF_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0xe184) }),
	     GEN8_SAMPLER_POWER_BYPASS_DIS(1 << 1));

/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
wa_masked_en(wal, HALF_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0xe180) }), GEN8_ST_PO_DISABLE(1 << 13));

/*
* Supporting preemption with fine-granularity requires changes in the
* batch buffer programming. Since we can't break old userspace, we
* need to set our default preemption level to safe value. Userspace is
* still able to use more fine-grained preemption levels, since in
* WaEnablePreemptionGranularityControlByUMD we're whitelisting the
* per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
* not real HW workarounds, but merely a way to start using preemption
* while maintaining old contract with userspace.
*/

/* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
wa_masked_dis(wal, GEN8_CS_CHICKEN1((const i915_reg_t){ .reg = (0x2580) }), GEN9_PREEMPT_3D_OBJECT_LEVEL(1 << 0));

/* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
wa_masked_field_set(wal, GEN8_CS_CHICKEN1((const i915_reg_t){ .reg = (0x2580) }),
	    GEN9_PREEMPT_GPGPU_LEVEL_MASK(((1) << 2) | ((1) << 1)),
	    GEN9_PREEMPT_GPGPU_COMMAND_LEVEL(((1) << 2) | ((0) << 1)));

/* WaClearHIZ_WM_CHICKEN3:bxt,glk */
if (IS_GEN9_LP(i915)(((&(i915)->__runtime)->graphics.ip.ver) == 9 &&
 ((&(i915)->__info)->is_lp)))
wa_masked_en(wal, GEN9_WM_CHICKEN3((const i915_reg_t){ .reg = (0x5588) }), GEN9_FACTOR_IN_CLR_VAL_HIZ(1 << 9));
417}

419static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
		struct i915_wa_list *wal)
421{
struct intel_gt *gt = engine->gt;
u8 vals[3] = { 0, 0, 0 };
unsigned int i;

for (i = 0; i < 3; i++) {
u8 ss;

/*
 * Only consider slices where one, and only one, subslice has 7
 * EUs
 */
if (!is_power_of_2(gt->info.sseu.subslice_7eu[i])(((gt->info.sseu.subslice_7eu[i]) != 0) && (((gt->
info.sseu.subslice_7eu[i]) - 1) & (gt->info.sseu.subslice_7eu
[i])) == 0))
	continue;

/*
 * subslice_7eu[i] != 0 (because of the check above) and
 * ss_max == 4 (maximum number of subslices possible per slice)
 *
 * ->    0 <= ss <= 3;
 */
ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
vals[i] = 3 - ss;
}

if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
return;

/* Tune IZ hashing. See intel_device_info_runtime_init() */
wa_masked_field_set(wal, GEN7_GT_MODE((const i915_reg_t){ .reg = (0x7008) }),
	    GEN9_IZ_HASHING_MASK(2)(0x3 << ((2) * 2)) |
	    GEN9_IZ_HASHING_MASK(1)(0x3 << ((1) * 2)) |
	    GEN9_IZ_HASHING_MASK(0)(0x3 << ((0) * 2)),
	    GEN9_IZ_HASHING(2, vals[2])((vals[2]) << ((2) * 2)) |
	    GEN9_IZ_HASHING(1, vals[1])((vals[1]) << ((1) * 2)) |
	    GEN9_IZ_HASHING(0, vals[0])((vals[0]) << ((0) * 2)));
457}

459static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
461{
gen9_ctx_workarounds_init(engine, wal);
skl_tune_iz_hashing(engine, wal);
464}

466static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
468{
gen9_ctx_workarounds_init(engine, wal);

/* WaDisableThreadStallDopClockGating:bxt */
wa_masked_en(wal, GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }),
     STALL_DOP_GATING_DISABLE((u32)((1UL << (5)) + 0)));

/* WaToEnableHwFixForPushConstHWBug:bxt */
wa_masked_en(wal, COMMON_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0x7014) }),
     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION(1 << 8));
478}

480static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
482{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

gen9_ctx_workarounds_init(engine, wal);

/* WaToEnableHwFixForPushConstHWBug:kbl */
if (IS_KBL_GRAPHICS_STEP(i915, STEP_C0, STEP_FOREVER)(IS_PLATFORM(i915, INTEL_KABYLAKE) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_C0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_FOREVER
))))
wa_masked_en(wal, COMMON_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0x7014) }),
	     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION(1 << 8));

/* WaDisableSbeCacheDispatchPortSharing:kbl */
wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0xe100) }),
     GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE(1 << 4));
495}

497static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
499{
gen9_ctx_workarounds_init(engine, wal);

/* WaToEnableHwFixForPushConstHWBug:glk */
wa_masked_en(wal, COMMON_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0x7014) }),
     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION(1 << 8));
505}

507static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
509{
gen9_ctx_workarounds_init(engine, wal);

/* WaToEnableHwFixForPushConstHWBug:cfl */
wa_masked_en(wal, COMMON_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0x7014) }),
     GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION(1 << 8));

/* WaDisableSbeCacheDispatchPortSharing:cfl */
wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0xe100) }),
     GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE(1 << 4));
519}

521static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
523{
/* Wa_1406697149 (WaDisableBankHangMode:icl) */
wa_write(wal,
 GEN8_L3CNTLREG((const i915_reg_t){ .reg = (0x7034) }),
 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG((const i915_reg_t){ .reg = (0x7034) })) |
 GEN8_ERRDETBCTRL(1 << 9));

/* WaForceEnableNonCoherent:icl
* This is not the same workaround as in early Gen9 platforms, where
* lacking this could cause system hangs, but coherency performance
* overhead is high and only a few compute workloads really need it
* (the register is whitelisted in hardware now, so UMDs can opt in
* for coherency if they have a good reason).
*/
wa_masked_en(wal, ICL_HDC_MODE((const i915_reg_t){ .reg = (0xe5f4) }), HDC_FORCE_NON_COHERENT(1 << 4));

/* WaEnableFloatBlendOptimization:icl */
wa_add(wal, GEN10_CACHE_MODE_SS((const i915_reg_t){ .reg = (0xe420) }), 0,
      _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE)({ typeof(((u32)((1UL << (4)) + 0))) _a = (((u32)((1UL <<
 (4)) + 0))); ({ if (__builtin_constant_p(_a)) do { } while (
0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p
(_a) && __builtin_constant_p(_a)) do { } while (0); (
(_a) << 16 | (_a)); }); }),
      0 /* write-only, so skip validation */,
      true1);

/* WaDisableGPGPUMidThreadPreemption:icl */
wa_masked_field_set(wal, GEN8_CS_CHICKEN1((const i915_reg_t){ .reg = (0x2580) }),
	    GEN9_PREEMPT_GPGPU_LEVEL_MASK(((1) << 2) | ((1) << 1)),
	    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL(((0) << 2) | ((1) << 1)));

/* allow headerless messages for preemptible GPGPU context */
wa_masked_en(wal, GEN10_SAMPLER_MODE((const i915_reg_t){ .reg = (0xe18c) }),
     GEN11_SAMPLER_ENABLE_HEADLESS_MSG((u32)((1UL << (5)) + 0)));

/* Wa_1604278689:icl,ehl */
wa_write(wal, IVB_FBC_RT_BASE((const i915_reg_t){ .reg = (0x7020) }), 0xFFFFFFFF & ~ILK_FBC_RT_VALID((u32)((1UL << (0)) + 0)));
wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER((const i915_reg_t){ .reg = (0x7024) }),
	 0, /* write-only register; skip validation */
	 0xFFFFFFFF);

/* Wa_1406306137:icl,ehl */
wa_masked_en(wal, GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }), GEN11_DIS_PICK_2ND_EU((u32)((1UL << (7)) + 0)));
562}

564/*
* These settings aren't actually workarounds, but general tuning settings that
* need to be programmed on dg2 platform.
*/
568static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine,
		   struct i915_wa_list *wal)
570{
wa_masked_en(wal, CHICKEN_RASTER_2((const i915_reg_t){ .reg = (0x6208) }), TBIMR_FAST_CLIP((u32)((1UL << (5)) + 0)));
wa_write_clr_set(wal, GEN11_L3SQCREG5((const i915_reg_t){ .reg = (0xb158) }), L3_PWM_TIMER_INIT_VAL_MASK((u32)((((~0UL) >> (64 - (9) - 1)) & ((~0UL) <<
 (0))) + 0)),
	 REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)((u32)((((typeof(((u32)((((~0UL) >> (64 - (9) - 1)) &
 ((~0UL) << (0))) + 0))))(0x7f) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (9) - 1)) & ((~0UL) <<
 (0))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (9)
 - 1)) & ((~0UL) << (0))) + 0)))) + 0 + 0 + 0 + 0)));
wa_add(wal,
      FF_MODE2((const i915_reg_t){ .reg = (0x6604) }),
      FF_MODE2_TDS_TIMER_MASK((u32)((((~0UL) >> (64 - (23) - 1)) & ((~0UL) <<
 (16))) + 0)),
      FF_MODE2_TDS_TIMER_128((u32)((((typeof(((u32)((((~0UL) >> (64 - (23) - 1)) &
 ((~0UL) << (16))) + 0))))(4) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (23) - 1)) & ((~0UL) <<
 (16))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (23
) - 1)) & ((~0UL) << (16))) + 0)))) + 0 + 0 + 0 + 0
)),
      0, false0);
579}

581/*
* These settings aren't actually workarounds, but general tuning settings that
* need to be programmed on several platforms.
*/
585static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
587{
/*
* Although some platforms refer to it as Wa_1604555607, we need to
* program it even on those that don't explicitly list that
* workaround.
*
* Note that the programming of this register is further modified
* according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
* Wa_1608008084 tells us the FF_MODE2 register will return the wrong
* value when read. The default value for this register is zero for all
* fields and there are no bit masks. So instead of doing a RMW we
* should just write TDS timer value. For the same reason read
* verification is ignored.
*/
wa_add(wal,
      FF_MODE2((const i915_reg_t){ .reg = (0x6604) }),
      FF_MODE2_TDS_TIMER_MASK((u32)((((~0UL) >> (64 - (23) - 1)) & ((~0UL) <<
 (16))) + 0)),
      FF_MODE2_TDS_TIMER_128((u32)((((typeof(((u32)((((~0UL) >> (64 - (23) - 1)) &
 ((~0UL) << (16))) + 0))))(4) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (23) - 1)) & ((~0UL) <<
 (16))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (23
) - 1)) & ((~0UL) << (16))) + 0)))) + 0 + 0 + 0 + 0
)),
      0, false0);
606}

608static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
		       struct i915_wa_list *wal)
610{
gen12_ctx_gt_tuning_init(engine, wal);

/*
* Wa_1409142259:tgl,dg1,adl-p
* Wa_1409347922:tgl,dg1,adl-p
* Wa_1409252684:tgl,dg1,adl-p
* Wa_1409217633:tgl,dg1,adl-p
* Wa_1409207793:tgl,dg1,adl-p
* Wa_1409178076:tgl,dg1,adl-p
* Wa_1408979724:tgl,dg1,adl-p
* Wa_14010443199:tgl,rkl,dg1,adl-p
* Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
* Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
*/
wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0x7304) }),
     GEN12_DISABLE_CPS_AWARE_COLOR_PIPE((u32)((1UL << (9)) + 0)));

/* WaDisableGPGPUMidThreadPreemption:gen12 */
wa_masked_field_set(wal, GEN8_CS_CHICKEN1((const i915_reg_t){ .reg = (0x2580) }),
	    GEN9_PREEMPT_GPGPU_LEVEL_MASK(((1) << 2) | ((1) << 1)),
	    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL(((0) << 2) | ((1) << 1)));

/*
* Wa_16011163337
*
* Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
* to Wa_1608008084.
*/
wa_add(wal,
      FF_MODE2((const i915_reg_t){ .reg = (0x6604) }),
      FF_MODE2_GS_TIMER_MASK((u32)((((~0UL) >> (64 - (31) - 1)) & ((~0UL) <<
 (24))) + 0)),
      FF_MODE2_GS_TIMER_224((u32)((((typeof(((u32)((((~0UL) >> (64 - (31) - 1)) &
 ((~0UL) << (24))) + 0))))(224) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (31) - 1)) & ((~0UL) <<
 (24))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (31
) - 1)) & ((~0UL) << (24))) + 0)))) + 0 + 0 + 0 + 0
)),
      0, false0);
644}

646static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
648{
gen12_ctx_workarounds_init(engine, wal);

/* Wa_1409044764 */
wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0x7304) }),
      DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN((u32)((1UL << (12)) + 0)));

/* Wa_22010493298 */
wa_masked_en(wal, HIZ_CHICKEN((const i915_reg_t){ .reg = (0x7018) }),
     DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE((u32)((1UL << (14)) + 0)));
658}

660static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine,
		     struct i915_wa_list *wal)
662{
dg2_ctx_gt_tuning_init(engine, wal);

/* Wa_16011186671:dg2_g11 */
if (IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 1) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0)))) {
wa_masked_dis(wal, VFLSKPD((const i915_reg_t){ .reg = (0x62a8) }), DIS_MULT_MISS_RD_SQUASH((u32)((1UL << (0)) + 0)));
wa_masked_en(wal, VFLSKPD((const i915_reg_t){ .reg = (0x62a8) }), DIS_OVER_FETCH_CACHE((u32)((1UL << (1)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0)))) {
/* Wa_14010469329:dg2_g10 */
wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0x7304) }),
	     XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE((u32)((1UL << (12)) + 0)));

/*
 * Wa_22010465075:dg2_g10
 * Wa_22010613112:dg2_g10
 * Wa_14010698770:dg2_g10
 */
wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0x7304) }),
	     GEN12_DISABLE_CPS_AWARE_COLOR_PIPE((u32)((1UL << (9)) + 0)));
}

/* Wa_16013271637:dg2 */
wa_masked_en(wal, SLICE_COMMON_ECO_CHICKEN1((const i915_reg_t){ .reg = (0x731c) }),
     MSC_MSAA_REODER_BUF_BYPASS_DISABLE((u32)((1UL << (14)) + 0)));

/* Wa_14014947963:dg2 */
if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_FOREVER))) ||
IS_DG2_G11(engine->i915)IS_SUBPLATFORM(engine->i915, INTEL_DG2, 1) || IS_DG2_G12(engine->i915)IS_SUBPLATFORM(engine->i915, INTEL_DG2, 2))
wa_masked_field_set(wal, VF_PREEMPTION((const i915_reg_t){ .reg = (0x83a4) }), PREEMPTION_VERTEX_COUNT((u32)((((~0UL) >> (64 - (15) - 1)) & ((~0UL) <<
 (0))) + 0)), 0x4000);

/* Wa_15010599737:dg2 */
wa_masked_en(wal, CHICKEN_RASTER_1((const i915_reg_t){ .reg = (0x6204) }), DIS_SF_ROUND_NEAREST_EVEN((u32)((1UL << (8)) + 0)));
696}

698static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine,
			 struct i915_wa_list *wal)
700{
/*
* This is a "fake" workaround defined by software to ensure we
* maintain reliable, backward-compatible behavior for userspace with
* regards to how nested MI_BATCH_BUFFER_START commands are handled.
*
* The per-context setting of MI_MODE[12] determines whether the bits
* of a nested MI_BATCH_BUFFER_START instruction should be interpreted
* in the traditional manner or whether they should instead use a new
* tgl+ meaning that breaks backward compatibility, but allows nesting
* into 3rd-level batchbuffers.  When this new capability was first
* added in TGL, it remained off by default unless a context
* intentionally opted in to the new behavior.  However Xe_HPG now
* flips this on by default and requires that we explicitly opt out if
* we don't want the new behavior.
*
* From a SW perspective, we want to maintain the backward-compatible
* behavior for userspace, so we'll apply a fake workaround to set it
* back to the legacy behavior on platforms where the hardware default
* is to break compatibility.  At the moment there is no Linux
* userspace that utilizes third-level batchbuffers, so this will avoid
* userspace from needing to make any changes.  using the legacy
* meaning is the correct thing to do.  If/when we have userspace
* consumers that want to utilize third-level batch nesting, we can
* provide a context parameter to allow them to opt-in.
*/
wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x9c) }
), TGL_NESTED_BB_EN((u32)((1UL << (12)) + 0)));
727}

729static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine,
		   struct i915_wa_list *wal)
731{
u8 mocs;

/*
* Some blitter commands do not have a field for MOCS, those
* commands will use MOCS index pointed by BLIT_CCTL.
* BLIT_CCTL registers are needed to be programmed to un-cached.
*/
if (engine->class == COPY_ENGINE_CLASS3) {
mocs = engine->gt->mocs.uc_index;
wa_write_clr_set(wal,
		 BLIT_CCTL(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x204) }
),
		 BLIT_CCTL_MASK(((u32)((((~0UL) >> (64 - (14) - 1)) & ((~0UL) <<
 (8))) + 0)) | ((u32)((((~0UL) >> (64 - (6) - 1)) &
 ((~0UL) << (0))) + 0))),
		 BLIT_CCTL_MOCS(mocs, mocs)(((u32)((((typeof(((u32)((((~0UL) >> (64 - (14) - 1)) &
 ((~0UL) << (8))) + 0))))((mocs) << 1) << (
__builtin_ffsll(((u32)((((~0UL) >> (64 - (14) - 1)) &
 ((~0UL) << (8))) + 0))) - 1)) & (((u32)((((~0UL) >>
 (64 - (14) - 1)) & ((~0UL) << (8))) + 0)))) + 0 + 0
 + 0 + 0)) | ((u32)((((typeof(((u32)((((~0UL) >> (64 - (
6) - 1)) & ((~0UL) << (0))) + 0))))((mocs) <<
 1) << (__builtin_ffsll(((u32)((((~0UL) >> (64 - (
6) - 1)) & ((~0UL) << (0))) + 0))) - 1)) & (((u32
)((((~0UL) >> (64 - (6) - 1)) & ((~0UL) << (0
))) + 0)))) + 0 + 0 + 0 + 0))));
}
746}

748/*
* gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround
* defined by the hardware team, but it programming general context registers.
* Adding those context register programming in context workaround
* allow us to use the wa framework for proper application and validation.
*/
754static void
755gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine,
	  struct i915_wa_list *wal)
757{
if (GRAPHICS_VER_FULL(engine->i915)(((&(engine->i915)->__runtime)->graphics.ip.ver)
 << 8 | ((&(engine->i915)->__runtime)->graphics
.ip.rel)) >= IP_VER(12, 55)((12) << 8 | (55)))
fakewa_disable_nestedbb_mode(engine, wal);

gen12_ctx_gt_mocs_init(engine, wal);
762}

764static void
765__intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
	   struct i915_wa_list *wal,
	   const char *name)
768{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

wa_init_start(wal, name, engine->name);

/* Applies to all engines */
/*
* Fake workarounds are not the actual workaround but
* programming of context registers using workaround framework.
*/
if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 12)
gen12_ctx_gt_fake_wa_init(engine, wal);

if (engine->class != RENDER_CLASS0)
goto done;

if (IS_PONTEVECCHIO(i915)IS_PLATFORM(i915, INTEL_PONTEVECCHIO))
; /* noop; none at this time */
else if (IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2))
dg2_ctx_workarounds_init(engine, wal);
else if (IS_XEHPSDV(i915)IS_PLATFORM(i915, INTEL_XEHPSDV))
; /* noop; none at this time */
else if (IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1))
dg1_ctx_workarounds_init(engine, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 12)
gen12_ctx_workarounds_init(engine, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 11)
icl_ctx_workarounds_init(engine, wal);
else if (IS_COFFEELAKE(i915)IS_PLATFORM(i915, INTEL_COFFEELAKE) || IS_COMETLAKE(i915)IS_PLATFORM(i915, INTEL_COMETLAKE))
cfl_ctx_workarounds_init(engine, wal);
else if (IS_GEMINILAKE(i915)IS_PLATFORM(i915, INTEL_GEMINILAKE))
glk_ctx_workarounds_init(engine, wal);
else if (IS_KABYLAKE(i915)IS_PLATFORM(i915, INTEL_KABYLAKE))
kbl_ctx_workarounds_init(engine, wal);
else if (IS_BROXTON(i915)IS_PLATFORM(i915, INTEL_BROXTON))
bxt_ctx_workarounds_init(engine, wal);
else if (IS_SKYLAKE(i915)IS_PLATFORM(i915, INTEL_SKYLAKE))
skl_ctx_workarounds_init(engine, wal);
else if (IS_CHERRYVIEW(i915)IS_PLATFORM(i915, INTEL_CHERRYVIEW))
chv_ctx_workarounds_init(engine, wal);
else if (IS_BROADWELL(i915)IS_PLATFORM(i915, INTEL_BROADWELL))
bdw_ctx_workarounds_init(engine, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 7)
gen7_ctx_workarounds_init(engine, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 6)
gen6_ctx_workarounds_init(engine, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) < 8)
;
else
MISSING_CASE(GRAPHICS_VER(i915))({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "((&(i915)->__runtime)->graphics.ip.ver)", (long)
(((&(i915)->__runtime)->graphics.ip.ver))); __builtin_expect
(!!(__ret), 0); });

819done:
wa_init_finish(wal);
821}

823void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
824{
__intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
826}

828int intel_engine_emit_ctx_wa(struct i915_request *rq)
829{
struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
struct i915_wa *wa;
unsigned int i;
u32 *cs;
int ret;

if (wal->count == 0)
return 0;

ret = rq->engine->emit_flush(rq, EMIT_BARRIER((1UL << (0)) | (1UL << (1))));
if (ret)
return ret;

cs = intel_ring_begin(rq, (wal->count * 2 + 2));
if (IS_ERR(cs))
return PTR_ERR(cs);

*cs++ = MI_LOAD_REGISTER_IMM(wal->count)(((0x0) << 29) | (0x22) << 23 | (2*(wal->count
)-1));
for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
*cs++ = i915_mmio_reg_offset(wa->reg);
*cs++ = wa->set;
}
*cs++ = MI_NOOP(((0x0) << 29) | (0) << 23 | (0));

intel_ring_advance(rq, cs);

ret = rq->engine->emit_flush(rq, EMIT_BARRIER((1UL << (0)) | (1UL << (1))));
if (ret)
return ret;

return 0;
861}

863static void
864gen4_gt_workarounds_init(struct intel_gt *gt,
	 struct i915_wa_list *wal)
866{
/* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
wa_masked_dis(wal, CACHE_MODE_0((const i915_reg_t){ .reg = (0x2120) }), RC_OP_FLUSH_ENABLE(1 << 0));
869}

871static void
872g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
873{
gen4_gt_workarounds_init(gt, wal);

/* WaDisableRenderCachePipelinedFlush:g4x,ilk */
wa_masked_en(wal, CACHE_MODE_0((const i915_reg_t){ .reg = (0x2120) }), CM0_PIPELINED_RENDER_FLUSH_DISABLE(1 << 8));
878}

880static void
881ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
882{
g4x_gt_workarounds_init(gt, wal);

wa_masked_en(wal, _3D_CHICKEN2((const i915_reg_t){ .reg = (0x208c) }), _3D_CHICKEN2_WM_READ_PIPELINED(1 << 14));
886}

888static void
889snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
890{
891}

893static void
894ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
895{
/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
wa_masked_dis(wal,
      GEN7_COMMON_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0x7010) }),
      GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC(1 << 10));

/* WaApplyL3ControlAndL3ChickenMode:ivb */
wa_write(wal, GEN7_L3CNTLREG1((const i915_reg_t){ .reg = (0xb01c) }), GEN7_WA_FOR_GEN7_L3_CONTROL0x3C47FF8C);
wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER((const i915_reg_t){ .reg = (0xb030) }), GEN7_WA_L3_CHICKEN_MODE0x20000000);

/* WaForceL3Serialization:ivb */
wa_write_clr(wal, GEN7_L3SQCREG4((const i915_reg_t){ .reg = (0xb034) }), L3SQ_URB_READ_CAM_MATCH_DISABLE(1 << 27));
907}

909static void
910vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
911{
/* WaForceL3Serialization:vlv */
wa_write_clr(wal, GEN7_L3SQCREG4((const i915_reg_t){ .reg = (0xb034) }), L3SQ_URB_READ_CAM_MATCH_DISABLE(1 << 27));

/*
* WaIncreaseL3CreditsForVLVB0:vlv
* This is the hardware default actually.
*/
wa_write(wal, GEN7_L3SQCREG1((const i915_reg_t){ .reg = (0xb010) }), VLV_B0_WA_L3SQCREG1_VALUE0x00D30000);
920}

922static void
923hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
924{
/* L3 caching of data atomics doesn't work -- disable it. */
wa_write(wal, HSW_SCRATCH1((const i915_reg_t){ .reg = (0xb038) }), HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE(1 << 27));

wa_add(wal,
      HSW_ROW_CHICKEN3((const i915_reg_t){ .reg = (0xe49c) }), 0,
      _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE)({ typeof((1 << 6)) _a = ((1 << 6)); ({ if (__builtin_constant_p
(_a)) do { } while (0); if (__builtin_constant_p(_a)) do { } while
 (0); if (__builtin_constant_p(_a) && __builtin_constant_p
(_a)) do { } while (0); ((_a) << 16 | (_a)); }); }),
      0 /* XXX does this reg exist? */, true1);

/* WaVSRefCountFullforceMissDisable:hsw */
wa_write_clr(wal, GEN7_FF_THREAD_MODE((const i915_reg_t){ .reg = (0x20a0) }), GEN7_FF_VS_REF_CNT_FFME(1 << 15));
935}

937static void
938gen9_wa_init_mcr(struct drm_i915_privateinteldrm_softc *i915, struct i915_wa_list *wal)
939{
const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu;
unsigned int slice, subslice;
u32 mcr, mcr_mask;

GEM_BUG_ON(GRAPHICS_VER(i915) != 9)((void)0);

/*
* WaProgramMgsrForCorrectSliceSpecificMmioReads:gen9,glk,kbl,cml
* Before any MMIO read into slice/subslice specific registers, MCR
* packet control register needs to be programmed to point to any
* enabled s/ss pair. Otherwise, incorrect values will be returned.
* This means each subsequent MMIO read will be forwarded to an
* specific s/ss combination, but this is OK since these registers
* are consistent across s/ss in almost all cases. In the rare
* occasions, such as INSTDONE, where this value is dependent
* on s/ss combo, the read should be done with read_subslice_reg.
*/
slice = ffs(sseu->slice_mask) - 1;
GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw))((void)0);
subslice = ffs(intel_sseu_get_hsw_subslices(sseu, slice));
GEM_BUG_ON(!subslice)((void)0);
subslice--;

/*
* We use GEN8_MCR..() macros to calculate the |mcr| value for
* Gen9 to address WaProgramMgsrForCorrectSliceSpecificMmioReads
*/
mcr = GEN8_MCR_SLICE(slice)(((slice) & 3) << 26) | GEN8_MCR_SUBSLICE(subslice)(((subslice) & 3) << 24);
mcr_mask = GEN8_MCR_SLICE_MASK(((3) & 3) << 26) | GEN8_MCR_SUBSLICE_MASK(((3) & 3) << 24);

drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr)__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "MCR slice:%d/subslice:%d = %x\n"
, slice, subslice, mcr);

wa_write_clr_set(wal, GEN8_MCR_SELECTOR((const i915_reg_t){ .reg = (0xfdc) }), mcr_mask, mcr);
973}

975static void
976gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
977{
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;

/* WaProgramMgsrForCorrectSliceSpecificMmioReads:glk,kbl,cml,gen9 */
gen9_wa_init_mcr(i915, wal);

/* WaDisableKillLogic:bxt,skl,kbl */
if (!IS_COFFEELAKE(i915)IS_PLATFORM(i915, INTEL_COFFEELAKE) && !IS_COMETLAKE(i915)IS_PLATFORM(i915, INTEL_COMETLAKE))
wa_write_or(wal,
	    GAM_ECOCHK((const i915_reg_t){ .reg = (0x4090) }),
	    ECOCHK_DIS_TLB(1 << 8));

if (HAS_LLC(i915)((&(i915)->__info)->has_llc)) {
/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 *
 * Must match Display Engine. See
 * WaCompressedResourceDisplayNewHashMode.
 */
wa_write_or(wal,
	    MMCD_MISC_CTRL((const i915_reg_t){ .reg = (0x4ddc) }),
	    MMCD_PCLA(1 << 31) | MMCD_HOTSPOT_EN(1 << 27));
}

/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
wa_write_or(wal,
    GAM_ECOCHK((const i915_reg_t){ .reg = (0x4090) }),
    BDW_DISABLE_HDC_INVALIDATION(1 << 25));
1004}

1006static void
1007skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1008{
gen9_gt_workarounds_init(gt, wal);

/* WaDisableGafsUnitClkGating:skl */
wa_write_or(wal,
    GEN7_UCGCTL4((const i915_reg_t){ .reg = (0x940c) }),
    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE(1 << 14));

/* WaInPlaceDecompressionHang:skl */
if (IS_SKL_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0)(IS_PLATFORM(gt->i915, INTEL_SKYLAKE) && (({ int __ret
 = !!((((&(gt->i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(gt->i915)->drm))->dev), "", "drm_WARN_ON(" "((&(gt->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(gt->i915
)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(gt->i915)->__runtime)->step.graphics_step) <
 (STEP_H0))))
wa_write_or(wal,
	    GEN9_GAMT_ECO_REG_RW_IA((const i915_reg_t){ .reg = (0x4ab0) }),
	    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS(1 << 18));
1021}

1023static void
1024kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1025{
gen9_gt_workarounds_init(gt, wal);

/* WaDisableDynamicCreditSharing:kbl */
if (IS_KBL_GRAPHICS_STEP(gt->i915, 0, STEP_C0)(IS_PLATFORM(gt->i915, INTEL_KABYLAKE) && (({ int __ret
 = !!((((&(gt->i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(gt->i915)->drm))->dev), "", "drm_WARN_ON(" "((&(gt->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(gt->i915
)->__runtime)->step.graphics_step) >= (0) &&
 ((&(gt->i915)->__runtime)->step.graphics_step) <
 (STEP_C0))))
wa_write_or(wal,
	    GAMT_CHKN_BIT_REG((const i915_reg_t){ .reg = (0x4ab8) }),
	    GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING(1 << 28));

/* WaDisableGafsUnitClkGating:kbl */
wa_write_or(wal,
    GEN7_UCGCTL4((const i915_reg_t){ .reg = (0x940c) }),
    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE(1 << 14));

/* WaInPlaceDecompressionHang:kbl */
wa_write_or(wal,
    GEN9_GAMT_ECO_REG_RW_IA((const i915_reg_t){ .reg = (0x4ab0) }),
    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS(1 << 18));
1043}

1045static void
1046glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1047{
gen9_gt_workarounds_init(gt, wal);
1049}

1051static void
1052cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1053{
gen9_gt_workarounds_init(gt, wal);

/* WaDisableGafsUnitClkGating:cfl */
wa_write_or(wal,
    GEN7_UCGCTL4((const i915_reg_t){ .reg = (0x940c) }),
    GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE(1 << 14));

/* WaInPlaceDecompressionHang:cfl */
wa_write_or(wal,
    GEN9_GAMT_ECO_REG_RW_IA((const i915_reg_t){ .reg = (0x4ab0) }),
    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS(1 << 18));
1065}

1067static void __set_mcr_steering(struct i915_wa_list *wal,
	       i915_reg_t steering_reg,
	       unsigned int slice, unsigned int subslice)
1070{
u32 mcr, mcr_mask;

mcr = GEN11_MCR_SLICE(slice)(((slice) & 0xf) << 27) | GEN11_MCR_SUBSLICE(subslice)(((subslice) & 0x7) << 24);
mcr_mask = GEN11_MCR_SLICE_MASK(((0xf) & 0xf) << 27) | GEN11_MCR_SUBSLICE_MASK(((0x7) & 0x7) << 24);

wa_write_clr_set(wal, steering_reg, mcr_mask, mcr);
1077}

1079static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal,
	 unsigned int slice, unsigned int subslice)
1081{
struct drm_printer p = drm_debug_printer("MCR Steering:");

__set_mcr_steering(wal, GEN8_MCR_SELECTOR((const i915_reg_t){ .reg = (0xfdc) }), slice, subslice);

gt->default_steering.groupid = slice;
gt->default_steering.instanceid = subslice;

if (drm_debug_enabled(DRM_UT_DRIVER)drm_debug_enabled_raw(DRM_UT_DRIVER))
intel_gt_mcr_report_steering(&p, gt, false0);
1091}

1093static void
1094icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1095{
const struct sseu_dev_info *sseu = &gt->info.sseu;
unsigned int subslice;

GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11)((void)0);
GEM_BUG_ON(hweight8(sseu->slice_mask) > 1)((void)0);

/*
* Although a platform may have subslices, we need to always steer
* reads to the lowest instance that isn't fused off.  When Render
* Power Gating is enabled, grabbing forcewake will only power up a
* single subslice (the "minconfig") if there isn't a real workload
* that needs to be run; this means that if we steer register reads to
* one of the higher subslices, we run the risk of reading back 0's or
* random garbage.
*/
subslice = __ffs(intel_sseu_get_hsw_subslices(sseu, 0))__builtin_ctzl(intel_sseu_get_hsw_subslices(sseu, 0));

/*
* If the subslice we picked above also steers us to a valid L3 bank,
* then we can just rely on the default steering and won't need to
* worry about explicitly re-steering L3BANK reads later.
*/
if (gt->info.l3bank_mask & BIT(subslice)(1UL << (subslice)))
gt->steering_table[L3BANK] = NULL((void *)0);

__add_mcr_wa(gt, wal, 0, subslice);
1122}

1124static void
1125xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1126{
const struct sseu_dev_info *sseu = &gt->info.sseu;
unsigned long slice, subslice = 0, slice_mask = 0;
u32 lncf_mask = 0;
int i;

/*
* On Xe_HP the steering increases in complexity. There are now several
* more units that require steering and we're not guaranteed to be able
* to find a common setting for all of them. These are:
* - GSLICE (fusable)
* - DSS (sub-unit within gslice; fusable)
* - L3 Bank (fusable)
* - MSLICE (fusable)
* - LNCF (sub-unit within mslice; always present if mslice is present)
*
* We'll do our default/implicit steering based on GSLICE (in the
* sliceid field) and DSS (in the subsliceid field).  If we can
* find overlap between the valid MSLICE and/or LNCF values with
* a suitable GSLICE, then we can just re-use the default value and
* skip and explicit steering at runtime.
*
* We only need to look for overlap between GSLICE/MSLICE/LNCF to find
* a valid sliceid value.  DSS steering is the only type of steering
* that utilizes the 'subsliceid' bits.
*
* Also note that, even though the steering domain is called "GSlice"
* and it is encoded in the register using the gslice format, the spec
* says that the combined (geometry | compute) fuse should be used to
* select the steering.
*/

/* Find the potential gslice candidates */
slice_mask = intel_slicemask_from_xehp_dssmask(sseu->subslice_mask,
				       GEN_DSS_PER_GSLICE4);

/*
* Find the potential LNCF candidates.  Either LNCF within a valid
* mslice is fine.
*/
for_each_set_bit(i, &gt->info.mslice_mask, GEN12_MAX_MSLICES)for ((i) = find_first_bit((&gt->info.mslice_mask), (4)
); (i) < (4); (i) = find_next_bit((&gt->info.mslice_mask
), (4), (i) + 1))
lncf_mask |= (0x3 << (i * 2));

/*
* Are there any sliceid values that work for both GSLICE and LNCF
* steering?
*/
if (slice_mask & lncf_mask) {
slice_mask &= lncf_mask;
gt->steering_table[LNCF] = NULL((void *)0);
}

/* How about sliceid values that also work for MSLICE steering? */
if (slice_mask & gt->info.mslice_mask) {
slice_mask &= gt->info.mslice_mask;
gt->steering_table[MSLICE] = NULL((void *)0);
}

slice = __ffs(slice_mask)__builtin_ctzl(slice_mask);
subslice = intel_sseu_find_first_xehp_dss(sseu, GEN_DSS_PER_GSLICE4, slice) %
GEN_DSS_PER_GSLICE4;

__add_mcr_wa(gt, wal, slice, subslice);

/*
* SQIDI ranges are special because they use different steering
* registers than everything else we work with.  On XeHP SDV and
* DG2-G10, any value in the steering registers will work fine since
* all instances are present, but DG2-G11 only has SQIDI instances at
* ID's 2 and 3, so we need to steer to one of those.  For simplicity
* we'll just steer to a hardcoded "2" since that value will work
* everywhere.
*/
__set_mcr_steering(wal, MCFG_MCR_SELECTOR((const i915_reg_t){ .reg = (0xfd0) }), 0, 2);
__set_mcr_steering(wal, SF_MCR_SELECTOR((const i915_reg_t){ .reg = (0xfd8) }), 0, 2);
1201}

1203static void
1204pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1205{
unsigned int dss;

/*
* Setup implicit steering for COMPUTE and DSS ranges to the first
* non-fused-off DSS.  All other types of MCR registers will be
* explicitly steered.
*/
dss = intel_sseu_find_first_xehp_dss(&gt->info.sseu, 0, 0);
__add_mcr_wa(gt, wal, dss / GEN_DSS_PER_CSLICE8, dss % GEN_DSS_PER_CSLICE8);
1215}

1217static void
1218icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1219{
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;

icl_wa_init_mcr(gt, wal);

/* WaModifyGamTlbPartitioning:icl */
wa_write_clr_set(wal,
	 GEN11_GACB_PERF_CTRL((const i915_reg_t){ .reg = (0x4b80) }),
	 GEN11_HASH_CTRL_MASK(0x3 << 12 | 0xf << 0),
	 GEN11_HASH_CTRL_BIT0(1 << 0) | GEN11_HASH_CTRL_BIT4(1 << 12));

/* Wa_1405766107:icl
* Formerly known as WaCL2SFHalfMaxAlloc
*/
wa_write_or(wal,
    GEN11_LSN_UNSLCVC((const i915_reg_t){ .reg = (0xb43c) }),
    GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC(1 << 7) |
    GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC(1 << 9));

/* Wa_220166154:icl
* Formerly known as WaDisCtxReload
*/
wa_write_or(wal,
    GEN8_GAMW_ECO_DEV_RW_IA((const i915_reg_t){ .reg = (0x4080) }),
    GAMW_ECO_DEV_CTX_RELOAD_DISABLE(1 << 7));

/* Wa_1406463099:icl
* Formerly known as WaGamTlbPendError
*/
wa_write_or(wal,
    GAMT_CHKN_BIT_REG((const i915_reg_t){ .reg = (0x4ab8) }),
    GAMT_CHKN_DISABLE_L3_COH_PIPE(1 << 31));

/*
* Wa_1408615072:icl,ehl  (vsunit)
* Wa_1407596294:icl,ehl  (hsunit)
*/
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9434) }),
    VSUNIT_CLKGATE_DIS((u32)((1UL << (3)) + 0)) | HSUNIT_CLKGATE_DIS((u32)((1UL << (8)) + 0)));

/* Wa_1407352427:icl,ehl */
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2((const i915_reg_t){ .reg = (0x94e4) }),
    PSDUNIT_CLKGATE_DIS((u32)((1UL << (5)) + 0)));

/* Wa_1406680159:icl,ehl */
wa_write_or(wal,
    SUBSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9524) }),
    GWUNIT_CLKGATE_DIS((u32)((1UL << (16)) + 0)));

/* Wa_1607087056:icl,ehl,jsl */
if (IS_ICELAKE(i915)IS_PLATFORM(i915, INTEL_ICELAKE) ||
   IS_JSL_EHL_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)((IS_PLATFORM(i915, INTEL_JASPERLAKE) || IS_PLATFORM(i915, INTEL_ELKHARTLAKE
)) && (({ int __ret = !!((((&(i915)->__runtime
)->step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(i915)->drm))->dev), ""
, "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal,
	    SLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x94d4) }),
	    L3_CLKGATE_DIS((u32)((1UL << (16)) + 0)) | L3_CR2X_CLKGATE_DIS((u32)((1UL << (17)) + 0)));

/*
* This is not a documented workaround, but rather an optimization
* to reduce sampler power.
*/
wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN((const i915_reg_t){ .reg = (0x9550) }), DFR_DISABLE(1 << 9));
1280}

1282/*
* Though there are per-engine instances of these registers,
* they retain their value through engine resets and should
* only be provided on the GT workaround list rather than
* the engine-specific workaround list.
*/
1288static void
1289wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal)
1290{
struct intel_engine_cs *engine;
int id;

for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
if (engine->class != VIDEO_DECODE_CLASS1 ||
    (engine->instance % 2))
	continue;

wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x3f10)
 }),
	    IECPUNIT_CLKGATE_DIS((u32)((1UL << (22)) + 0)));
}
1302}

1304static void
1305gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1306{
icl_wa_init_mcr(gt, wal);

/* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
wa_14011060649(gt, wal);

/* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN((const i915_reg_t){ .reg = (0x9550) }), DFR_DISABLE(1 << 9));
1314}

1316static void
1317tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1318{
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;

gen12_gt_workarounds_init(gt, wal);

/* Wa_1409420604:tgl */
if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_TIGERLAKE, (0)) && (({ int
 __ret = !!((((&(i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal,
	    SUBSLICE_UNIT_LEVEL_CLKGATE2((const i915_reg_t){ .reg = (0x9528) }),
	    CPSSUNIT_CLKGATE_DIS((u32)((1UL << (9)) + 0)));

/* Wa_1607087056:tgl also know as BUG:1409180338 */
if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_TIGERLAKE, (0)) && (({ int
 __ret = !!((((&(i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal,
	    SLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x94d4) }),
	    L3_CLKGATE_DIS((u32)((1UL << (16)) + 0)) | L3_CR2X_CLKGATE_DIS((u32)((1UL << (17)) + 0)));

/* Wa_1408615072:tgl[a0] */
if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_TIGERLAKE, (0)) && (({ int
 __ret = !!((((&(i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2((const i915_reg_t){ .reg = (0x94e4) }),
	    VSUNIT_CLKGATE_DIS_TGL((u32)((1UL << (19)) + 0)));
1339}

1341static void
1342dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1343{
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;

gen12_gt_workarounds_init(gt, wal);

/* Wa_1607087056:dg1 */
if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_PLATFORM(i915, INTEL_DG1) && (({ int __ret = !!((
((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal,
	    SLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x94d4) }),
	    L3_CLKGATE_DIS((u32)((1UL << (16)) + 0)) | L3_CR2X_CLKGATE_DIS((u32)((1UL << (17)) + 0)));

/* Wa_1409420604:dg1 */
if (IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1))
wa_write_or(wal,
	    SUBSLICE_UNIT_LEVEL_CLKGATE2((const i915_reg_t){ .reg = (0x9528) }),
	    CPSSUNIT_CLKGATE_DIS((u32)((1UL << (9)) + 0)));

/* Wa_1408615072:dg1 */
/* Empirical testing shows this register is unaffected by engine reset. */
if (IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1))
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2((const i915_reg_t){ .reg = (0x94e4) }),
	    VSUNIT_CLKGATE_DIS_TGL((u32)((1UL << (19)) + 0)));
1365}

1367static void
1368xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1369{
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;

xehp_init_mcr(gt, wal);

/* Wa_1409757795:xehpsdv */
wa_write_or(wal, SCCGCTL94DC((const i915_reg_t){ .reg = (0x94dc) }), CG3DDISURB((u32)((1UL << (14)) + 0)));

/* Wa_16011155590:xehpsdv */
if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_PLATFORM(i915, INTEL_XEHPSDV) && (({ int __ret = !
!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9434) }),
	    TSGUNIT_CLKGATE_DIS((u32)((1UL << (17)) + 0)));

/* Wa_14011780169:xehpsdv */
if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_B0, STEP_FOREVER)(IS_PLATFORM(i915, INTEL_XEHPSDV) && (({ int __ret = !
!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_FOREVER
)))) {
wa_write_or(wal, UNSLCGCTL9440((const i915_reg_t){ .reg = (0x9440) }), GAMTLBOACS_CLKGATE_DIS((u32)((1UL << (28)) + 0)) |
	    GAMTLBVDBOX7_CLKGATE_DIS((u32)((1UL << (22)) + 0)) |
	    GAMTLBVDBOX6_CLKGATE_DIS((u32)((1UL << (26)) + 0)) |
	    GAMTLBVDBOX5_CLKGATE_DIS((u32)((1UL << (27)) + 0)) |
	    GAMTLBVDBOX4_CLKGATE_DIS((u32)((1UL << (23)) + 0)) |
	    GAMTLBVDBOX3_CLKGATE_DIS((u32)((1UL << (24)) + 0)) |
	    GAMTLBVDBOX2_CLKGATE_DIS((u32)((1UL << (21)) + 0)) |
	    GAMTLBVDBOX1_CLKGATE_DIS((u32)((1UL << (6)) + 0)) |
	    GAMTLBVDBOX0_CLKGATE_DIS((u32)((1UL << (17)) + 0)) |
	    GAMTLBKCR_CLKGATE_DIS((u32)((1UL << (16)) + 0)) |
	    GAMTLBGUC_CLKGATE_DIS((u32)((1UL << (15)) + 0)) |
	    GAMTLBBLT_CLKGATE_DIS((u32)((1UL << (14)) + 0)));
wa_write_or(wal, UNSLCGCTL9444((const i915_reg_t){ .reg = (0x9444) }), GAMTLBGFXA0_CLKGATE_DIS((u32)((1UL << (30)) + 0)) |
	    GAMTLBGFXA1_CLKGATE_DIS((u32)((1UL << (29)) + 0)) |
	    GAMTLBCOMPA0_CLKGATE_DIS((u32)((1UL << (28)) + 0)) |
	    GAMTLBCOMPA1_CLKGATE_DIS((u32)((1UL << (27)) + 0)) |
	    GAMTLBCOMPB0_CLKGATE_DIS((u32)((1UL << (26)) + 0)) |
	    GAMTLBCOMPB1_CLKGATE_DIS((u32)((1UL << (25)) + 0)) |
	    GAMTLBCOMPC0_CLKGATE_DIS((u32)((1UL << (24)) + 0)) |
	    GAMTLBCOMPC1_CLKGATE_DIS((u32)((1UL << (23)) + 0)) |
	    GAMTLBCOMPD0_CLKGATE_DIS((u32)((1UL << (22)) + 0)) |
	    GAMTLBCOMPD1_CLKGATE_DIS((u32)((1UL << (21)) + 0)) |
	    GAMTLBMERT_CLKGATE_DIS((u32)((1UL << (20)) + 0))   |
	    GAMTLBVEBOX3_CLKGATE_DIS((u32)((1UL << (19)) + 0)) |
	    GAMTLBVEBOX2_CLKGATE_DIS((u32)((1UL << (18)) + 0)) |
	    GAMTLBVEBOX1_CLKGATE_DIS((u32)((1UL << (17)) + 0)) |
	    GAMTLBVEBOX0_CLKGATE_DIS((u32)((1UL << (16)) + 0)));
}

/* Wa_16012725990:xehpsdv */
if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER)(IS_PLATFORM(i915, INTEL_XEHPSDV) && (({ int __ret = !
!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A1) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_FOREVER
))))
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9434) }), VFUNIT_CLKGATE_DIS((u32)((1UL << (20)) + 0)));

/* Wa_14011060649:xehpsdv */
wa_14011060649(gt, wal);
1419}

1421static void
1422dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1423{
struct intel_engine_cs *engine;
int id;

xehp_init_mcr(gt, wal);

/* Wa_14011060649:dg2 */
wa_14011060649(gt, wal);

/*
* Although there are per-engine instances of these registers,
* they technically exist outside the engine itself and are not
* impacted by engine resets.  Furthermore, they're part of the
* GuC blacklist so trying to treat them as engine workarounds
* will result in GuC initialization failure and a wedged GPU.
*/
for_each_engine(engine, gt, id)for ((id) = 0; (id) < I915_NUM_ENGINES; (id)++) if (!((engine
) = (gt)->engine[(id)])) {} else {
if (engine->class != VIDEO_DECODE_CLASS1)
	continue;

/* Wa_16010515920:dg2_g10 */
if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(gt->i915, INTEL_DG2, 0) && (({ int
 __ret = !!((((&(gt->i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(gt->i915)->drm))->dev), "", "drm_WARN_ON(" "((&(gt->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(gt->i915
)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(gt->i915)->__runtime)->step.graphics_step) <
 (STEP_B0))))
	wa_write_or(wal, VDBOX_CGCTL3F18(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x3f18)
 }),
		    ALNUNIT_CLKGATE_DIS((u32)((1UL << (13)) + 0)));
}

if (IS_DG2_G10(gt->i915)IS_SUBPLATFORM(gt->i915, INTEL_DG2, 0)) {
/* Wa_22010523718:dg2 */
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9434) }),
	    CG3DDISCFEG_CLKGATE_DIS((u32)((1UL << (17)) + 0)));

/* Wa_14011006942:dg2 */
wa_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9524) }),
	    DSS_ROUTER_CLKGATE_DIS((u32)((1UL << (28)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(gt->i915, INTEL_DG2, 0) && (({ int
 __ret = !!((((&(gt->i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(gt->i915)->drm))->dev), "", "drm_WARN_ON(" "((&(gt->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(gt->i915
)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(gt->i915)->__runtime)->step.graphics_step) <
 (STEP_B0)))) {
/* Wa_14010948348:dg2_g10 */
wa_write_or(wal, UNSLCGCTL9430((const i915_reg_t){ .reg = (0x9430) }), MSQDUNIT_CLKGATE_DIS((u32)((1UL << (3)) + 0)));

/* Wa_14011037102:dg2_g10 */
wa_write_or(wal, UNSLCGCTL9444((const i915_reg_t){ .reg = (0x9444) }), LTCDD_CLKGATE_DIS((u32)((1UL << (10)) + 0)));

/* Wa_14011371254:dg2_g10 */
wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x94d4) }), NODEDSS_CLKGATE_DIS((u32)((1UL << (12)) + 0)));

/* Wa_14011431319:dg2_g10 */
wa_write_or(wal, UNSLCGCTL9440((const i915_reg_t){ .reg = (0x9440) }), GAMTLBOACS_CLKGATE_DIS((u32)((1UL << (28)) + 0)) |
	    GAMTLBVDBOX7_CLKGATE_DIS((u32)((1UL << (22)) + 0)) |
	    GAMTLBVDBOX6_CLKGATE_DIS((u32)((1UL << (26)) + 0)) |
	    GAMTLBVDBOX5_CLKGATE_DIS((u32)((1UL << (27)) + 0)) |
	    GAMTLBVDBOX4_CLKGATE_DIS((u32)((1UL << (23)) + 0)) |
	    GAMTLBVDBOX3_CLKGATE_DIS((u32)((1UL << (24)) + 0)) |
	    GAMTLBVDBOX2_CLKGATE_DIS((u32)((1UL << (21)) + 0)) |
	    GAMTLBVDBOX1_CLKGATE_DIS((u32)((1UL << (6)) + 0)) |
	    GAMTLBVDBOX0_CLKGATE_DIS((u32)((1UL << (17)) + 0)) |
	    GAMTLBKCR_CLKGATE_DIS((u32)((1UL << (16)) + 0)) |
	    GAMTLBGUC_CLKGATE_DIS((u32)((1UL << (15)) + 0)) |
	    GAMTLBBLT_CLKGATE_DIS((u32)((1UL << (14)) + 0)));
wa_write_or(wal, UNSLCGCTL9444((const i915_reg_t){ .reg = (0x9444) }), GAMTLBGFXA0_CLKGATE_DIS((u32)((1UL << (30)) + 0)) |
	    GAMTLBGFXA1_CLKGATE_DIS((u32)((1UL << (29)) + 0)) |
	    GAMTLBCOMPA0_CLKGATE_DIS((u32)((1UL << (28)) + 0)) |
	    GAMTLBCOMPA1_CLKGATE_DIS((u32)((1UL << (27)) + 0)) |
	    GAMTLBCOMPB0_CLKGATE_DIS((u32)((1UL << (26)) + 0)) |
	    GAMTLBCOMPB1_CLKGATE_DIS((u32)((1UL << (25)) + 0)) |
	    GAMTLBCOMPC0_CLKGATE_DIS((u32)((1UL << (24)) + 0)) |
	    GAMTLBCOMPC1_CLKGATE_DIS((u32)((1UL << (23)) + 0)) |
	    GAMTLBCOMPD0_CLKGATE_DIS((u32)((1UL << (22)) + 0)) |
	    GAMTLBCOMPD1_CLKGATE_DIS((u32)((1UL << (21)) + 0)) |
	    GAMTLBMERT_CLKGATE_DIS((u32)((1UL << (20)) + 0))   |
	    GAMTLBVEBOX3_CLKGATE_DIS((u32)((1UL << (19)) + 0)) |
	    GAMTLBVEBOX2_CLKGATE_DIS((u32)((1UL << (18)) + 0)) |
	    GAMTLBVEBOX1_CLKGATE_DIS((u32)((1UL << (17)) + 0)) |
	    GAMTLBVEBOX0_CLKGATE_DIS((u32)((1UL << (16)) + 0)));

/* Wa_14010569222:dg2_g10 */
wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE((const i915_reg_t){ .reg = (0x9434) }),
	    GAMEDIA_CLKGATE_DIS((u32)((1UL << (11)) + 0)));

/* Wa_14011028019:dg2_g10 */
wa_write_or(wal, SSMCGCTL9530((const i915_reg_t){ .reg = (0x9530) }), RTFUNIT_CLKGATE_DIS((u32)((1UL << (18)) + 0)));
}

/* Wa_14014830051:dg2 */
wa_write_clr(wal, SARB_CHICKEN1((const i915_reg_t){ .reg = (0xe90c) }), COMP_CKN_IN((u32)((((~0UL) >> (64 - (30) - 1)) & ((~0UL) <<
 (29))) + 0)));

/*
* The following are not actually "workarounds" but rather
* recommended tuning settings documented in the bspec's
* performance guide section.
*/
wa_write_or(wal, GEN12_SQCM((const i915_reg_t){ .reg = (0x8724) }), EN_32B_ACCESS((u32)((1UL << (30)) + 0)));

/* Wa_14015795083 */
wa_write_clr(wal, GEN7_MISCCPCTL((const i915_reg_t){ .reg = (0x9424) }), GEN12_DOP_CLOCK_GATE_RENDER_ENABLE((u32)((1UL << (1)) + 0)));
1518}

1520static void
1521pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1522{
pvc_init_mcr(gt, wal);

/* Wa_14015795083 */
wa_write_clr(wal, GEN7_MISCCPCTL((const i915_reg_t){ .reg = (0x9424) }), GEN12_DOP_CLOCK_GATE_RENDER_ENABLE((u32)((1UL << (1)) + 0)));
1527}

1529static void
1530gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
1531{
struct drm_i915_privateinteldrm_softc *i915 = gt->i915;

if (IS_PONTEVECCHIO(i915)IS_PLATFORM(i915, INTEL_PONTEVECCHIO))
pvc_gt_workarounds_init(gt, wal);
else if (IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2))
dg2_gt_workarounds_init(gt, wal);
else if (IS_XEHPSDV(i915)IS_PLATFORM(i915, INTEL_XEHPSDV))
xehpsdv_gt_workarounds_init(gt, wal);
else if (IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1))
dg1_gt_workarounds_init(gt, wal);
else if (IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE))
tgl_gt_workarounds_init(gt, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 12)
gen12_gt_workarounds_init(gt, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 11)
icl_gt_workarounds_init(gt, wal);
else if (IS_COFFEELAKE(i915)IS_PLATFORM(i915, INTEL_COFFEELAKE) || IS_COMETLAKE(i915)IS_PLATFORM(i915, INTEL_COMETLAKE))
cfl_gt_workarounds_init(gt, wal);
else if (IS_GEMINILAKE(i915)IS_PLATFORM(i915, INTEL_GEMINILAKE))
glk_gt_workarounds_init(gt, wal);
else if (IS_KABYLAKE(i915)IS_PLATFORM(i915, INTEL_KABYLAKE))
kbl_gt_workarounds_init(gt, wal);
else if (IS_BROXTON(i915)IS_PLATFORM(i915, INTEL_BROXTON))
gen9_gt_workarounds_init(gt, wal);
else if (IS_SKYLAKE(i915)IS_PLATFORM(i915, INTEL_SKYLAKE))
skl_gt_workarounds_init(gt, wal);
else if (IS_HASWELL(i915)IS_PLATFORM(i915, INTEL_HASWELL))
hsw_gt_workarounds_init(gt, wal);
else if (IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW))
vlv_gt_workarounds_init(gt, wal);
else if (IS_IVYBRIDGE(i915)IS_PLATFORM(i915, INTEL_IVYBRIDGE))
ivb_gt_workarounds_init(gt, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 6)
snb_gt_workarounds_init(gt, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 5)
ilk_gt_workarounds_init(gt, wal);
else if (IS_G4X(i915)(IS_PLATFORM(i915, INTEL_G45) || IS_PLATFORM(i915, INTEL_GM45
)))
g4x_gt_workarounds_init(gt, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 4)
gen4_gt_workarounds_init(gt, wal);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) <= 8)
;
else
MISSING_CASE(GRAPHICS_VER(i915))({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "((&(i915)->__runtime)->graphics.ip.ver)", (long)
(((&(i915)->__runtime)->graphics.ip.ver))); __builtin_expect
(!!(__ret), 0); });
1576}

1578void intel_gt_init_workarounds(struct intel_gt *gt)
1579{
struct i915_wa_list *wal = &gt->wa_list;

wa_init_start(wal, "GT", "global");
gt_init_workarounds(gt, wal);
wa_init_finish(wal);
1585}

1587static enum forcewake_domains
1588wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1589{
enum forcewake_domains fw = 0;
struct i915_wa *wa;
unsigned int i;

for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
fw |= intel_uncore_forcewake_for_reg(uncore,
				     wa->reg,
				     FW_REG_READ(1) |
				     FW_REG_WRITE(2));

return fw;
1601}

1603static bool_Bool
1604wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1605{
if ((cur ^ wa->set) & wa->read) {
DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",__drm_err("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n"
, name, from, i915_mmio_reg_offset(wa->reg), cur, cur &
 wa->read, wa->set & wa->read)
	  name, from, i915_mmio_reg_offset(wa->reg),__drm_err("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n"
, name, from, i915_mmio_reg_offset(wa->reg), cur, cur &
 wa->read, wa->set & wa->read)
	  cur, cur & wa->read, wa->set & wa->read)__drm_err("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n"
, name, from, i915_mmio_reg_offset(wa->reg), cur, cur &
 wa->read, wa->set & wa->read);

return false0;
}

return true1;
1615}

1617static void
1618wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal)
1619{
struct intel_uncore *uncore = gt->uncore;
enum forcewake_domains fw;
unsigned long flags;
struct i915_wa *wa;
unsigned int i;

if (!wal->count)
return;

fw = wal_get_fw_for_rmw(uncore, wal);

spin_lock_irqsave(&uncore->lock, flags)do { flags = 0; mtx_enter(&uncore->lock); } while (0);
intel_uncore_forcewake_get__locked(uncore, fw);

for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
u32 val, old = 0;

/* open-coded rmw due to steering */
old = wa->clr ? intel_gt_mcr_read_any_fw(gt, wa->reg) : 0;
val = (old & ~wa->clr) | wa->set;
if (val != old || !wa->clr)
	intel_uncore_write_fw(uncore, wa->reg, val)__raw_uncore_write32(uncore, wa->reg, val);

if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)0)
	wa_verify(wa, intel_gt_mcr_read_any_fw(gt, wa->reg),
		  wal->name, "application");
}

intel_uncore_forcewake_put__locked(uncore, fw);
spin_unlock_irqrestore(&uncore->lock, flags)do { (void)(flags); mtx_leave(&uncore->lock); } while (
0);
1650}

1652void intel_gt_apply_workarounds(struct intel_gt *gt)
1653{
wa_list_apply(gt, &gt->wa_list);
1655}

1657static bool_Bool wa_list_verify(struct intel_gt *gt,
	   const struct i915_wa_list *wal,
	   const char *from)
1660{
struct intel_uncore *uncore = gt->uncore;
struct i915_wa *wa;
enum forcewake_domains fw;
unsigned long flags;
unsigned int i;
bool_Bool ok = true1;

fw = wal_get_fw_for_rmw(uncore, wal);

spin_lock_irqsave(&uncore->lock, flags)do { flags = 0; mtx_enter(&uncore->lock); } while (0);
intel_uncore_forcewake_get__locked(uncore, fw);

for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
ok &= wa_verify(wa,
		intel_gt_mcr_read_any_fw(gt, wa->reg),
		wal->name, from);

intel_uncore_forcewake_put__locked(uncore, fw);
spin_unlock_irqrestore(&uncore->lock, flags)do { (void)(flags); mtx_leave(&uncore->lock); } while (
0);

return ok;
1682}

1684bool_Bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1685{
return wa_list_verify(gt, &gt->wa_list, from);
1687}

1689__maybe_unused__attribute__((__unused__))
1690static bool_Bool is_nonpriv_flags_valid(u32 flags)
1691{
/* Check only valid flag bits are set */
if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID((3 << 0) | (3 << 28) | ((u32)((1UL << (30)
) + 0))))
return false0;

/* NB: Only 3 out of 4 enum values are valid for access field */
if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK(3 << 28)) ==
   RING_FORCE_TO_NONPRIV_ACCESS_INVALID(3 << 28))
return false0;

return true1;
1702}

1704static void
1705whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1706{
struct i915_wa wa = {
.reg = reg
};

if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)({ ((void)0); 0; }))
return;

if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags))({ ((void)0); 0; }))
return;

wa.reg.reg |= flags;
_wa_add(wal, &wa);
1719}

1721static void
1722whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1723{
whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW(0 << 28));
1725}

1727static void gen9_whitelist_build(struct i915_wa_list *w)
1728{
/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
whitelist_reg(w, GEN9_CTX_PREEMPT_REG((const i915_reg_t){ .reg = (0x2248) }));

/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
whitelist_reg(w, GEN8_CS_CHICKEN1((const i915_reg_t){ .reg = (0x2580) }));

/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
whitelist_reg(w, GEN8_HDC_CHICKEN1((const i915_reg_t){ .reg = (0x7304) }));

/* WaSendPushConstantsFromMMIO:skl,bxt */
whitelist_reg(w, COMMON_SLICE_CHICKEN2((const i915_reg_t){ .reg = (0x7014) }));
1740}

1742static void skl_whitelist_build(struct intel_engine_cs *engine)
1743{
struct i915_wa_list *w = &engine->whitelist;

if (engine->class != RENDER_CLASS0)
return;

gen9_whitelist_build(w);

/* WaDisableLSQCROPERFforOCL:skl */
whitelist_reg(w, GEN8_L3SQCREG4((const i915_reg_t){ .reg = (0xb118) }));
1753}

1755static void bxt_whitelist_build(struct intel_engine_cs *engine)
1756{
if (engine->class != RENDER_CLASS0)
return;

gen9_whitelist_build(&engine->whitelist);
1761}

1763static void kbl_whitelist_build(struct intel_engine_cs *engine)
1764{
struct i915_wa_list *w = &engine->whitelist;

if (engine->class != RENDER_CLASS0)
return;

gen9_whitelist_build(w);

/* WaDisableLSQCROPERFforOCL:kbl */
whitelist_reg(w, GEN8_L3SQCREG4((const i915_reg_t){ .reg = (0xb118) }));
1774}

1776static void glk_whitelist_build(struct intel_engine_cs *engine)
1777{
struct i915_wa_list *w = &engine->whitelist;

if (engine->class != RENDER_CLASS0)
return;

gen9_whitelist_build(w);

/* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1((const i915_reg_t){ .reg = (0x731c) }));
1787}

1789static void cfl_whitelist_build(struct intel_engine_cs *engine)
1790{
struct i915_wa_list *w = &engine->whitelist;

if (engine->class != RENDER_CLASS0)
return;

gen9_whitelist_build(w);

/*
* WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
*
* This covers 4 register which are next to one another :
*   - PS_INVOCATION_COUNT
*   - PS_INVOCATION_COUNT_UDW
*   - PS_DEPTH_COUNT
*   - PS_DEPTH_COUNT_UDW
*/
whitelist_reg_ext(w, PS_INVOCATION_COUNT((const i915_reg_t){ .reg = (0x2348) }),
	  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28) |
	  RING_FORCE_TO_NONPRIV_RANGE_4(1 << 0));
1810}

1812static void allow_read_ctx_timestamp(struct intel_engine_cs *engine)
1813{
struct i915_wa_list *w = &engine->whitelist;

if (engine->class != RENDER_CLASS0)
whitelist_reg_ext(w,
		  RING_CTX_TIMESTAMP(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x3a8) }
),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28));
1820}

1822static void cml_whitelist_build(struct intel_engine_cs *engine)
1823{
allow_read_ctx_timestamp(engine);

cfl_whitelist_build(engine);
1827}

1829static void icl_whitelist_build(struct intel_engine_cs *engine)
1830{
struct i915_wa_list *w = &engine->whitelist;

allow_read_ctx_timestamp(engine);

switch (engine->class) {
case RENDER_CLASS0:
/* WaAllowUMDToModifyHalfSliceChicken7:icl */
whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7((const i915_reg_t){ .reg = (0xe194) }));

/* WaAllowUMDToModifySamplerMode:icl */
whitelist_reg(w, GEN10_SAMPLER_MODE((const i915_reg_t){ .reg = (0xe18c) }));

/* WaEnableStateCacheRedirectToCS:icl */
whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1((const i915_reg_t){ .reg = (0x731c) }));

/*
 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
 *
 * This covers 4 register which are next to one another :
 *   - PS_INVOCATION_COUNT
 *   - PS_INVOCATION_COUNT_UDW
 *   - PS_DEPTH_COUNT
 *   - PS_DEPTH_COUNT_UDW
 */
whitelist_reg_ext(w, PS_INVOCATION_COUNT((const i915_reg_t){ .reg = (0x2348) }),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28) |
		  RING_FORCE_TO_NONPRIV_RANGE_4(1 << 0));
break;

case VIDEO_DECODE_CLASS1:
/* hucStatusRegOffset */
whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base)((const i915_reg_t){ .reg = (0x2000 + engine->mmio_base) }
),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28));
/* hucUKernelHdrInfoRegOffset */
whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base)((const i915_reg_t){ .reg = (0x2014 + engine->mmio_base) }
),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28));
/* hucStatus2RegOffset */
whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base)((const i915_reg_t){ .reg = (0x23B0 + engine->mmio_base) }
),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28));
break;

default:
break;
}
1875}

1877static void tgl_whitelist_build(struct intel_engine_cs *engine)
1878{
struct i915_wa_list *w = &engine->whitelist;

allow_read_ctx_timestamp(engine);

switch (engine->class) {
case RENDER_CLASS0:
/*
 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
 * Wa_1408556865:tgl
 *
 * This covers 4 registers which are next to one another :
 *   - PS_INVOCATION_COUNT
 *   - PS_INVOCATION_COUNT_UDW
 *   - PS_DEPTH_COUNT
 *   - PS_DEPTH_COUNT_UDW
 */
whitelist_reg_ext(w, PS_INVOCATION_COUNT((const i915_reg_t){ .reg = (0x2348) }),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28) |
		  RING_FORCE_TO_NONPRIV_RANGE_4(1 << 0));

/*
 * Wa_1808121037:tgl
 * Wa_14012131227:dg1
 * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p
 */
whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0x7010) }));

/* Wa_1806527549:tgl */
whitelist_reg(w, HIZ_CHICKEN((const i915_reg_t){ .reg = (0x7018) }));
break;
default:
break;
}
1912}

1914static void dg1_whitelist_build(struct intel_engine_cs *engine)
1915{
struct i915_wa_list *w = &engine->whitelist;

tgl_whitelist_build(engine);

/* GEN:BUG:1409280441:dg1 */
if (IS_DG1_GRAPHICS_STEP(engine->i915, STEP_A0, STEP_B0)(IS_PLATFORM(engine->i915, INTEL_DG1) && (({ int __ret
 = !!((((&(engine->i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(engine->i915)->drm))->dev), "", "drm_WARN_ON("
 "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0))) &&
   (engine->class == RENDER_CLASS0 ||
    engine->class == COPY_ENGINE_CLASS3))
whitelist_reg_ext(w, RING_ID(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x8c) }
),
		  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28));
1926}

1928static void xehpsdv_whitelist_build(struct intel_engine_cs *engine)
1929{
allow_read_ctx_timestamp(engine);
1931}

1933static void dg2_whitelist_build(struct intel_engine_cs *engine)
1934{
struct i915_wa_list *w = &engine->whitelist;

allow_read_ctx_timestamp(engine);

switch (engine->class) {
case RENDER_CLASS0:
/*
 * Wa_1507100340:dg2_g10
 *
 * This covers 4 registers which are next to one another :
 *   - PS_INVOCATION_COUNT
 *   - PS_INVOCATION_COUNT_UDW
 *   - PS_DEPTH_COUNT
 *   - PS_DEPTH_COUNT_UDW
 */
if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0))))
	whitelist_reg_ext(w, PS_INVOCATION_COUNT((const i915_reg_t){ .reg = (0x2348) }),
			  RING_FORCE_TO_NONPRIV_ACCESS_RD(1 << 28) |
			  RING_FORCE_TO_NONPRIV_RANGE_4(1 << 0));

break;
case COMPUTE_CLASS5:
/* Wa_16011157294:dg2_g10 */
if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0))))
	whitelist_reg(w, GEN9_CTX_PREEMPT_REG((const i915_reg_t){ .reg = (0x2248) }));
break;
default:
break;
}
1964}

1966static void blacklist_trtt(struct intel_engine_cs *engine)
1967{
struct i915_wa_list *w = &engine->whitelist;

/*
* Prevent read/write access to [0x4400, 0x4600) which covers
* the TRTT range across all engines. Note that normally userspace
* cannot access the other engines' trtt control, but for simplicity
* we cover the entire range on each engine.
*/
whitelist_reg_ext(w, _MMIO(0x4400)((const i915_reg_t){ .reg = (0x4400) }),
	  RING_FORCE_TO_NONPRIV_DENY((u32)((1UL << (30)) + 0)) |
	  RING_FORCE_TO_NONPRIV_RANGE_64(3 << 0));
whitelist_reg_ext(w, _MMIO(0x4500)((const i915_reg_t){ .reg = (0x4500) }),
	  RING_FORCE_TO_NONPRIV_DENY((u32)((1UL << (30)) + 0)) |
	  RING_FORCE_TO_NONPRIV_RANGE_64(3 << 0));
1982}

1984static void pvc_whitelist_build(struct intel_engine_cs *engine)
1985{
allow_read_ctx_timestamp(engine);

/* Wa_16014440446:pvc */
blacklist_trtt(engine);
1990}

1992void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1993{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;
struct i915_wa_list *w = &engine->whitelist;

wa_init_start(w, "whitelist", engine->name);

if (IS_PONTEVECCHIO(i915)IS_PLATFORM(i915, INTEL_PONTEVECCHIO))
pvc_whitelist_build(engine);
else if (IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2))
dg2_whitelist_build(engine);
else if (IS_XEHPSDV(i915)IS_PLATFORM(i915, INTEL_XEHPSDV))
xehpsdv_whitelist_build(engine);
else if (IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1))
dg1_whitelist_build(engine);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 12)
tgl_whitelist_build(engine);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 11)
icl_whitelist_build(engine);
else if (IS_COMETLAKE(i915)IS_PLATFORM(i915, INTEL_COMETLAKE))
cml_whitelist_build(engine);
else if (IS_COFFEELAKE(i915)IS_PLATFORM(i915, INTEL_COFFEELAKE))
cfl_whitelist_build(engine);
else if (IS_GEMINILAKE(i915)IS_PLATFORM(i915, INTEL_GEMINILAKE))
glk_whitelist_build(engine);
else if (IS_KABYLAKE(i915)IS_PLATFORM(i915, INTEL_KABYLAKE))
kbl_whitelist_build(engine);
else if (IS_BROXTON(i915)IS_PLATFORM(i915, INTEL_BROXTON))
bxt_whitelist_build(engine);
else if (IS_SKYLAKE(i915)IS_PLATFORM(i915, INTEL_SKYLAKE))
skl_whitelist_build(engine);
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) <= 8)
;
else
MISSING_CASE(GRAPHICS_VER(i915))({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "((&(i915)->__runtime)->graphics.ip.ver)", (long)
(((&(i915)->__runtime)->graphics.ip.ver))); __builtin_expect
(!!(__ret), 0); });

wa_init_finish(w);
2029}

2031void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
2032{
const struct i915_wa_list *wal = &engine->whitelist;
struct intel_uncore *uncore = engine->uncore;
const u32 base = engine->mmio_base;
struct i915_wa *wa;
unsigned int i;

if (!wal->count)
return;

for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
intel_uncore_write(uncore,
		   RING_FORCE_TO_NONPRIV(base, i)((const i915_reg_t){ .reg = (((base) + 0x4D0) + (i) * 4) }),
		   i915_mmio_reg_offset(wa->reg));

/* And clear the rest just in case of garbage */
for (; i < RING_MAX_NONPRIV_SLOTS12; i++)
intel_uncore_write(uncore,
		   RING_FORCE_TO_NONPRIV(base, i)((const i915_reg_t){ .reg = (((base) + 0x4D0) + (i) * 4) }),
		   i915_mmio_reg_offset(RING_NOPID(base)((const i915_reg_t){ .reg = ((base) + 0x94) })));
2052}

2054/*
* engine_fake_wa_init(), a place holder to program the registers
* which are not part of an official workaround defined by the
* hardware team.
* Adding programming of those register inside workaround will
* allow utilizing wa framework to proper application and verification.
*/
2061static void
2062engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2063{
u8 mocs_w, mocs_r;

/*
* RING_CMD_CCTL specifies the default MOCS entry that will be used
* by the command streamer when executing commands that don't have
* a way to explicitly specify a MOCS setting.  The default should
* usually reference whichever MOCS entry corresponds to uncached
* behavior, although use of a WB cached entry is recommended by the
* spec in certain circumstances on specific platforms.
*/
if (GRAPHICS_VER(engine->i915)((&(engine->i915)->__runtime)->graphics.ip.ver) >= 12) {
mocs_r = engine->gt->mocs.uc_index;
mocs_w = engine->gt->mocs.uc_index;

if (HAS_L3_CCS_READ(engine->i915)((&(engine->i915)->__info)->has_l3_ccs_read) &&
    engine->class == COMPUTE_CLASS5) {
	mocs_r = engine->gt->mocs.wb_index;

	/*
	 * Even on the few platforms where MOCS 0 is a
	 * legitimate table entry, it's never the correct
	 * setting to use here; we can assume the MOCS init
	 * just forgot to initialize wb_index.
	 */
	drm_WARN_ON(&engine->i915->drm, mocs_r == 0)({ int __ret = !!((mocs_r == 0)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&engine->i915->drm))->
dev), "", "drm_WARN_ON(" "mocs_r == 0" ")"); __builtin_expect
(!!(__ret), 0); });
}

wa_masked_field_set(wal,
		    RING_CMD_CCTL(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0xc4) }
),
		    CMD_CCTL_MOCS_MASK(((u32)((((~0UL) >> (64 - (13) - 1)) & ((~0UL) <<
 (7))) + 0)) | ((u32)((((~0UL) >> (64 - (6) - 1)) &
 ((~0UL) << (0))) + 0))),
		    CMD_CCTL_MOCS_OVERRIDE(mocs_w, mocs_r)(((u32)((((typeof(((u32)((((~0UL) >> (64 - (13) - 1)) &
 ((~0UL) << (7))) + 0))))((mocs_w) << 1) <<
 (__builtin_ffsll(((u32)((((~0UL) >> (64 - (13) - 1)) &
 ((~0UL) << (7))) + 0))) - 1)) & (((u32)((((~0UL) >>
 (64 - (13) - 1)) & ((~0UL) << (7))) + 0)))) + 0 + 0
 + 0 + 0)) | ((u32)((((typeof(((u32)((((~0UL) >> (64 - (
6) - 1)) & ((~0UL) << (0))) + 0))))((mocs_r) <<
 1) << (__builtin_ffsll(((u32)((((~0UL) >> (64 - (
6) - 1)) & ((~0UL) << (0))) + 0))) - 1)) & (((u32
)((((~0UL) >> (64 - (6) - 1)) & ((~0UL) << (0
))) + 0)))) + 0 + 0 + 0 + 0))));
}
2096}

2098static bool_Bool needs_wa_1308578152(struct intel_engine_cs *engine)
2099{
return intel_sseu_find_first_xehp_dss(&engine->gt->info.sseu, 0, 0) >=
GEN_DSS_PER_GSLICE4;
2102}

2104static void
2105rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2106{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

if (IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2)) {
/* Wa_1509235366:dg2 */
wa_write_or(wal, GEN12_GAMCNTRL_CTRL((const i915_reg_t){ .reg = (0xcf54) }), INVALIDATION_BROADCAST_MODE_DIS((u32)((1UL << (12)) + 0)) |
	    GLOBAL_INVALIDATION_MODE((u32)((1UL << (2)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_DG2, 1) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
/* Wa_14013392000:dg2_g11 */
wa_masked_en(wal, GEN7_ROW_CHICKEN2((const i915_reg_t){ .reg = (0xe4f4) }), GEN12_ENABLE_LARGE_GRF_MODE((u32)((1UL << (12)) + 0)));

/* Wa_16011620976:dg2_g11 */
wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW((const i915_reg_t){ .reg = (0xe7c8 + 4) }), DIS_CHAIN_2XSIMD8((u32)((1UL << (55 - 32)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_FOREVER
))) ||
   IS_DG2_G11(i915)IS_SUBPLATFORM(i915, INTEL_DG2, 1) || IS_DG2_G12(i915)IS_SUBPLATFORM(i915, INTEL_DG2, 2)) {
/* Wa_1509727124:dg2 */
wa_masked_en(wal, GEN10_SAMPLER_MODE((const i915_reg_t){ .reg = (0xe18c) }),
	     SC_DISABLE_POWER_OPTIMIZATION_EBB((u32)((1UL << (9)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))) ||
   IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_DG2, 1) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
/* Wa_14012419201:dg2 */
wa_masked_en(wal, GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }),
	     GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX((u32)((1UL << (4)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_C0
))) ||
   IS_DG2_G11(i915)IS_SUBPLATFORM(i915, INTEL_DG2, 1)) {
/*
 * Wa_22012826095:dg2
 * Wa_22013059131:dg2
 */
wa_write_clr_set(wal, LSC_CHICKEN_BIT_0_UDW((const i915_reg_t){ .reg = (0xe7c8 + 4) }),
		 MAXREQS_PER_BANK((u32)((((~0UL) >> (64 - (39 - 32) - 1)) & ((~0UL) <<
 (37 - 32))) + 0)),
		 REG_FIELD_PREP(MAXREQS_PER_BANK, 2)((u32)((((typeof(((u32)((((~0UL) >> (64 - (39 - 32) - 1
)) & ((~0UL) << (37 - 32))) + 0))))(2) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (39 - 32) - 1)) & ((~0UL)
 << (37 - 32))) + 0))) - 1)) & (((u32)((((~0UL) >>
 (64 - (39 - 32) - 1)) & ((~0UL) << (37 - 32))) + 0
)))) + 0 + 0 + 0 + 0)));

/* Wa_22013059131:dg2 */
wa_write_or(wal, LSC_CHICKEN_BIT_0((const i915_reg_t){ .reg = (0xe7c8) }),
	    FORCE_1_SUB_MESSAGE_PER_FRAGMENT((u32)((1UL << (15)) + 0)));
}

/* Wa_1308578152:dg2_g10 when first gslice is fused off */
if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_C0
))) &&
   needs_wa_1308578152(engine)) {
wa_masked_dis(wal, GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON((const i915_reg_t){ .reg = (0x20ec) }),
	      GEN12_REPLAY_MODE_GRANULARITY((u32)((1UL << (0)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_FOREVER
))) ||
   IS_DG2_G11(i915)IS_SUBPLATFORM(i915, INTEL_DG2, 1) || IS_DG2_G12(i915)IS_SUBPLATFORM(i915, INTEL_DG2, 2)) {
/* Wa_22013037850:dg2 */
wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW((const i915_reg_t){ .reg = (0xe7c8 + 4) }),
	    DISABLE_128B_EVICTION_COMMAND_UDW((u32)((1UL << (36 - 32)) + 0)));

/* Wa_22012856258:dg2 */
wa_masked_en(wal, GEN7_ROW_CHICKEN2((const i915_reg_t){ .reg = (0xe4f4) }),
	     GEN12_DISABLE_READ_SUPPRESSION((u32)((1UL << (15)) + 0)));

/*
 * Wa_22010960976:dg2
 * Wa_14013347512:dg2
 */
wa_masked_dis(wal, GEN12_HDC_CHICKEN0((const i915_reg_t){ .reg = (0xe5f0) }),
	      LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK((u32)((((~0UL) >> (64 - (13) - 1)) & ((~0UL) <<
 (11))) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
/*
 * Wa_1608949956:dg2_g10
 * Wa_14010198302:dg2_g10
 */
wa_masked_en(wal, GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }),
	     MDQ_ARBITRATION_MODE((u32)((1UL << (12)) + 0)) | UGM_BACKUP_MODE((u32)((1UL << (13)) + 0)));

/*
 * Wa_14010918519:dg2_g10
 *
 * LSC_CHICKEN_BIT_0 always reads back as 0 is this stepping,
 * so ignoring verification.
 */
wa_add(wal, LSC_CHICKEN_BIT_0_UDW((const i915_reg_t){ .reg = (0xe7c8 + 4) }), 0,
       FORCE_SLM_FENCE_SCOPE_TO_TILE((u32)((1UL << (42 - 32)) + 0)) | FORCE_UGM_FENCE_SCOPE_TO_TILE((u32)((1UL << (41 - 32)) + 0)),
       0, false0);
}

if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
/* Wa_22010430635:dg2 */
wa_masked_en(wal,
	     GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }),
	     GEN12_DISABLE_GRF_CLEAR((u32)((1UL << (13)) + 0)));

/* Wa_14010648519:dg2 */
wa_write_or(wal, XEHP_L3NODEARBCFG((const i915_reg_t){ .reg = (0xb0b4) }), XEHP_LNESPARE((u32)((1UL << (19)) + 0)));
}

/* Wa_14013202645:dg2 */
if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0)(IS_SUBPLATFORM(i915, INTEL_DG2, 0) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_C0
))) ||
   IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_DG2, 1) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))))
wa_write_or(wal, RT_CTRL((const i915_reg_t){ .reg = (0xe530) }), DIS_NULL_QUERY((u32)((1UL << (10)) + 0)));

/* Wa_22012532006:dg2 */
if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_C0))) ||
   IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 1) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0))))
wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7((const i915_reg_t){ .reg = (0xe194) }),
	     DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA((u32)((1UL << (15)) + 0)));

if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0)))) {
/* Wa_14010680813:dg2_g10 */
wa_write_or(wal, GEN12_GAMSTLB_CTRL((const i915_reg_t){ .reg = (0xcf4c) }), CONTROL_BLOCK_CLKGATE_DIS((u32)((1UL << (12)) + 0)) |
	    EGRESS_BLOCK_CLKGATE_DIS((u32)((1UL << (11)) + 0)) | TAG_BLOCK_CLKGATE_DIS((u32)((1UL << (7)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 0) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0))) ||
   IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)(IS_SUBPLATFORM(engine->i915, INTEL_DG2, 1) && (({
 int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_B0)))) {
/* Wa_14012362059:dg2 */
wa_write_or(wal, GEN12_MERT_MOD_CTRL((const i915_reg_t){ .reg = (0xcf28) }), FORCE_MISS_FTLB((u32)((1UL << (3)) + 0)));
}

if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_B0, STEP_FOREVER)(IS_SUBPLATFORM(i915, INTEL_DG2, 1) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_B0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_FOREVER
))) ||
   IS_DG2_G10(i915)IS_SUBPLATFORM(i915, INTEL_DG2, 0)) {
/* Wa_22014600077:dg2 */
wa_add(wal, GEN10_CACHE_MODE_SS((const i915_reg_t){ .reg = (0xe420) }), 0,
       _MASKED_BIT_ENABLE(ENABLE_EU_COUNT_FOR_TDL_FLUSH)({ typeof(((u32)((1UL << (10)) + 0))) _a = (((u32)((1UL
 << (10)) + 0))); ({ if (__builtin_constant_p(_a)) do {
 } while (0); if (__builtin_constant_p(_a)) do { } while (0);
 if (__builtin_constant_p(_a) && __builtin_constant_p
(_a)) do { } while (0); ((_a) << 16 | (_a)); }); }),
       0 /* Wa_14012342262 :write-only reg, so skip
	    verification */,
       true1);
}

if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_PLATFORM(i915, INTEL_DG1) && (({ int __ret = !!((
((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))) ||
   IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_TIGERLAKE, (0)) && (({ int
 __ret = !!((((&(i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
/*
 * Wa_1607138336:tgl[a0],dg1[a0]
 * Wa_1607063988:tgl[a0],dg1[a0]
 */
wa_write_or(wal,
	    GEN9_CTX_PREEMPT_REG((const i915_reg_t){ .reg = (0x2248) }),
	    GEN12_DISABLE_POSH_BUSY_FF_DOP_CG((u32)((1UL << (11)) + 0)));
}

if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_SUBPLATFORM(i915, INTEL_TIGERLAKE, (0)) && (({ int
 __ret = !!((((&(i915)->__runtime)->step.graphics_step
) == STEP_NONE)); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
/*
 * Wa_1606679103:tgl
 * (see also Wa_1606682166:icl)
 */
wa_write_or(wal,
	    GEN7_SARCHKMD((const i915_reg_t){ .reg = (0xb000) }),
	    GEN7_DISABLE_SAMPLER_PREFETCH(1 << 30));
}

if (IS_ALDERLAKE_P(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_P) || IS_ALDERLAKE_S(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_S) || IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1) ||
   IS_ROCKETLAKE(i915)IS_PLATFORM(i915, INTEL_ROCKETLAKE) || IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE)) {
/* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
wa_masked_en(wal, GEN7_ROW_CHICKEN2((const i915_reg_t){ .reg = (0xe4f4) }), GEN12_DISABLE_EARLY_READ((u32)((1UL << (14)) + 0)));

/*
 * Wa_1407928979:tgl A*
 * Wa_18011464164:tgl[B0+],dg1[B0+]
 * Wa_22010931296:tgl[B0+],dg1[B0+]
 * Wa_14010919138:rkl,dg1,adl-s,adl-p
 */
wa_write_or(wal, GEN7_FF_THREAD_MODE((const i915_reg_t){ .reg = (0x20a0) }),
	    GEN12_FF_TESSELATION_DOP_GATE_DISABLE(1UL << (19)));
}

if (IS_ALDERLAKE_P(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_P) || IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2) || IS_ALDERLAKE_S(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_S) ||
   IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1) || IS_ROCKETLAKE(i915)IS_PLATFORM(i915, INTEL_ROCKETLAKE) || IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE)) {
/*
 * Wa_1606700617:tgl,dg1,adl-p
 * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
 * Wa_14010826681:tgl,dg1,rkl,adl-p
 * Wa_18019627453:dg2
 */
wa_masked_en(wal,
	     GEN9_CS_DEBUG_MODE1((const i915_reg_t){ .reg = (0x20ec) }),
	     FF_DOP_CLOCK_GATE_DISABLE((u32)((1UL << (1)) + 0)));
}

if (IS_ALDERLAKE_P(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_P) || IS_ALDERLAKE_S(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_S) ||
   IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_PLATFORM(i915, INTEL_DG1) && (({ int __ret = !!((
((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))) ||
   IS_ROCKETLAKE(i915)IS_PLATFORM(i915, INTEL_ROCKETLAKE) || IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE)) {
/* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
wa_masked_en(wal, GEN7_ROW_CHICKEN2((const i915_reg_t){ .reg = (0xe4f4) }),
	     GEN12_PUSH_CONST_DEREF_HOLD_DIS((u32)((1UL << (8)) + 0)));

/*
 * Wa_1409085225:tgl
 * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
 */
wa_masked_en(wal, GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }), GEN12_DISABLE_TDL_PUSH((u32)((1UL << (9)) + 0)));
}

if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)(IS_PLATFORM(i915, INTEL_DG1) && (({ int __ret = !!((
((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
))) ||
   IS_ROCKETLAKE(i915)IS_PLATFORM(i915, INTEL_ROCKETLAKE) || IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE) || IS_ALDERLAKE_P(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_P)) {
/*
 * Wa_1607030317:tgl
 * Wa_1607186500:tgl
 * Wa_1607297627:tgl,rkl,dg1[a0],adlp
 *
 * On TGL and RKL there are multiple entries for this WA in the
 * BSpec; some indicate this is an A0-only WA, others indicate
 * it applies to all steppings so we trust the "all steppings."
 * For DG1 this only applies to A0.
 */
wa_masked_en(wal,
	     RING_PSMI_CTL(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x50) }),
	     GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE((u32)((1UL << (7)) + 0)) |
	     GEN8_RC_SEMA_IDLE_MSG_DISABLE((u32)((1UL << (12)) + 0)));
}

if (IS_DG1(i915)IS_PLATFORM(i915, INTEL_DG1) || IS_ROCKETLAKE(i915)IS_PLATFORM(i915, INTEL_ROCKETLAKE) || IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE) ||
   IS_ALDERLAKE_S(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_S) || IS_ALDERLAKE_P(i915)IS_PLATFORM(i915, INTEL_ALDERLAKE_P)) {
/* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
wa_masked_en(wal,
	     GEN10_SAMPLER_MODE((const i915_reg_t){ .reg = (0xe18c) }),
	     ENABLE_SMALLPL((u32)((1UL << (15)) + 0)));
}

if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 11) {
/* This is not an Wa. Enable for better image quality */
wa_masked_en(wal,
	     _3D_CHICKEN3((const i915_reg_t){ .reg = (0x2090) }),
	     _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE(1 << 5));

/*
 * Wa_1405543622:icl
 * Formerly known as WaGAPZPriorityScheme
 */
wa_write_or(wal,
	    GEN8_GARBCNTL((const i915_reg_t){ .reg = (0xb004) }),
	    GEN11_ARBITRATION_PRIO_ORDER_MASK(0x3f << 22));

/*
 * Wa_1604223664:icl
 * Formerly known as WaL3BankAddressHashing
 */
wa_write_clr_set(wal,
		 GEN8_GARBCNTL((const i915_reg_t){ .reg = (0xb004) }),
		 GEN11_HASH_CTRL_EXCL_MASK(0x7f << 0),
		 GEN11_HASH_CTRL_EXCL_BIT0(1 << 0));
wa_write_clr_set(wal,
		 GEN11_GLBLINVL((const i915_reg_t){ .reg = (0xb404) }),
		 GEN11_BANK_HASH_ADDR_EXCL_MASK(0x7f << 5),
		 GEN11_BANK_HASH_ADDR_EXCL_BIT0(1 << 5));

/*
 * Wa_1405733216:icl
 * Formerly known as WaDisableCleanEvicts
 */
wa_write_or(wal,
	    GEN8_L3SQCREG4((const i915_reg_t){ .reg = (0xb118) }),
	    GEN11_LQSC_CLEAN_EVICT_DISABLE(1 << 6));

/* Wa_1606682166:icl */
wa_write_or(wal,
	    GEN7_SARCHKMD((const i915_reg_t){ .reg = (0xb000) }),
	    GEN7_DISABLE_SAMPLER_PREFETCH(1 << 30));

/* Wa_1409178092:icl */
wa_write_clr_set(wal,
		 GEN11_SCRATCH2((const i915_reg_t){ .reg = (0xb140) }),
		 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE(1 << 19),
		 0);

/* WaEnable32PlaneMode:icl */
wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS((const i915_reg_t){ .reg = (0x20d4) }),
	     GEN11_ENABLE_32_PLANE_MODE(1 << 7));

/*
 * Wa_1408767742:icl[a2..forever],ehl[all]
 * Wa_1605460711:icl[a0..c0]
 */
wa_write_or(wal,
	    GEN7_FF_THREAD_MODE((const i915_reg_t){ .reg = (0x20a0) }),
	    GEN12_FF_TESSELATION_DOP_GATE_DISABLE(1UL << (19)));

/* Wa_22010271021 */
wa_masked_en(wal,
	     GEN9_CS_DEBUG_MODE1((const i915_reg_t){ .reg = (0x20ec) }),
	     FF_DOP_CLOCK_GATE_DISABLE((u32)((1UL << (1)) + 0)));
}

if (IS_GRAPHICS_VER(i915, 9, 12)(((&(i915)->__runtime)->graphics.ip.ver) >= (9) &&
 ((&(i915)->__runtime)->graphics.ip.ver) <= (12)
)) {
/* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
wa_masked_en(wal,
	     GEN7_FF_SLICE_CS_CHICKEN1((const i915_reg_t){ .reg = (0x20e0) }),
	     GEN9_FFSC_PERCTX_PREEMPT_CTRL(1 << 14));
}

if (IS_SKYLAKE(i915)IS_PLATFORM(i915, INTEL_SKYLAKE) ||
   IS_KABYLAKE(i915)IS_PLATFORM(i915, INTEL_KABYLAKE) ||
   IS_COFFEELAKE(i915)IS_PLATFORM(i915, INTEL_COFFEELAKE) ||
   IS_COMETLAKE(i915)IS_PLATFORM(i915, INTEL_COMETLAKE)) {
/* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
wa_write_or(wal,
	    GEN8_GARBCNTL((const i915_reg_t){ .reg = (0xb004) }),
	    GEN9_GAPS_TSV_CREDIT_DISABLE(1 << 7));
}

if (IS_BROXTON(i915)IS_PLATFORM(i915, INTEL_BROXTON)) {
/* WaDisablePooledEuLoadBalancingFix:bxt */
wa_masked_en(wal,
	     FF_SLICE_CS_CHICKEN2((const i915_reg_t){ .reg = (0x20e4) }),
	     GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE(1 << 10));
}

if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 9) {
/* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
wa_masked_en(wal,
	     GEN9_CSFE_CHICKEN1_RCS((const i915_reg_t){ .reg = (0x20d4) }),
	     GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE(1 << 2));

/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
wa_write_or(wal,
	    BDW_SCRATCH1((const i915_reg_t){ .reg = (0xb11c) }),
	    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE(1 << 2));

/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
if (IS_GEN9_LP(i915)(((&(i915)->__runtime)->graphics.ip.ver) == 9 &&
 ((&(i915)->__info)->is_lp)))
	wa_write_clr_set(wal,
			 GEN8_L3SQCREG1((const i915_reg_t){ .reg = (0xb100) }),
			 L3_PRIO_CREDITS_MASK((0x1f << 19) | (0x1f << 14)),
			 L3_GENERAL_PRIO_CREDITS(62)(((62) >> 1) << 19) |
			 L3_HIGH_PRIO_CREDITS(2)(((2) >> 1) << 14));

/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
wa_write_or(wal,
	    GEN8_L3SQCREG4((const i915_reg_t){ .reg = (0xb118) }),
	    GEN8_LQSC_FLUSH_COHERENT_LINES(1 << 21));

/* Disable atomics in L3 to prevent unrecoverable hangs */
wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1((const i915_reg_t){ .reg = (0xb008) }),
		 GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE((u32)((1UL << (0)) + 0)), 0);
wa_write_clr_set(wal, GEN8_L3SQCREG4((const i915_reg_t){ .reg = (0xb118) }),
		 GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE((u32)((1UL << (22)) + 0)), 0);
wa_write_clr_set(wal, GEN9_SCRATCH1((const i915_reg_t){ .reg = (0xb11c) }),
		 EVICTION_PERF_FIX_ENABLE((u32)((1UL << (8)) + 0)), 0);
}

if (IS_HASWELL(i915)IS_PLATFORM(i915, INTEL_HASWELL)) {
/* WaSampleCChickenBitEnable:hsw */
wa_masked_en(wal,
	     HALF_SLICE_CHICKEN3((const i915_reg_t){ .reg = (0xe184) }), HSW_SAMPLE_C_PERFORMANCE(1 << 9));

wa_masked_dis(wal,
	      CACHE_MODE_0_GEN7((const i915_reg_t){ .reg = (0x7000) }),
	      /* enable HiZ Raw Stall Optimization */
	      HIZ_RAW_STALL_OPT_DISABLE(1 << 2));
}

if (IS_VALLEYVIEW(i915)IS_PLATFORM(i915, INTEL_VALLEYVIEW)) {
/* WaDisableEarlyCull:vlv */
wa_masked_en(wal,
	     _3D_CHICKEN3((const i915_reg_t){ .reg = (0x2090) }),
	     _3D_CHICKEN_SF_DISABLE_OBJEND_CULL(1 << 10));

/*
 * WaVSThreadDispatchOverride:ivb,vlv
 *
 * This actually overrides the dispatch
 * mode for all thread types.
 */
wa_write_clr_set(wal,
		 GEN7_FF_THREAD_MODE((const i915_reg_t){ .reg = (0x20a0) }),
		 GEN7_FF_SCHED_MASK0x0077070,
		 GEN7_FF_TS_SCHED_HW(0x0 << 16) |
		 GEN7_FF_VS_SCHED_HW(0x0 << 12) |
		 GEN7_FF_DS_SCHED_HW(0x0 << 4));

/* WaPsdDispatchEnable:vlv */
/* WaDisablePSDDualDispatchEnable:vlv */
wa_masked_en(wal,
	     GEN7_HALF_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0xe100) }),
	     GEN7_MAX_PS_THREAD_DEP(8 << 12) |
	     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE(1 << 3));
}

if (IS_IVYBRIDGE(i915)IS_PLATFORM(i915, INTEL_IVYBRIDGE)) {
/* WaDisableEarlyCull:ivb */
wa_masked_en(wal,
	     _3D_CHICKEN3((const i915_reg_t){ .reg = (0x2090) }),
	     _3D_CHICKEN_SF_DISABLE_OBJEND_CULL(1 << 10));

if (0) { /* causes HiZ corruption on ivb:gt1 */
	/* enable HiZ Raw Stall Optimization */
	wa_masked_dis(wal,
		      CACHE_MODE_0_GEN7((const i915_reg_t){ .reg = (0x7000) }),
		      HIZ_RAW_STALL_OPT_DISABLE(1 << 2));
}

/*
 * WaVSThreadDispatchOverride:ivb,vlv
 *
 * This actually overrides the dispatch
 * mode for all thread types.
 */
wa_write_clr_set(wal,
		 GEN7_FF_THREAD_MODE((const i915_reg_t){ .reg = (0x20a0) }),
		 GEN7_FF_SCHED_MASK0x0077070,
		 GEN7_FF_TS_SCHED_HW(0x0 << 16) |
		 GEN7_FF_VS_SCHED_HW(0x0 << 12) |
		 GEN7_FF_DS_SCHED_HW(0x0 << 4));

/* WaDisablePSDDualDispatchEnable:ivb */
if (IS_IVB_GT1(i915)(IS_PLATFORM(i915, INTEL_IVYBRIDGE) && (&(i915)->
__info)->gt == 1))
	wa_masked_en(wal,
		     GEN7_HALF_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0xe100) }),
		     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE(1 << 3));
}

if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 7) {
/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
wa_masked_en(wal,
	     RING_MODE_GEN7(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x29c) }),
	     GFX_TLB_INVALIDATE_EXPLICIT(1 << 13) | GFX_REPLAY_MODE(1 << 11));

/* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
wa_masked_dis(wal, CACHE_MODE_0_GEN7((const i915_reg_t){ .reg = (0x7000) }), RC_OP_FLUSH_ENABLE(1 << 0));

/*
 * BSpec says this must be set, even though
 * WaDisable4x2SubspanOptimization:ivb,hsw
 * WaDisable4x2SubspanOptimization isn't listed for VLV.
 */
wa_masked_en(wal,
	     CACHE_MODE_1((const i915_reg_t){ .reg = (0x7004) }),
	     PIXEL_SUBSPAN_COLLECT_OPT_DISABLE(1 << 6));

/*
 * BSpec recommends 8x4 when MSAA is used,
 * however in practice 16x4 seems fastest.
 *
 * Note that PS/WM thread counts depend on the WIZ hashing
 * disable bit, which we don't touch here, but it's good
 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 */
wa_masked_field_set(wal,
		    GEN7_GT_MODE((const i915_reg_t){ .reg = (0x7008) }),
		    GEN6_WIZ_HASHING_MASK(((1) << 9) | ((1) << 7)),
		    GEN6_WIZ_HASHING_16x4(((1) << 9) | ((0) << 7)));
}

if (IS_GRAPHICS_VER(i915, 6, 7)(((&(i915)->__runtime)->graphics.ip.ver) >= (6) &&
 ((&(i915)->__runtime)->graphics.ip.ver) <= (7)))
/*
 * We need to disable the AsyncFlip performance optimisations in
 * order to use MI_WAIT_FOR_EVENT within the CS. It should
 * already be programmed to '1' on all products.
 *
 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
 */
wa_masked_en(wal,
	     RING_MI_MODE(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x9c) }),
	     ASYNC_FLIP_PERF_DISABLE((u32)((1UL << (14)) + 0)));

if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 6) {
/*
 * Required for the hardware to program scanline values for
 * waiting
 * WaEnableFlushTlbInvalidationMode:snb
 */
wa_masked_en(wal,
	     GFX_MODE((const i915_reg_t){ .reg = (0x2520) }),
	     GFX_TLB_INVALIDATE_EXPLICIT(1 << 13));

/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
wa_masked_en(wal,
	     _3D_CHICKEN((const i915_reg_t){ .reg = (0x2084) }),
	     _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB(1 << 10));

wa_masked_en(wal,
	     _3D_CHICKEN3((const i915_reg_t){ .reg = (0x2090) }),
	     /* WaStripsFansDisableFastClipPerformanceFix:snb */
	     _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL(1 << 5) |
	     /*
	      * Bspec says:
	      * "This bit must be set if 3DSTATE_CLIP clip mode is set
	      * to normal and 3DSTATE_SF number of SF output attributes
	      * is more than 16."
	      */
	     _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH(1 << 1));

/*
 * BSpec recommends 8x4 when MSAA is used,
 * however in practice 16x4 seems fastest.
 *
 * Note that PS/WM thread counts depend on the WIZ hashing
 * disable bit, which we don't touch here, but it's good
 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 */
wa_masked_field_set(wal,
		    GEN6_GT_MODE((const i915_reg_t){ .reg = (0x20d0) }),
		    GEN6_WIZ_HASHING_MASK(((1) << 9) | ((1) << 7)),
		    GEN6_WIZ_HASHING_16x4(((1) << 9) | ((0) << 7)));

/* WaDisable_RenderCache_OperationalFlush:snb */
wa_masked_dis(wal, CACHE_MODE_0((const i915_reg_t){ .reg = (0x2120) }), RC_OP_FLUSH_ENABLE(1 << 0));

/*
 * From the Sandybridge PRM, volume 1 part 3, page 24:
 * "If this bit is set, STCunit will have LRA as replacement
 *  policy. [...] This bit must be reset. LRA replacement
 *  policy is not supported."
 */
wa_masked_dis(wal,
	      CACHE_MODE_0((const i915_reg_t){ .reg = (0x2120) }),
	      CM0_STC_EVICT_DISABLE_LRA_SNB(1 << 5));
}

if (IS_GRAPHICS_VER(i915, 4, 6)(((&(i915)->__runtime)->graphics.ip.ver) >= (4) &&
 ((&(i915)->__runtime)->graphics.ip.ver) <= (6)))
/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
wa_add(wal, RING_MI_MODE(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x9c) }),
       0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)({ typeof(((u32)((1UL << (6)) + 0))) _a = (((u32)((1UL <<
 (6)) + 0))); ({ if (__builtin_constant_p(_a)) do { } while (
0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p
(_a) && __builtin_constant_p(_a)) do { } while (0); (
(_a) << 16 | (_a)); }); }),
       /* XXX bit doesn't stick on Broadwater */
       IS_I965G(i915)IS_PLATFORM(i915, INTEL_I965G) ? 0 : VS_TIMER_DISPATCH((u32)((1UL << (6)) + 0)), true1);

if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) == 4)
/*
 * Disable CONSTANT_BUFFER before it is loaded from the context
 * image. For as it is loaded, it is executed and the stored
 * address may no longer be valid, leading to a GPU hang.
 *
 * This imposes the requirement that userspace reload their
 * CONSTANT_BUFFER on every batch, fortunately a requirement
 * they are already accustomed to from before contexts were
 * enabled.
 */
wa_add(wal, ECOSKPD(RENDER_RING_BASE)((const i915_reg_t){ .reg = ((0x02000) + 0x1d0) }),
       0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE)({ typeof(((u32)((1UL << (4)) + 0))) _a = (((u32)((1UL <<
 (4)) + 0))); ({ if (__builtin_constant_p(_a)) do { } while (
0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p
(_a) && __builtin_constant_p(_a)) do { } while (0); (
(_a) << 16 | (_a)); }); }),
       0 /* XXX bit doesn't stick on Broadwater */,
       true1);
2640}

2642static void
2643xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2644{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

/* WaKBLVECSSemaphoreWaitPoll:kbl */
if (IS_KBL_GRAPHICS_STEP(i915, STEP_A0, STEP_F0)(IS_PLATFORM(i915, INTEL_KABYLAKE) && (({ int __ret =
 !!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_F0
)))) {
12
←
Assuming the condition is true→
13
←
Assuming field 'graphics_step' is not equal to STEP_NONE→
14
←
Taking false branch→
15
←
Assuming field 'graphics_step' is < STEP_F0→
16
←
Taking true branch→
wa_write(wal,
17
←
Calling 'wa_write'→
	 RING_SEMA_WAIT_POLL(engine->mmio_base)((const i915_reg_t){ .reg = ((engine->mmio_base) + 0x24c) }
),
	 1);
}
2653}

2655static void
2656ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2657{
if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)(IS_PLATFORM(engine->i915, INTEL_PONTEVECCHIO) && (
({ int __ret = !!((((&(engine->i915)->__runtime)->
step.graphics_step) == STEP_NONE)); if (__ret) printf("%s %s: "
 "%s", dev_driver_string(((&(engine->i915)->drm))->
dev), "", "drm_WARN_ON(" "((&(engine->i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(engine->
i915)->__runtime)->step.graphics_step) >= (STEP_A0) &&
 ((&(engine->i915)->__runtime)->step.graphics_step
) < (STEP_C0)))) {
/* Wa_14014999345:pvc */
wa_masked_en(wal, GEN10_CACHE_MODE_SS((const i915_reg_t){ .reg = (0xe420) }), DISABLE_ECC((u32)((1UL << (5)) + 0)));
}
2662}

2664/*
* The bspec performance guide has recommended MMIO tuning settings.  These
* aren't truly "workarounds" but we want to program them with the same
* workaround infrastructure to ensure that they're automatically added to
* the GuC save/restore lists, re-applied at the right times, and checked for
* any conflicting programming requested by real workarounds.
*
* Programming settings should be added here only if their registers are not
* part of an engine's register state context.  If a register is part of a
* context, then any tuning settings should be programmed in an appropriate
* function invoked by __intel_engine_init_ctx_wa().
*/
2676static void
2677add_render_compute_tuning_settings(struct drm_i915_privateinteldrm_softc *i915,
		   struct i915_wa_list *wal)
2679{
if (IS_PONTEVECCHIO(i915)IS_PLATFORM(i915, INTEL_PONTEVECCHIO)) {
wa_write(wal, XEHPC_L3SCRUB((const i915_reg_t){ .reg = (0xb18c) }),
	 SCRUB_CL_DWNGRADE_SHARED((u32)((1UL << (12)) + 0)) | SCRUB_RATE_4B_PER_CLK((u32)((((typeof(((u32)((((~0UL) >> (64 - (2) - 1)) &
 ((~0UL) << (0))) + 0))))(0x6) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (2) - 1)) & ((~0UL) <<
 (0))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (2)
 - 1)) & ((~0UL) << (0))) + 0)))) + 0 + 0 + 0 + 0)));
}

if (IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2)) {
wa_write_or(wal, XEHP_L3SCQREG7((const i915_reg_t){ .reg = (0xb188) }), BLEND_FILL_CACHING_OPT_DIS((u32)((1UL << (3)) + 0)));
wa_write_clr_set(wal, RT_CTRL((const i915_reg_t){ .reg = (0xe530) }), STACKID_CTRL((u32)((((~0UL) >> (64 - (6) - 1)) & ((~0UL) <<
 (5))) + 0)), STACKID_CTRL_512((u32)((((typeof(((u32)((((~0UL) >> (64 - (6) - 1)) &
 ((~0UL) << (5))) + 0))))(0x2) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (6) - 1)) & ((~0UL) <<
 (5))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (6)
 - 1)) & ((~0UL) << (5))) + 0)))) + 0 + 0 + 0 + 0)));

/*
 * This is also listed as Wa_22012654132 for certain DG2
 * steppings, but the tuning setting programming is a superset
 * since it applies to all DG2 variants and steppings.
 *
 * Note that register 0xE420 is write-only and cannot be read
 * back for verification on DG2 (due to Wa_14012342262), so
 * we need to explicitly skip the readback.
 */
wa_add(wal, GEN10_CACHE_MODE_SS((const i915_reg_t){ .reg = (0xe420) }), 0,
       _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC)({ typeof(((u32)((1UL << (3)) + 0))) _a = (((u32)((1UL <<
 (3)) + 0))); ({ if (__builtin_constant_p(_a)) do { } while (
0); if (__builtin_constant_p(_a)) do { } while (0); if (__builtin_constant_p
(_a) && __builtin_constant_p(_a)) do { } while (0); (
(_a) << 16 | (_a)); }); }),
       0 /* write-only, so skip validation */,
       true1);
}

/*
* This tuning setting proves beneficial only on ATS-M designs; the
* default "age based" setting is optimal on regular DG2 and other
* platforms.
*/
if (INTEL_INFO(i915)(&(i915)->__info)->tuning_thread_rr_after_dep)
wa_masked_field_set(wal, GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }), THREAD_EX_ARB_MODE((u32)((((~0UL) >> (64 - (3) - 1)) & ((~0UL) <<
 (2))) + 0)),
		    THREAD_EX_ARB_MODE_RR_AFTER_DEP((u32)((((typeof(((u32)((((~0UL) >> (64 - (3) - 1)) &
 ((~0UL) << (2))) + 0))))(0x2) << (__builtin_ffsll
(((u32)((((~0UL) >> (64 - (3) - 1)) & ((~0UL) <<
 (2))) + 0))) - 1)) & (((u32)((((~0UL) >> (64 - (3)
 - 1)) & ((~0UL) << (2))) + 0)))) + 0 + 0 + 0 + 0)));
2712}

2714/*
* The workarounds in this function apply to shared registers in
* the general render reset domain that aren't tied to a
* specific engine.  Since all render+compute engines get reset
* together, and the contents of these registers are lost during
* the shared render domain reset, we'll define such workarounds
* here and then add them to just a single RCS or CCS engine's
* workaround list (whichever engine has the XXXX flag).
*/
2723static void
2724general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2725{
struct drm_i915_privateinteldrm_softc *i915 = engine->i915;

add_render_compute_tuning_settings(i915, wal);

if (IS_PONTEVECCHIO(i915)IS_PLATFORM(i915, INTEL_PONTEVECCHIO)) {
/* Wa_16016694945 */
wa_masked_en(wal, XEHPC_LNCFMISCCFGREG0((const i915_reg_t){ .reg = (0xb01c) }), XEHPC_OVRLSCCC((u32)((1UL << (0)) + 0)));
}

if (IS_XEHPSDV(i915)IS_PLATFORM(i915, INTEL_XEHPSDV)) {
/* Wa_1409954639 */
wa_masked_en(wal,
	     GEN8_ROW_CHICKEN((const i915_reg_t){ .reg = (0xe4f0) }),
	     SYSTOLIC_DOP_CLOCK_GATING_DIS((u32)((1UL << (10)) + 0)));

/* Wa_1607196519 */
wa_masked_en(wal,
	     GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }),
	     GEN12_DISABLE_GRF_CLEAR((u32)((1UL << (13)) + 0)));

/* Wa_14010670810:xehpsdv */
wa_write_or(wal, XEHP_L3NODEARBCFG((const i915_reg_t){ .reg = (0xb0b4) }), XEHP_LNESPARE((u32)((1UL << (19)) + 0)));

/* Wa_14010449647:xehpsdv */
wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1((const i915_reg_t){ .reg = (0xe100) }),
	     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE(1 << 3));

/* Wa_18011725039:xehpsdv */
if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)(IS_PLATFORM(i915, INTEL_XEHPSDV) && (({ int __ret = !
!((((&(i915)->__runtime)->step.graphics_step) == STEP_NONE
)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&
(i915)->drm))->dev), "", "drm_WARN_ON(" "((&(i915)->__runtime)->step.graphics_step) == STEP_NONE"
 ")"); __builtin_expect(!!(__ret), 0); }), ((&(i915)->
__runtime)->step.graphics_step) >= (STEP_A1) &&
 ((&(i915)->__runtime)->step.graphics_step) < (STEP_B0
)))) {
	wa_masked_dis(wal, MLTICTXCTL((const i915_reg_t){ .reg = (0xb170) }), TDONRENDER((u32)((1UL << (2)) + 0)));
	wa_write_or(wal, L3SQCREG1_CCS0((const i915_reg_t){ .reg = (0xb200) }), FLUSHALLNONCOH((u32)((1UL << (5)) + 0)));
}

/* Wa_14012362059:xehpsdv */
wa_write_or(wal, GEN12_MERT_MOD_CTRL((const i915_reg_t){ .reg = (0xcf28) }), FORCE_MISS_FTLB((u32)((1UL << (3)) + 0)));

/* Wa_14014368820:xehpsdv */
wa_write_or(wal, GEN12_GAMCNTRL_CTRL((const i915_reg_t){ .reg = (0xcf54) }), INVALIDATION_BROADCAST_MODE_DIS((u32)((1UL << (12)) + 0)) |
		GLOBAL_INVALIDATION_MODE((u32)((1UL << (2)) + 0)));
}

if (IS_DG2(i915)IS_PLATFORM(i915, INTEL_DG2) || IS_PONTEVECCHIO(i915)IS_PLATFORM(i915, INTEL_PONTEVECCHIO)) {
/* Wa_14015227452:dg2,pvc */
wa_masked_en(wal, GEN9_ROW_CHICKEN4((const i915_reg_t){ .reg = (0xe48c) }), XEHP_DIS_BBL_SYSPIPE((u32)((1UL << (11)) + 0)));

/* Wa_22014226127:dg2,pvc */
wa_write_or(wal, LSC_CHICKEN_BIT_0((const i915_reg_t){ .reg = (0xe7c8) }), DISABLE_D8_D16_COASLESCE((u32)((1UL << (30)) + 0)));

/* Wa_16015675438:dg2,pvc */
wa_masked_en(wal, FF_SLICE_CS_CHICKEN2((const i915_reg_t){ .reg = (0x20e4) }), GEN12_PERF_FIX_BALANCING_CFE_DISABLE((u32)((1UL << (15)) + 0)));

/* Wa_18018781329:dg2,pvc */
wa_write_or(wal, RENDER_MOD_CTRL((const i915_reg_t){ .reg = (0xcf2c) }), FORCE_MISS_FTLB((u32)((1UL << (3)) + 0)));
wa_write_or(wal, COMP_MOD_CTRL((const i915_reg_t){ .reg = (0xcf30) }), FORCE_MISS_FTLB((u32)((1UL << (3)) + 0)));
wa_write_or(wal, VDBX_MOD_CTRL((const i915_reg_t){ .reg = (0xcf34) }), FORCE_MISS_FTLB((u32)((1UL << (3)) + 0)));
wa_write_or(wal, VEBX_MOD_CTRL((const i915_reg_t){ .reg = (0xcf38) }), FORCE_MISS_FTLB((u32)((1UL << (3)) + 0)));
}
2783}

2785static void
2786engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2787{
if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4)0)
4
←
Taking false branch→
return;

engine_fake_wa_init(engine, wal);

/*
* These are common workarounds that just need to applied
* to a single RCS/CCS engine's workaround list since
* they're reset as part of the general render domain reset.
*/
if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE(1UL << (11)))
5
←
Assuming the condition is false→
6
←
Taking false branch→
general_render_compute_wa_init(engine, wal);

if (engine->class == COMPUTE_CLASS5)
7
←
Assuming field 'class' is not equal to COMPUTE_CLASS→
8
←
Taking false branch→
ccs_engine_wa_init(engine, wal);
else if (engine->class == RENDER_CLASS0)
9
←
Assuming field 'class' is not equal to RENDER_CLASS→
10
←
Taking false branch→
rcs_engine_wa_init(engine, wal);
else
xcs_engine_wa_init(engine, wal);
11
←
Calling 'xcs_engine_wa_init'→
2807}

2809void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2810{
struct i915_wa_list *wal = &engine->wa_list;

if (GRAPHICS_VER(engine->i915)((&(engine->i915)->__runtime)->graphics.ip.ver) < 4)
1
Assuming field 'ver' is >= 4→
2
←
Taking false branch→
return;

wa_init_start(wal, "engine", engine->name);
engine_init_workarounds(engine, wal);
3
←
Calling 'engine_init_workarounds'→
wa_init_finish(wal);
2819}

2821void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2822{
wa_list_apply(engine->gt, &engine->wa_list);
2824}

2826static const struct i915_range mcr_ranges_gen8[] = {
{ .start = 0x5500, .end = 0x55ff },
{ .start = 0x7000, .end = 0x7fff },
{ .start = 0x9400, .end = 0x97ff },
{ .start = 0xb000, .end = 0xb3ff },
{ .start = 0xe000, .end = 0xe7ff },
{},
2833};

2835static const struct i915_range mcr_ranges_gen12[] = {
{ .start =  0x8150, .end =  0x815f },
{ .start =  0x9520, .end =  0x955f },
{ .start =  0xb100, .end =  0xb3ff },
{ .start =  0xde80, .end =  0xe8ff },
{ .start = 0x24a00, .end = 0x24a7f },
{},
2842};

2844static const struct i915_range mcr_ranges_xehp[] = {
{ .start =  0x4000, .end =  0x4aff },
{ .start =  0x5200, .end =  0x52ff },
{ .start =  0x5400, .end =  0x7fff },
{ .start =  0x8140, .end =  0x815f },
{ .start =  0x8c80, .end =  0x8dff },
{ .start =  0x94d0, .end =  0x955f },
{ .start =  0x9680, .end =  0x96ff },
{ .start =  0xb000, .end =  0xb3ff },
{ .start =  0xc800, .end =  0xcfff },
{ .start =  0xd800, .end =  0xd8ff },
{ .start =  0xdc00, .end =  0xffff },
{ .start = 0x17000, .end = 0x17fff },
{ .start = 0x24a00, .end = 0x24a7f },
{},
2859};

2861static bool_Bool mcr_range(struct drm_i915_privateinteldrm_softc *i915, u32 offset)
2862{
const struct i915_range *mcr_ranges;
int i;

if (GRAPHICS_VER_FULL(i915)(((&(i915)->__runtime)->graphics.ip.ver) << 8
 | ((&(i915)->__runtime)->graphics.ip.rel)) >= IP_VER(12, 50)((12) << 8 | (50)))
mcr_ranges = mcr_ranges_xehp;
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 12)
mcr_ranges = mcr_ranges_gen12;
else if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 8)
mcr_ranges = mcr_ranges_gen8;
else
return false0;

/*
* Registers in these ranges are affected by the MCR selector
* which only controls CPU initiated MMIO. Routing does not
* work for CS access so we cannot verify them on this path.
*/
for (i = 0; mcr_ranges[i].start; i++)
if (offset >= mcr_ranges[i].start &&
    offset <= mcr_ranges[i].end)
	return true1;

return false0;
2886}

2888static int
2889wa_list_srm(struct i915_request *rq,
   const struct i915_wa_list *wal,
   struct i915_vma *vma)
2892{
struct drm_i915_privateinteldrm_softc *i915 = rq->engine->i915;
unsigned int i, count = 0;
const struct i915_wa *wa;
u32 srm, *cs;

srm = MI_STORE_REGISTER_MEM(((0x0) << 29) | (0x24) << 23 | (1)) | MI_SRM_LRM_GLOBAL_GTT(1<<22);
if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 8)
srm++;

for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
	count++;
}

cs = intel_ring_begin(rq, 4 * count);
if (IS_ERR(cs))
return PTR_ERR(cs);

for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
u32 offset = i915_mmio_reg_offset(wa->reg);

if (mcr_range(i915, offset))
	continue;

*cs++ = srm;
*cs++ = offset;
*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
*cs++ = 0;
}
intel_ring_advance(rq, cs);

return 0;
2925}

2927static int engine_wa_list_verify(struct intel_context *ce,
		 const struct i915_wa_list * const wal,
		 const char *from)
2930{
const struct i915_wa *wa;
struct i915_request *rq;
struct i915_vma *vma;
struct i915_gem_ww_ctx ww;
unsigned int i;
u32 *results;
int err;

if (!wal->count)
return 0;

vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
			   wal->count * sizeof(u32));
if (IS_ERR(vma))
return PTR_ERR(vma);

intel_engine_pm_get(ce->engine);
i915_gem_ww_ctx_init(&ww, false0);
2949retry:
err = i915_gem_object_lock(vma->obj, &ww);
if (err == 0)
err = intel_context_pin_ww(ce, &ww);
if (err)
goto err_pm;

err = i915_vma_pin_ww(vma, &ww, 0, 0,
	   i915_vma_is_ggtt(vma) ? PIN_GLOBAL(1ULL << (10)) : PIN_USER(1ULL << (11)));
if (err)
goto err_unpin;

rq = i915_request_create(ce);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
goto err_vma;
}

err = i915_request_await_object(rq, vma->obj, true1);
if (err == 0)
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE(1<<2));
if (err == 0)
err = wa_list_srm(rq, wal, vma);

i915_request_get(rq);
if (err)
i915_request_set_error_once(rq, err);
i915_request_add(rq);

if (err)
goto err_rq;

if (i915_request_wait(rq, 0, HZhz / 5) < 0) {
err = -ETIME60;
goto err_rq;
}

results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
if (IS_ERR(results)) {
err = PTR_ERR(results);
goto err_rq;
}

err = 0;
for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
	continue;

if (!wa_verify(wa, results[i], wal->name, from))
	err = -ENXIO6;
}

i915_gem_object_unpin_map(vma->obj);

3003err_rq:
i915_request_put(rq);
3005err_vma:
i915_vma_unpin(vma);
3007err_unpin:
intel_context_unpin(ce);
3009err_pm:
if (err == -EDEADLK11) {
err = i915_gem_ww_ctx_backoff(&ww);
if (!err)
	goto retry;
}
i915_gem_ww_ctx_fini(&ww);
intel_engine_pm_put(ce->engine);
i915_vma_put(vma);
return err;
3019}

3021int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
		    const char *from)
3023{
return engine_wa_list_verify(engine->kernel_context,
		     &engine->wa_list,
		     from);
3027}

3029#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
3030#include "selftest_workarounds.c"
3031#endif

←

/usr/src/sys/dev/pci/drm/i915/i915_reg_defs.h

1/* SPDX-License-Identifier: MIT */
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5 
6#ifndef __I915_REG_DEFS__
7#define __I915_REG_DEFS__
8 
9#include <linux/bitfield.h>
10#include <linux/bits.h>
11 
12/**
13 * REG_BIT() - Prepare a u32 bit value
14 * @__n: 0-based bit number
15 *
16 * Local wrapper for BIT() to force u32, with compile time checks.
17 *
18 * @return: Value with bit @__n set.
19 */
20#define REG_BIT(__n)((u32)((1UL << (__n)) + 0))							\
21	((u32)(BIT(__n)(1UL << (__n)) +						\
22	       BUILD_BUG_ON_ZERO(__is_constexpr(__n) &&		\0
23				 ((__n) < 0 || (__n) > 31))0))
24 
25/**
26 * REG_GENMASK() - Prepare a continuous u32 bitmask
27 * @__high: 0-based high bit
28 * @__low: 0-based low bit
29 *
30 * Local wrapper for GENMASK() to force u32, with compile time checks.
31 *
32 * @return: Continuous bitmask from @__high to @__low, inclusive.
33 */
34#define REG_GENMASK(__high, __low)((u32)((((~0UL) >> (64 - (__high) - 1)) & ((~0UL) <<
 (__low))) + 0))					\
35	((u32)(GENMASK(__high, __low)(((~0UL) >> (64 - (__high) - 1)) & ((~0UL) <<
 (__low))) +					\
36	       BUILD_BUG_ON_ZERO(__is_constexpr(__high) &&	\0
37				 __is_constexpr(__low) &&		\0
38				 ((__low) < 0 || (__high) > 31 || (__low) > (__high)))0))
39 
40/**
41 * REG_GENMASK64() - Prepare a continuous u64 bitmask
42 * @__high: 0-based high bit
43 * @__low: 0-based low bit
44 *
45 * Local wrapper for GENMASK_ULL() to force u64, with compile time checks.
46 *
47 * @return: Continuous bitmask from @__high to @__low, inclusive.
48 */
49#define REG_GENMASK64(__high, __low)((u64)((((~0ULL) >> (64 - (__high) - 1)) & ((~0ULL)
 << (__low))) + 0))					\
50	((u64)(GENMASK_ULL(__high, __low)(((~0ULL) >> (64 - (__high) - 1)) & ((~0ULL) <<
 (__low))) +				\
51	       BUILD_BUG_ON_ZERO(__is_constexpr(__high) &&		\0
52				 __is_constexpr(__low) &&		\0
53				 ((__low) < 0 || (__high) > 63 || (__low) > (__high)))0))
54 
55/*
56 * Local integer constant expression version of is_power_of_2().
57 */
58#define IS_POWER_OF_2(__x)((__x) && (((__x) & ((__x) - 1)) == 0))		((__x) && (((__x) & ((__x) - 1)) == 0))
59 
60/**
61 * REG_FIELD_PREP() - Prepare a u32 bitfield value
62 * @__mask: shifted mask defining the field's length and position
63 * @__val: value to put in the field
64 *
65 * Local copy of FIELD_PREP() to generate an integer constant expression, force
66 * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK().
67 *
68 * @return: @__val masked and shifted into the field defined by @__mask.
69 */
70#define REG_FIELD_PREP(__mask, __val)((u32)((((typeof(__mask))(__val) << (__builtin_ffsll(__mask
) - 1)) & (__mask)) + 0 + 0 + 0 + 0))						\
71	((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)(__builtin_ffsll(__mask) - 1)) & (__mask)) +	\
72	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask))0 +		\
73	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX)0 +		\
74	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask))))0 + \
75	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))0))
76 
77/**
78 * REG_FIELD_GET() - Extract a u32 bitfield value
79 * @__mask: shifted mask defining the field's length and position
80 * @__val: value to extract the bitfield value from
81 *
82 * Local wrapper for FIELD_GET() to force u32 and for consistency with
83 * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK().
84 *
85 * @return: Masked and shifted value of the field defined by @__mask in @__val.
86 */
87#define REG_FIELD_GET(__mask, __val)((u32)((typeof(__mask))(((__val) & (__mask)) >> (__builtin_ffsll
(__mask) - 1))))	((u32)FIELD_GET(__mask, __val)((typeof(__mask))(((__val) & (__mask)) >> (__builtin_ffsll
(__mask) - 1))))
88 
89/**
90 * REG_FIELD_GET64() - Extract a u64 bitfield value
91 * @__mask: shifted mask defining the field's length and position
92 * @__val: value to extract the bitfield value from
93 *
94 * Local wrapper for FIELD_GET() to force u64 and for consistency with
95 * REG_GENMASK64().
96 *
97 * @return: Masked and shifted value of the field defined by @__mask in @__val.
98 */
99#define REG_FIELD_GET64(__mask, __val)((u64)((typeof(__mask))(((__val) & (__mask)) >> (__builtin_ffsll
(__mask) - 1))))	((u64)FIELD_GET(__mask, __val)((typeof(__mask))(((__val) & (__mask)) >> (__builtin_ffsll
(__mask) - 1))))
100 
101typedef struct {
102	u32 reg;
103} i915_reg_t;
104 
105#define _MMIO(r)((const i915_reg_t){ .reg = (r) }) ((const i915_reg_t){ .reg = (r) })
106 
107#define INVALID_MMIO_REG((const i915_reg_t){ .reg = (0) }) _MMIO(0)((const i915_reg_t){ .reg = (0) })
108 
109static __always_inlineinline __attribute__((__always_inline__)) u32 i915_mmio_reg_offset(i915_reg_t reg)
110{
111	return reg.reg;
31
←
Undefined or garbage value returned to caller
112}
113 
114static inline bool_Bool i915_mmio_reg_equal(i915_reg_t a, i915_reg_t b)
115{
116	return i915_mmio_reg_offset(a) == i915_mmio_reg_offset(b);
117}
118 
119static inline bool_Bool i915_mmio_reg_valid(i915_reg_t reg)
120{
121	return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG((const i915_reg_t){ .reg = (0) }));
122}
123 
124#define VLV_DISPLAY_BASE0x180000		0x180000
125 
126#endif /* __I915_REG_DEFS__ */