/usr/src/sys/dev/pci/drm/i915/gem/i915_gem

Bug Summary

File:	dev/pci/drm/i915/gem/i915_gem_execbuffer.c
Warning:	line 354, column 52 The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'unsigned long'
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name i915_gem_execbuffer.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/i915/gem/i915_gem_execbuffer.c
1/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2008,2010 Intel Corporation
*/

7#include <linux/intel-iommu.h>
8#include <linux/dma-resv.h>
9#include <linux/sync_file.h>
10#include <linux/uaccess.h>

12#include <drm/drm_syncobj.h>

14#include <dev/pci/pcivar.h>
15#include <dev/pci/agpvar.h>

17#include "display/intel_frontbuffer.h"

19#include "gem/i915_gem_ioctls.h"
20#include "gt/intel_context.h"
21#include "gt/intel_gt.h"
22#include "gt/intel_gt_buffer_pool.h"
23#include "gt/intel_gt_pm.h"
24#include "gt/intel_ring.h"

26#include "i915_drv.h"
27#include "i915_gem_clflush.h"
28#include "i915_gem_context.h"
29#include "i915_gem_ioctls.h"
30#include "i915_trace.h"
31#include "i915_user_extensions.h"

33struct eb_vma {
struct i915_vma *vma;
unsigned int flags;

/** This vma's place in the execbuf reservation list */
struct drm_i915_gem_exec_object2 *exec;
struct list_head bind_link;
struct list_head reloc_link;

struct hlist_node node;
u32 handle;
44};

46enum {
FORCE_CPU_RELOC = 1,
FORCE_GTT_RELOC,
FORCE_GPU_RELOC,
50#define DBG_FORCE_RELOC0 0 /* choose one of the above! */
51};

53#define __EXEC_OBJECT_HAS_PIN(1UL << (31))		BIT(31)(1UL << (31))
54#define __EXEC_OBJECT_HAS_FENCE(1UL << (30))		BIT(30)(1UL << (30))
55#define __EXEC_OBJECT_NEEDS_MAP(1UL << (29))		BIT(29)(1UL << (29))
56#define __EXEC_OBJECT_NEEDS_BIAS(1UL << (28))	BIT(28)(1UL << (28))
57#define __EXEC_OBJECT_INTERNAL_FLAGS(~0u << 28)	(~0u << 28) /* all of the above */
58#define __EXEC_OBJECT_RESERVED((1UL << (31)) | (1UL << (30))) (__EXEC_OBJECT_HAS_PIN(1UL << (31)) | __EXEC_OBJECT_HAS_FENCE(1UL << (30)))

60#define __EXEC_HAS_RELOC(1UL << (31))	BIT(31)(1UL << (31))
61#define __EXEC_ENGINE_PINNED(1UL << (30))	BIT(30)(1UL << (30))
62#define __EXEC_INTERNAL_FLAGS(~0u << 30)	(~0u << 30)
63#define UPDATE(1ULL << (7))			PIN_OFFSET_FIXED(1ULL << (7))

65#define BATCH_OFFSET_BIAS(256*1024) (256*1024)

67#define __I915_EXEC_ILLEGAL_FLAGS((-((1 << 21) << 1)) | (3<<6) | (1<<15
)) \
(__I915_EXEC_UNKNOWN_FLAGS(-((1 << 21) << 1)) | \
I915_EXEC_CONSTANTS_MASK(3<<6)  | \
I915_EXEC_RESOURCE_STREAMER(1<<15))

72/* Catch emission of unexpected errors for CI! */
73#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)0
74#undef EINVAL22
75#define EINVAL22 ({ \
DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__)__drm_dbg(DRM_UT_DRIVER, "EINVAL at %s:%d\n", __func__, 76); \
22; \
78})
79#endif

81/**
* DOC: User command execution
*
* Userspace submits commands to be executed on the GPU as an instruction
* stream within a GEM object we call a batchbuffer. This instructions may
* refer to other GEM objects containing auxiliary state such as kernels,
* samplers, render targets and even secondary batchbuffers. Userspace does
* not know where in the GPU memory these objects reside and so before the
* batchbuffer is passed to the GPU for execution, those addresses in the
* batchbuffer and auxiliary objects are updated. This is known as relocation,
* or patching. To try and avoid having to relocate each object on the next
* execution, userspace is told the location of those objects in this pass,
* but this remains just a hint as the kernel may choose a new location for
* any object in the future.
*
* At the level of talking to the hardware, submitting a batchbuffer for the
* GPU to execute is to add content to a buffer from which the HW
* command streamer is reading.
*
* 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
*    Execlists, this command is not placed on the same buffer as the
*    remaining items.
*
* 2. Add a command to invalidate caches to the buffer.
*
* 3. Add a batchbuffer start command to the buffer; the start command is
*    essentially a token together with the GPU address of the batchbuffer
*    to be executed.
*
* 4. Add a pipeline flush to the buffer.
*
* 5. Add a memory write command to the buffer to record when the GPU
*    is done executing the batchbuffer. The memory write writes the
*    global sequence number of the request, ``i915_request::global_seqno``;
*    the i915 driver uses the current value in the register to determine
*    if the GPU has completed the batchbuffer.
*
* 6. Add a user interrupt command to the buffer. This command instructs
*    the GPU to issue an interrupt when the command, pipeline flush and
*    memory write are completed.
*
* 7. Inform the hardware of the additional commands added to the buffer
*    (by updating the tail pointer).
*
* Processing an execbuf ioctl is conceptually split up into a few phases.
*
* 1. Validation - Ensure all the pointers, handles and flags are valid.
* 2. Reservation - Assign GPU address space for every object
* 3. Relocation - Update any addresses to point to the final locations
* 4. Serialisation - Order the request with respect to its dependencies
* 5. Construction - Construct a request to execute the batchbuffer
* 6. Submission (at some point in the future execution)
*
* Reserving resources for the execbuf is the most complicated phase. We
* neither want to have to migrate the object in the address space, nor do
* we want to have to update any relocations pointing to this object. Ideally,
* we want to leave the object where it is and for all the existing relocations
* to match. If the object is given a new address, or if userspace thinks the
* object is elsewhere, we have to parse all the relocation entries and update
* the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
* all the target addresses in all of its objects match the value in the
* relocation entries and that they all match the presumed offsets given by the
* list of execbuffer objects. Using this knowledge, we know that if we haven't
* moved any buffers, all the relocation entries are valid and we can skip
* the update. (If userspace is wrong, the likely outcome is an impromptu GPU
* hang.) The requirement for using I915_EXEC_NO_RELOC are:
*
*      The addresses written in the objects must match the corresponding
*      reloc.presumed_offset which in turn must match the corresponding
*      execobject.offset.
*
*      Any render targets written to in the batch must be flagged with
*      EXEC_OBJECT_WRITE.
*
*      To avoid stalling, execobject.offset should match the current
*      address of that object within the active context.
*
* The reservation is done is multiple phases. First we try and keep any
* object already bound in its current location - so as long as meets the
* constraints imposed by the new execbuffer. Any object left unbound after the
* first pass is then fitted into any available idle space. If an object does
* not fit, all objects are removed from the reservation and the process rerun
* after sorting the objects into a priority order (more difficult to fit
* objects are tried first). Failing that, the entire VM is cleared and we try
* to fit the execbuf once last time before concluding that it simply will not
* fit.
*
* A small complication to all of this is that we allow userspace not only to
* specify an alignment and a size for the object in the address space, but
* we also allow userspace to specify the exact offset. This objects are
* simpler to place (the location is known a priori) all we have to do is make
* sure the space is available.
*
* Once all the objects are in place, patching up the buried pointers to point
* to the final locations is a fairly simple job of walking over the relocation
* entry arrays, looking up the right address and rewriting the value into
* the object. Simple! ... The relocation entries are stored in user memory
* and so to access them we have to copy them into a local buffer. That copy
* has to avoid taking any pagefaults as they may lead back to a GEM object
* requiring the struct_mutex (i.e. recursive deadlock). So once again we split
* the relocation into multiple passes. First we try to do everything within an
* atomic context (avoid the pagefaults) which requires that we never wait. If
* we detect that we may wait, or if we need to fault, then we have to fallback
* to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
* bells yet?) Dropping the mutex means that we lose all the state we have
* built up so far for the execbuf and we must reset any global data. However,
* we do leave the objects pinned in their final locations - which is a
* potential issue for concurrent execbufs. Once we have left the mutex, we can
* allocate and copy all the relocation entries into a large array at our
* leisure, reacquire the mutex, reclaim all the objects and other state and
* then proceed to update any incorrect addresses with the objects.
*
* As we process the relocation entries, we maintain a record of whether the
* object is being written to. Using NORELOC, we expect userspace to provide
* this information instead. We also check whether we can skip the relocation
* by comparing the expected value inside the relocation entry with the target's
* final address. If they differ, we have to map the current object and rewrite
* the 4 or 8 byte pointer within.
*
* Serialising an execbuf is quite simple according to the rules of the GEM
* ABI. Execution within each context is ordered by the order of submission.
* Writes to any GEM object are in order of submission and are exclusive. Reads
* from a GEM object are unordered with respect to other reads, but ordered by
* writes. A write submitted after a read cannot occur before the read, and
* similarly any read submitted after a write cannot occur before the write.
* Writes are ordered between engines such that only one write occurs at any
* time (completing any reads beforehand) - using semaphores where available
* and CPU serialisation otherwise. Other GEM access obey the same rules, any
* write (either via mmaps using set-domain, or via pwrite) must flush all GPU
* reads before starting, and any read (either using set-domain or pread) must
* flush all GPU writes before starting. (Note we only employ a barrier before,
* we currently rely on userspace not concurrently starting a new execution
* whilst reading or writing to an object. This may be an advantage or not
* depending on how much you trust userspace not to shoot themselves in the
* foot.) Serialisation may just result in the request being inserted into
* a DAG awaiting its turn, but most simple is to wait on the CPU until
* all dependencies are resolved.
*
* After all of that, is just a matter of closing the request and handing it to
* the hardware (well, leaving it in a queue to be executed). However, we also
* offer the ability for batchbuffers to be run with elevated privileges so
* that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
* Before any batch is given extra privileges we first must check that it
* contains no nefarious instructions, we check that each instruction is from
* our whitelist and all registers are also from an allowed list. We first
* copy the user's batchbuffer to a shadow (so that the user doesn't have
* access to it, either by the CPU or GPU as we scan it) and then parse each
* instruction. If everything is ok, we set a flag telling the hardware to run
* the batchbuffer in trusted mode, otherwise the ioctl is rejected.
*/

232struct eb_fence {
struct drm_syncobj *syncobj; /* Use with ptr_mask_bits() */
struct dma_fence *dma_fence;
u64 value;
struct dma_fence_chain *chain_fence;
237};

239struct i915_execbuffer {
struct drm_i915_privateinteldrm_softc *i915; /** i915 backpointer */
struct drm_file *file; /** per-file lookup tables and limits */
struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
struct eb_vma *vma;

struct intel_engine_cs *engine; /** engine to queue the request to */
struct intel_context *context; /* logical state for the request */
struct i915_gem_context *gem_context; /** caller's context */

struct i915_request *request; /** our request to build */
struct eb_vma *batch; /** identity of the batch obj/vma */
struct i915_vma *trampoline; /** trampoline used for chaining */

/** actual size of execobj[] as we may extend it for the cmdparser */
unsigned int buffer_count;

/** list of vma not yet bound during reservation phase */
struct list_head unbound;

/** list of vma that have execobj.relocation_count */
struct list_head relocs;

struct i915_gem_ww_ctx ww;

/**
* Track the most recently used object for relocations, as we
* frequently have to perform multiple relocations within the same
* obj/page
*/
struct reloc_cache {
struct drm_mm_node node; /** temporary GTT binding */
unsigned long vaddr; /** Current kmap address */
unsigned long page; /** Currently mapped page index */
unsigned int gen; /** Cached value of INTEL_GEN */
bool_Bool use_64bit_reloc : 1;
bool_Bool has_llc : 1;
bool_Bool has_fence : 1;
bool_Bool needs_unfenced : 1;

struct i915_request *rq;
u32 *rq_cmd;
unsigned int rq_size;
struct intel_gt_buffer_pool_node *pool;

struct agp_map *map;
bus_space_tag_t iot;
bus_space_handle_t ioh;
} reloc_cache;

struct intel_gt_buffer_pool_node *reloc_pool; /** relocation pool for -EDEADLK handling */
struct intel_context *reloc_context;

u64 invalid_flags; /** Set of execobj.flags that are invalid */
u32 context_flags; /** Set of execobj.flags to insert from the ctx */

u64 batch_len; /** Length of batch within object */
u32 batch_start_offset; /** Location within object of batch */
u32 batch_flags; /** Flags composed for emit_bb_start() */
struct intel_gt_buffer_pool_node *batch_pool; /** pool node for batch buffer */

/**
* Indicate either the size of the hastable used to resolve
* relocation handles, or if negative that we are using a direct
* index into the execobj[].
*/
int lut_size;
struct hlist_head *buckets; /** ht for relocation handles */

struct eb_fence *fences;
unsigned long num_fences;
311};

313static int eb_parse(struct i915_execbuffer *eb);
314static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb,
			  bool_Bool throttle);
316static void eb_unpin_engine(struct i915_execbuffer *eb);

318static inline bool_Bool eb_use_cmdparser(const struct i915_execbuffer *eb)
319{
return intel_engine_requires_cmd_parser(eb->engine) ||
(intel_engine_using_cmd_parser(eb->engine) &&
 eb->args->batch_len);
323}

325static int eb_create(struct i915_execbuffer *eb)
326{
if (!(eb->args->flags & I915_EXEC_HANDLE_LUT(1<<12))) {
28
←
Assuming the condition is true→
29
←
Taking true branch→
unsigned int size = 1 + ilog2(eb->buffer_count)((sizeof(eb->buffer_count) <= 4) ? (fls(eb->buffer_count
) - 1) : (flsl(eb->buffer_count) - 1));
30
←
'?' condition is true→

/*
 * Without a 1:1 association between relocation handles and
 * the execobject[] index, we instead create a hashtable.
 * We size it dynamically based on available memory, starting
 * first with 1:1 assocative hash and scaling back until
 * the allocation succeeds.
 *
 * Later on we use a positive lut_size to indicate we are
 * using this hashtable, and a negative value to indicate a
 * direct lookup.
 */
do {
36
←
Loop condition is true. Execution continues on line 342→
	gfp_t flags;

	/* While we can still reduce the allocation size, don't
	 * raise a warning and allow the allocation to fail.
	 * On the last pass though, we want to try as hard
	 * as possible to perform the allocation and warn
	 * if it fails.
	 */
	flags = GFP_KERNEL(0x0001 | 0x0004);
	if (size > 1)
31
←
Assuming 'size' is <= 1→
32
←
Taking false branch→
37
←
Assuming 'size' is > 1→
38
←
Taking true branch→
		flags |= __GFP_NORETRY0 | __GFP_NOWARN0;

	eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
39
←
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'unsigned long'
			      flags);
	if (eb->buckets)
33
←
Assuming field 'buckets' is null→
34
←
Taking false branch→
		break;
} while (--size);
35
←
Value assigned to 'size'→

if (unlikely(!size)__builtin_expect(!!(!size), 0))
	return -ENOMEM12;

eb->lut_size = size;
} else {
eb->lut_size = -eb->buffer_count;
}

return 0;
369}

371static bool_Bool
372eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
 const struct i915_vma *vma,
 unsigned int flags)
375{
if (vma->node.size < entry->pad_to_size)
return true1;

if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment)(((vma->node.start) & ((entry->alignment) - 1)) == 0
))
return true1;

if (flags & EXEC_OBJECT_PINNED(1<<4) &&
   vma->node.start != entry->offset)
return true1;

if (flags & __EXEC_OBJECT_NEEDS_BIAS(1UL << (28)) &&
   vma->node.start < BATCH_OFFSET_BIAS(256*1024))
return true1;

if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS(1<<3)) &&
   (vma->node.start + vma->node.size + 4095) >> 32)
return true1;

if (flags & __EXEC_OBJECT_NEEDS_MAP(1UL << (29)) &&
   !i915_vma_is_map_and_fenceable(vma))
return true1;

return false0;
399}

401static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry,
	unsigned int exec_flags)
403{
u64 pin_flags = 0;

if (exec_flags & EXEC_OBJECT_NEEDS_GTT(1<<1))
pin_flags |= PIN_GLOBAL(1ULL << (10));

/*
* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
* limit address to the first 4GBs for unflagged objects.
*/
if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS(1<<3)))
pin_flags |= PIN_ZONE_4G(1ULL << (4));

if (exec_flags & __EXEC_OBJECT_NEEDS_MAP(1UL << (29)))
pin_flags |= PIN_MAPPABLE(1ULL << (3));

if (exec_flags & EXEC_OBJECT_PINNED(1<<4))
pin_flags |= entry->offset | PIN_OFFSET_FIXED(1ULL << (7));
else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS(1UL << (28)))
pin_flags |= BATCH_OFFSET_BIAS(256*1024) | PIN_OFFSET_BIAS(1ULL << (6));

return pin_flags;
425}

427static inline bool_Bool
428eb_pin_vma(struct i915_execbuffer *eb,
  const struct drm_i915_gem_exec_object2 *entry,
  struct eb_vma *ev)
431{
struct i915_vma *vma = ev->vma;
u64 pin_flags;

if (vma->node.size)
pin_flags = vma->node.start;
else
pin_flags = entry->offset & PIN_OFFSET_MASK-(1ULL << (12));

pin_flags |= PIN_USER(1ULL << (11)) | PIN_NOEVICT(1ULL << (0)) | PIN_OFFSET_FIXED(1ULL << (7));
if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT)__builtin_expect(!!(ev->flags & (1<<1)), 0))
pin_flags |= PIN_GLOBAL(1ULL << (10));

/* Attempt to reuse the current location if available */
/* TODO: Add -EDEADLK handling here */
if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))__builtin_expect(!!(i915_vma_pin_ww(vma, &eb->ww, 0, 0
, pin_flags)), 0)) {
if (entry->flags & EXEC_OBJECT_PINNED(1<<4))
	return false0;

/* Failing that pick any _free_ space if suitable */
if (unlikely(i915_vma_pin_ww(vma, &eb->ww,__builtin_expect(!!(i915_vma_pin_ww(vma, &eb->ww, entry
->pad_to_size, entry->alignment, eb_pin_flags(entry, ev
->flags) | (1ULL << (11)) | (1ULL << (0)))), 0
)
			     entry->pad_to_size,__builtin_expect(!!(i915_vma_pin_ww(vma, &eb->ww, entry
->pad_to_size, entry->alignment, eb_pin_flags(entry, ev
->flags) | (1ULL << (11)) | (1ULL << (0)))), 0
)
			     entry->alignment,__builtin_expect(!!(i915_vma_pin_ww(vma, &eb->ww, entry
->pad_to_size, entry->alignment, eb_pin_flags(entry, ev
->flags) | (1ULL << (11)) | (1ULL << (0)))), 0
)
			     eb_pin_flags(entry, ev->flags) |__builtin_expect(!!(i915_vma_pin_ww(vma, &eb->ww, entry
->pad_to_size, entry->alignment, eb_pin_flags(entry, ev
->flags) | (1ULL << (11)) | (1ULL << (0)))), 0
)
			     PIN_USER | PIN_NOEVICT))__builtin_expect(!!(i915_vma_pin_ww(vma, &eb->ww, entry
->pad_to_size, entry->alignment, eb_pin_flags(entry, ev
->flags) | (1ULL << (11)) | (1ULL << (0)))), 0
))
	return false0;
}

if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)__builtin_expect(!!(ev->flags & (1<<0)), 0)) {
if (unlikely(i915_vma_pin_fence(vma))__builtin_expect(!!(i915_vma_pin_fence(vma)), 0)) {
	i915_vma_unpin(vma);
	return false0;
}

if (vma->fence)
	ev->flags |= __EXEC_OBJECT_HAS_FENCE(1UL << (30));
}

ev->flags |= __EXEC_OBJECT_HAS_PIN(1UL << (31));
return !eb_vma_misplaced(entry, vma, ev->flags);
471}

473static inline void
474eb_unreserve_vma(struct eb_vma *ev)
475{
if (!(ev->flags & __EXEC_OBJECT_HAS_PIN(1UL << (31))))
return;

if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)__builtin_expect(!!(ev->flags & (1UL << (30))), 0
))
__i915_vma_unpin_fence(ev->vma);

__i915_vma_unpin(ev->vma);
ev->flags &= ~__EXEC_OBJECT_RESERVED((1UL << (31)) | (1UL << (30)));
484}

486static int
487eb_validate_vma(struct i915_execbuffer *eb,
struct drm_i915_gem_exec_object2 *entry,
struct i915_vma *vma)
490{
if (unlikely(entry->flags & eb->invalid_flags)__builtin_expect(!!(entry->flags & eb->invalid_flags
), 0))
return -EINVAL22;

if (unlikely(entry->alignment &&__builtin_expect(!!(entry->alignment && !is_power_of_2_u64
(entry->alignment)), 0)
     !is_power_of_2_u64(entry->alignment))__builtin_expect(!!(entry->alignment && !is_power_of_2_u64
(entry->alignment)), 0))
return -EINVAL22;

/*
* Offset can be used as input (EXEC_OBJECT_PINNED), reject
* any non-page-aligned or non-canonical addresses.
*/
if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&__builtin_expect(!!(entry->flags & (1<<4) &&
 entry->offset != gen8_canonical_addr(entry->offset &
 -(1ULL << (12)))), 0)
     entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK))__builtin_expect(!!(entry->flags & (1<<4) &&
 entry->offset != gen8_canonical_addr(entry->offset &
 -(1ULL << (12)))), 0))
return -EINVAL22;

/* pad_to_size was once a reserved field, so sanitize it */
if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE(1<<5)) {
if (unlikely(offset_in_page(entry->pad_to_size))__builtin_expect(!!(((vaddr_t)(entry->pad_to_size) & (
(1 << 12) - 1))), 0))
	return -EINVAL22;
} else {
entry->pad_to_size = 0;
}
/*
* From drm_mm perspective address space is continuous,
* so from this point we're always using non-canonical
* form internally.
*/
entry->offset = gen8_noncanonical_addr(entry->offset);

if (!eb->reloc_cache.has_fence) {
entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE(1<<0);
} else {
if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE(1<<0) ||
     eb->reloc_cache.needs_unfenced) &&
    i915_gem_object_is_tiled(vma->obj))
	entry->flags |= EXEC_OBJECT_NEEDS_GTT(1<<1) | __EXEC_OBJECT_NEEDS_MAP(1UL << (29));
}

if (!(entry->flags & EXEC_OBJECT_PINNED(1<<4)))
entry->flags |= eb->context_flags;

return 0;
533}

535static void
536eb_add_vma(struct i915_execbuffer *eb,
  unsigned int i, unsigned batch_idx,
  struct i915_vma *vma)
539{
struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
struct eb_vma *ev = &eb->vma[i];

GEM_BUG_ON(i915_vma_is_closed(vma))((void)0);

ev->vma = vma;
ev->exec = entry;
ev->flags = entry->flags;

if (eb->lut_size > 0) {
ev->handle = entry->handle;
hlist_add_head(&ev->node,
	       &eb->buckets[hash_32(entry->handle,
				    eb->lut_size)]);
}

if (entry->relocation_count)
list_add_tail(&ev->reloc_link, &eb->relocs);

/*
* SNA is doing fancy tricks with compressing batch buffers, which leads
* to negative relocation deltas. Usually that works out ok since the
* relocate address is still positive, except when the batch is placed
* very low in the GTT. Ensure this doesn't happen.
*
* Note that actual hangs have only been observed on gen7, but for
* paranoia do it everywhere.
*/
if (i == batch_idx) {
if (entry->relocation_count &&
    !(ev->flags & EXEC_OBJECT_PINNED(1<<4)))
	ev->flags |= __EXEC_OBJECT_NEEDS_BIAS(1UL << (28));
if (eb->reloc_cache.has_fence)
	ev->flags |= EXEC_OBJECT_NEEDS_FENCE(1<<0);

eb->batch = ev;
}
577}

579static inline int use_cpu_reloc(const struct reloc_cache *cache,
		const struct drm_i915_gem_object *obj)
581{
if (!i915_gem_object_has_struct_page(obj))
return false0;

if (DBG_FORCE_RELOC0 == FORCE_CPU_RELOC)
return true1;

if (DBG_FORCE_RELOC0 == FORCE_GTT_RELOC)
return false0;

return (cache->has_llc ||
obj->cache_dirty ||
obj->cache_level != I915_CACHE_NONE);
594}

596static int eb_reserve_vma(struct i915_execbuffer *eb,
	  struct eb_vma *ev,
	  u64 pin_flags)
599{
struct drm_i915_gem_exec_object2 *entry = ev->exec;
struct i915_vma *vma = ev->vma;
int err;

if (drm_mm_node_allocated(&vma->node) &&
   eb_vma_misplaced(entry, vma, ev->flags)) {
err = i915_vma_unbind(vma);
if (err)
	return err;
}

err = i915_vma_pin_ww(vma, &eb->ww,
	   entry->pad_to_size, entry->alignment,
	   eb_pin_flags(entry, ev->flags) | pin_flags);
if (err)
return err;

if (entry->offset != vma->node.start) {
entry->offset = vma->node.start | UPDATE(1ULL << (7));
eb->args->flags |= __EXEC_HAS_RELOC(1UL << (31));
}

if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)__builtin_expect(!!(ev->flags & (1<<0)), 0)) {
err = i915_vma_pin_fence(vma);
if (unlikely(err)__builtin_expect(!!(err), 0)) {
	i915_vma_unpin(vma);
	return err;
}

if (vma->fence)
	ev->flags |= __EXEC_OBJECT_HAS_FENCE(1UL << (30));
}

ev->flags |= __EXEC_OBJECT_HAS_PIN(1UL << (31));
GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags))((void)0);

return 0;
637}

639static int eb_reserve(struct i915_execbuffer *eb)
640{
const unsigned int count = eb->buffer_count;
unsigned int pin_flags = PIN_USER(1ULL << (11)) | PIN_NONBLOCK(1ULL << (2));
struct list_head last;
struct eb_vma *ev;
unsigned int i, pass;
int err = 0;

/*
* Attempt to pin all of the buffers into the GTT.
* This is done in 3 phases:
*
* 1a. Unbind all objects that do not match the GTT constraints for
*     the execbuffer (fenceable, mappable, alignment etc).
* 1b. Increment pin count for already bound objects.
* 2.  Bind new objects.
* 3.  Decrement pin count.
*
* This avoid unnecessary unbinding of later objects in order to make
* room for the earlier objects *unless* we need to defragment.
*/
pass = 0;
do {
list_for_each_entry(ev, &eb->unbound, bind_link)for (ev = ({ const __typeof( ((__typeof(*ev) *)0)->bind_link
 ) *__mptr = ((&eb->unbound)->next); (__typeof(*ev)
 *)( (char *)__mptr - __builtin_offsetof(__typeof(*ev), bind_link
) );}); &ev->bind_link != (&eb->unbound); ev = (
{ const __typeof( ((__typeof(*ev) *)0)->bind_link ) *__mptr
 = (ev->bind_link.next); (__typeof(*ev) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*ev), bind_link) );})) {
	err = eb_reserve_vma(eb, ev, pin_flags);
	if (err)
		break;
}
if (err != -ENOSPC28)
	return err;

/* Resort *all* the objects into priority order */
INIT_LIST_HEAD(&eb->unbound);
INIT_LIST_HEAD(&last);
for (i = 0; i < count; i++) {
	unsigned int flags;

	ev = &eb->vma[i];
	flags = ev->flags;
	if (flags & EXEC_OBJECT_PINNED(1<<4) &&
	    flags & __EXEC_OBJECT_HAS_PIN(1UL << (31)))
		continue;

	eb_unreserve_vma(ev);

	if (flags & EXEC_OBJECT_PINNED(1<<4))
		/* Pinned must have their slot */
		list_add(&ev->bind_link, &eb->unbound);
	else if (flags & __EXEC_OBJECT_NEEDS_MAP(1UL << (29)))
		/* Map require the lowest 256MiB (aperture) */
		list_add_tail(&ev->bind_link, &eb->unbound);
	else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS(1<<3)))
		/* Prioritise 4GiB region for restricted bo */
		list_add(&ev->bind_link, &last);
	else
		list_add_tail(&ev->bind_link, &last);
}
list_splice_tail(&last, &eb->unbound);

switch (pass++) {
case 0:
	break;

case 1:
	/* Too fragmented, unbind everything and retry */
	mutex_lock(&eb->context->vm->mutex)rw_enter_write(&eb->context->vm->mutex);
	err = i915_gem_evict_vm(eb->context->vm);
	mutex_unlock(&eb->context->vm->mutex)rw_exit_write(&eb->context->vm->mutex);
	if (err)
		return err;
	break;

default:
	return -ENOSPC28;
}

pin_flags = PIN_USER(1ULL << (11));
} while (1);
718}

720static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
721{
if (eb->args->flags & I915_EXEC_BATCH_FIRST(1<<18))
return 0;
else
return eb->buffer_count - 1;
726}

728static int eb_select_context(struct i915_execbuffer *eb)
729{
struct i915_gem_context *ctx;

ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
if (unlikely(!ctx)__builtin_expect(!!(!ctx), 0))
return -ENOENT2;

eb->gem_context = ctx;
if (rcu_access_pointer(ctx->vm)(ctx->vm))
eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT(1<<1);

eb->context_flags = 0;
if (test_bit(UCONTEXT_NO_ZEROMAP0, &ctx->user_flags))
eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS(1UL << (28));

return 0;
745}

747static int __eb_add_lut(struct i915_execbuffer *eb,
	u32 handle, struct i915_vma *vma)
749{
struct i915_gem_context *ctx = eb->gem_context;
struct i915_lut_handle *lut;
int err;

lut = i915_lut_handle_alloc();
if (unlikely(!lut)__builtin_expect(!!(!lut), 0))
return -ENOMEM12;

i915_vma_get(vma);
if (!atomic_fetch_inc(&vma->open_count)__sync_fetch_and_add(&vma->open_count, 1))
i915_vma_reopen(vma);
lut->handle = handle;
lut->ctx = ctx;

/* Check that the context hasn't been closed in the meantime */
err = -EINTR4;
if (!mutex_lock_interruptible(&ctx->lut_mutex)) {
struct i915_address_space *vm = rcu_access_pointer(ctx->vm)(ctx->vm);

if (unlikely(vm && vma->vm != vm)__builtin_expect(!!(vm && vma->vm != vm), 0))
	err = -EAGAIN35; /* user racing with ctx set-vm */
else if (likely(!i915_gem_context_is_closed(ctx))__builtin_expect(!!(!i915_gem_context_is_closed(ctx)), 1))
	err = radix_tree_insert(&ctx->handles_vma, handle, vma);
else
	err = -ENOENT2;
if (err == 0) { /* And nor has this handle */
	struct drm_i915_gem_object *obj = vma->obj;

	spin_lock(&obj->lut_lock)mtx_enter(&obj->lut_lock);
	if (idr_find(&eb->file->object_idr, handle) == obj) {
		list_add(&lut->obj_link, &obj->lut_list);
	} else {
		radix_tree_delete(&ctx->handles_vma, handle);
		err = -ENOENT2;
	}
	spin_unlock(&obj->lut_lock)mtx_leave(&obj->lut_lock);
}
mutex_unlock(&ctx->lut_mutex)rw_exit_write(&ctx->lut_mutex);
}
if (unlikely(err)__builtin_expect(!!(err), 0))
goto err;

return 0;

794err:
i915_vma_close(vma);
i915_vma_put(vma);
i915_lut_handle_free(lut);
return err;
799}

801static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle)
802{
struct i915_address_space *vm = eb->context->vm;

do {
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int err;

rcu_read_lock();
vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle);
if (likely(vma && vma->vm == vm)__builtin_expect(!!(vma && vma->vm == vm), 1))
	vma = i915_vma_tryget(vma);
rcu_read_unlock();
if (likely(vma)__builtin_expect(!!(vma), 1))
	return vma;

obj = i915_gem_object_lookup(eb->file, handle);
if (unlikely(!obj)__builtin_expect(!!(!obj), 0))
	return ERR_PTR(-ENOENT2);

vma = i915_vma_instance(obj, vm, NULL((void *)0));
if (IS_ERR(vma)) {
	i915_gem_object_put(obj);
	return vma;
}

err = __eb_add_lut(eb, handle, vma);
if (likely(!err)__builtin_expect(!!(!err), 1))
	return vma;

i915_gem_object_put(obj);
if (err != -EEXIST17)
	return ERR_PTR(err);
} while (1);
836}

838static int eb_lookup_vmas(struct i915_execbuffer *eb)
839{
struct drm_i915_privateinteldrm_softc *i915 = eb->i915;
unsigned int batch = eb_batch_index(eb);
unsigned int i;
int err = 0;

INIT_LIST_HEAD(&eb->relocs);

for (i = 0; i < eb->buffer_count; i++) {
struct i915_vma *vma;

vma = eb_lookup_vma(eb, eb->exec[i].handle);
if (IS_ERR(vma)) {
	err = PTR_ERR(vma);
	goto err;
}

err = eb_validate_vma(eb, &eb->exec[i], vma);
if (unlikely(err)__builtin_expect(!!(err), 0)) {
	i915_vma_put(vma);
	goto err;
}

eb_add_vma(eb, i, batch, vma);
}

if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)__builtin_expect(!!(eb->batch->flags & (1<<2)
), 0)) {
drm_dbg(&i915->drm,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Attempting to use self-modifying batch buffer\n"
)
	"Attempting to use self-modifying batch buffer\n")drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Attempting to use self-modifying batch buffer\n"
);
return -EINVAL22;
}

if (range_overflows_t(u64,({ typeof((u64)(eb->batch_start_offset)) start__ = ((u64)(
eb->batch_start_offset)); typeof((u64)(eb->batch_len)) size__
 = ((u64)(eb->batch_len)); typeof((u64)(eb->batch->vma
->size)) max__ = ((u64)(eb->batch->vma->size)); (
void)(&start__ == &size__); (void)(&start__ == &
max__); start__ >= max__ || size__ > max__ - start__; }
)
	      eb->batch_start_offset, eb->batch_len,({ typeof((u64)(eb->batch_start_offset)) start__ = ((u64)(
eb->batch_start_offset)); typeof((u64)(eb->batch_len)) size__
 = ((u64)(eb->batch_len)); typeof((u64)(eb->batch->vma
->size)) max__ = ((u64)(eb->batch->vma->size)); (
void)(&start__ == &size__); (void)(&start__ == &
max__); start__ >= max__ || size__ > max__ - start__; }
)
	      eb->batch->vma->size)({ typeof((u64)(eb->batch_start_offset)) start__ = ((u64)(
eb->batch_start_offset)); typeof((u64)(eb->batch_len)) size__
 = ((u64)(eb->batch_len)); typeof((u64)(eb->batch->vma
->size)) max__ = ((u64)(eb->batch->vma->size)); (
void)(&start__ == &size__); (void)(&start__ == &
max__); start__ >= max__ || size__ > max__ - start__; }
)) {
drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n")drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Attempting to use out-of-bounds batch\n"
);
return -EINVAL22;
}

if (eb->batch_len == 0)
eb->batch_len = eb->batch->vma->size - eb->batch_start_offset;
if (unlikely(eb->batch_len == 0)__builtin_expect(!!(eb->batch_len == 0), 0)) { /* impossible! */
drm_dbg(&i915->drm, "Invalid batch length\n")drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Invalid batch length\n"
);
return -EINVAL22;
}

return 0;

887err:
eb->vma[i].vma = NULL((void *)0);
return err;
890}

892static int eb_validate_vmas(struct i915_execbuffer *eb)
893{
unsigned int i;
int err;

INIT_LIST_HEAD(&eb->unbound);

for (i = 0; i < eb->buffer_count; i++) {
struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
struct eb_vma *ev = &eb->vma[i];
struct i915_vma *vma = ev->vma;

err = i915_gem_object_lock(vma->obj, &eb->ww);
if (err)
	return err;

if (eb_pin_vma(eb, entry, ev)) {
	if (entry->offset != vma->node.start) {
		entry->offset = vma->node.start | UPDATE(1ULL << (7));
		eb->args->flags |= __EXEC_HAS_RELOC(1UL << (31));
	}
} else {
	eb_unreserve_vma(ev);

	list_add_tail(&ev->bind_link, &eb->unbound);
	if (drm_mm_node_allocated(&vma->node)) {
		err = i915_vma_unbind(vma);
		if (err)
			return err;
	}
}

GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&((void)0)
	   eb_vma_misplaced(&eb->exec[i], vma, ev->flags))((void)0);
}

if (!list_empty(&eb->unbound))
return eb_reserve(eb);

return 0;
932}

934static struct eb_vma *
935eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
936{
if (eb->lut_size < 0) {
if (handle >= -eb->lut_size)
	return NULL((void *)0);
return &eb->vma[handle];
} else {
struct hlist_head *head;
struct eb_vma *ev;

head = &eb->buckets[hash_32(handle, eb->lut_size)];
hlist_for_each_entry(ev, head, node)for (ev = (((head)->first) ? ({ const __typeof( ((__typeof
(*ev) *)0)->node ) *__mptr = ((head)->first); (__typeof
(*ev) *)( (char *)__mptr - __builtin_offsetof(__typeof(*ev), node
) );}) : ((void *)0)); ev != ((void *)0); ev = (((ev)->node
.next) ? ({ const __typeof( ((__typeof(*ev) *)0)->node ) *
__mptr = ((ev)->node.next); (__typeof(*ev) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*ev), node) );}) : ((void *)0)
)) {
	if (ev->handle == handle)
		return ev;
}
return NULL((void *)0);
}
952}

954static void eb_release_vmas(struct i915_execbuffer *eb, bool_Bool final)
955{
const unsigned int count = eb->buffer_count;
unsigned int i;

for (i = 0; i < count; i++) {
struct eb_vma *ev = &eb->vma[i];
struct i915_vma *vma = ev->vma;

if (!vma)
	break;

eb_unreserve_vma(ev);

if (final)
	i915_vma_put(vma);
}

eb_unpin_engine(eb);
973}

975static void eb_destroy(const struct i915_execbuffer *eb)
976{
GEM_BUG_ON(eb->reloc_cache.rq)((void)0);

if (eb->lut_size > 0)
kfree(eb->buckets);
981}

983static inline u64
984relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
  const struct i915_vma *target)
986{
return gen8_canonical_addr((int)reloc->delta + target->node.start);
988}

990static void reloc_cache_clear(struct reloc_cache *cache)
991{
cache->rq = NULL((void *)0);
cache->rq_cmd = NULL((void *)0);
cache->pool = NULL((void *)0);
cache->rq_size = 0;
996}

998static void reloc_cache_init(struct reloc_cache *cache,
	     struct drm_i915_privateinteldrm_softc *i915)
1000{
cache->page = -1;
cache->vaddr = 0;
/* Must be a variable in the struct to allow GCC to unroll. */
cache->gen = INTEL_GEN(i915)((&(i915)->__info)->gen);
cache->has_llc = HAS_LLC(i915)((&(i915)->__info)->has_llc);
cache->use_64bit_reloc = HAS_64BIT_RELOC(i915)((&(i915)->__info)->has_64bit_reloc);
cache->has_fence = cache->gen < 4;
cache->needs_unfenced = INTEL_INFO(i915)(&(i915)->__info)->unfenced_needs_alignment;
cache->node.flags = 0;

cache->map = i915->agph;
cache->iot = i915->bst;

reloc_cache_clear(cache);
1015}

1017static inline void *unmask_page(unsigned long p)
1018{
return (void *)(uintptr_t)(p & LINUX_PAGE_MASK(~((1 << 12) - 1)));
1020}

1022static inline unsigned int unmask_flags(unsigned long p)
1023{
return p & ~LINUX_PAGE_MASK(~((1 << 12) - 1));
1025}

1027#define KMAP0x4 0x4 /* after CLFLUSH_FLAGS */

1029static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
1030{
struct drm_i915_privateinteldrm_softc *i915 =
container_of(cache, struct i915_execbuffer, reloc_cache)({ const __typeof( ((struct i915_execbuffer *)0)->reloc_cache
 ) *__mptr = (cache); (struct i915_execbuffer *)( (char *)__mptr
 - __builtin_offsetof(struct i915_execbuffer, reloc_cache) );
})->i915;
return &i915->ggtt;
1034}

1036static void reloc_cache_put_pool(struct i915_execbuffer *eb, struct reloc_cache *cache)
1037{
if (!cache->pool)
return;

/*
* This is a bit nasty, normally we keep objects locked until the end
* of execbuffer, but we already submit this, and have to unlock before
* dropping the reference. Fortunately we can only hold 1 pool node at
* a time, so this should be harmless.
*/
i915_gem_ww_unlock_single(cache->pool->obj);
intel_gt_buffer_pool_put(cache->pool);
cache->pool = NULL((void *)0);
1050}

1052static void reloc_gpu_flush(struct i915_execbuffer *eb, struct reloc_cache *cache)
1053{
struct drm_i915_gem_object *obj = cache->rq->batch->obj;

GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32))((void)0);
cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END(((0x0a) << 23) | (0));

i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj);

intel_gt_chipset_flush(cache->rq->engine->gt);

i915_request_add(cache->rq);
reloc_cache_put_pool(eb, cache);
reloc_cache_clear(cache);

eb->reloc_pool = NULL((void *)0);
1069}

1071static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb)
1072{
void *vaddr;

if (cache->rq)
reloc_gpu_flush(eb, cache);

if (!cache->vaddr)
return;

vaddr = unmask_page(cache->vaddr);
if (cache->vaddr & KMAP0x4) {
struct drm_i915_gem_object *obj =
	(struct drm_i915_gem_object *)cache->node.mm;
if (cache->vaddr & CLFLUSH_AFTER(1UL << (1)))
	mb()do { __asm volatile("mfence" ::: "memory"); } while (0);

kunmap_atomic(vaddr);
i915_gem_object_finish_access(obj);
} else {
struct i915_ggtt *ggtt = cache_to_ggtt(cache);

intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1094#ifdef __linux__
io_mapping_unmap_atomic((void __iomem *)vaddr);
1096#else
agp_unmap_atomic(cache->map, cache->ioh);
1098#endif

if (drm_mm_node_allocated(&cache->node)) {
	ggtt->vm.clear_range(&ggtt->vm,
			     cache->node.start,
			     cache->node.size);
	mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex);
	drm_mm_remove_node(&cache->node);
	mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);
} else {
	i915_vma_unpin((struct i915_vma *)cache->node.mm);
}
}

cache->vaddr = 0;
cache->page = -1;
1114}

1116static void *reloc_kmap(struct drm_i915_gem_object *obj,
	struct reloc_cache *cache,
	unsigned long pageno)
1119{
void *vaddr;
struct vm_page *page;

if (cache->vaddr) {
kunmap_atomic(unmask_page(cache->vaddr));
} else {
unsigned int flushes;
int err;

err = i915_gem_object_prepare_write(obj, &flushes);
if (err)
	return ERR_PTR(err);

BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS)extern char _ctassert[(!(0x4 & ((1UL << (0)) | (1UL
 << (1))))) ? 1 : -1 ] __attribute__((__unused__));
BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & LINUX_PAGE_MASK)extern char _ctassert[(!((0x4 | ((1UL << (0)) | (1UL <<
 (1)))) & (~((1 << 12) - 1)))) ? 1 : -1 ] __attribute__
((__unused__));

cache->vaddr = flushes | KMAP0x4;
cache->node.mm = (void *)obj;
if (flushes)
	mb()do { __asm volatile("mfence" ::: "memory"); } while (0);
}

page = i915_gem_object_get_page(obj, pageno);
if (!obj->mm.dirty)
set_page_dirty(page)x86_atomic_clearbits_u32(&page->pg_flags, 0x00000008);

vaddr = kmap_atomic(page);
cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
cache->page = pageno;

return vaddr;
1151}

1153static void *reloc_iomap(struct drm_i915_gem_object *obj,
	 struct i915_execbuffer *eb,
	 unsigned long page)
1156{
struct reloc_cache *cache = &eb->reloc_cache;
struct i915_ggtt *ggtt = cache_to_ggtt(cache);
unsigned long offset;
void *vaddr;

if (cache->vaddr) {
intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1164#ifdef __linux__
io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1166#else
agp_unmap_atomic(cache->map, cache->ioh);
1168#endif
} else {
struct i915_vma *vma;
int err;

if (i915_gem_object_is_tiled(obj))
	return ERR_PTR(-EINVAL22);

if (use_cpu_reloc(cache, obj))
	return NULL((void *)0);

err = i915_gem_object_set_to_gtt_domain(obj, true1);
if (err)
	return ERR_PTR(err);

vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL((void *)0), 0, 0,
				  PIN_MAPPABLE(1ULL << (3)) |
				  PIN_NONBLOCK(1ULL << (2)) /* NOWARN */ |
				  PIN_NOEVICT(1ULL << (0)));
if (vma == ERR_PTR(-EDEADLK11))
	return vma;

if (IS_ERR(vma)) {
	memset(&cache->node, 0, sizeof(cache->node))__builtin_memset((&cache->node), (0), (sizeof(cache->
node)));
	mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex);
	err = drm_mm_insert_node_in_range
		(&ggtt->vm.mm, &cache->node,
		 PAGE_SIZE(1 << 12), 0, I915_COLOR_UNEVICTABLE(-1),
		 0, ggtt->mappable_end,
		 DRM_MM_INSERT_LOW);
	mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);
	if (err) /* no inactive aperture space, use cpu reloc */
		return NULL((void *)0);
} else {
	cache->node.start = vma->node.start;
	cache->node.mm = (void *)vma;
}
}

offset = cache->node.start;
if (drm_mm_node_allocated(&cache->node)) {
ggtt->vm.insert_page(&ggtt->vm,
		     i915_gem_object_get_dma_address(obj, page),
		     offset, I915_CACHE_NONE, 0);
} else {
offset += page << PAGE_SHIFT12;
}

1216#ifdef __linux__
vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
					 offset);
1219#else
agp_map_atomic(cache->map, offset, &cache->ioh);
vaddr = bus_space_vaddr(cache->iot, cache->ioh)((cache->iot)->vaddr((cache->ioh)));
1222#endif
cache->page = page;
cache->vaddr = (unsigned long)vaddr;

return vaddr;
1227}

1229static void *reloc_vaddr(struct drm_i915_gem_object *obj,
	 struct i915_execbuffer *eb,
	 unsigned long page)
1232{
struct reloc_cache *cache = &eb->reloc_cache;
void *vaddr;

if (cache->page == page) {
vaddr = unmask_page(cache->vaddr);
} else {
vaddr = NULL((void *)0);
if ((cache->vaddr & KMAP0x4) == 0)
	vaddr = reloc_iomap(obj, eb, page);
if (!vaddr)
	vaddr = reloc_kmap(obj, cache, page);
}

return vaddr;
1247}

1249static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1250{
if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))__builtin_expect(!!(flushes & ((1UL << (0)) | (1UL <<
 (1)))), 0)) {
if (flushes & CLFLUSH_BEFORE(1UL << (0))) {
	clflushopt(addr);
	mb()do { __asm volatile("mfence" ::: "memory"); } while (0);
}

*addr = value;

/*
 * Writes to the same cacheline are serialised by the CPU
 * (including clflush). On the write path, we only require
 * that it hits memory in an orderly fashion and place
 * mb barriers at the start and end of the relocation phase
 * to ensure ordering of clflush wrt to the system.
 */
if (flushes & CLFLUSH_AFTER(1UL << (1)))
	clflushopt(addr);
} else
*addr = value;
1270}

1272static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma)
1273{
struct drm_i915_gem_object *obj = vma->obj;
int err;

assert_vma_held(vma)do { (void)(&((vma)->resv)->lock.base); } while(0);

if (obj->cache_dirty & ~obj->cache_coherent)
i915_gem_clflush_object(obj, 0);
obj->write_domain = 0;

err = i915_request_await_object(rq, vma->obj, true1);
if (err == 0)
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE(1<<2));

return err;
1288}

1290static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
	     struct intel_engine_cs *engine,
	     struct i915_vma *vma,
	     unsigned int len)
1294{
struct reloc_cache *cache = &eb->reloc_cache;
struct intel_gt_buffer_pool_node *pool = eb->reloc_pool;
struct i915_request *rq;
struct i915_vma *batch;
u32 *cmd;
int err;

if (!pool) {
pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE(1 << 12));
if (IS_ERR(pool))
	return PTR_ERR(pool);
}
eb->reloc_pool = NULL((void *)0);

err = i915_gem_object_lock(pool->obj, &eb->ww);
if (err)
goto err_pool;

cmd = i915_gem_object_pin_map(pool->obj,
		      cache->has_llc ?
		      I915_MAP_FORCE_WB :
		      I915_MAP_FORCE_WC);
if (IS_ERR(cmd)) {
err = PTR_ERR(cmd);
goto err_pool;
}

memset32(cmd, 0, pool->obj->base.size / sizeof(u32));

batch = i915_vma_instance(pool->obj, vma->vm, NULL((void *)0));
if (IS_ERR(batch)) {
err = PTR_ERR(batch);
goto err_unmap;
}

err = i915_vma_pin_ww(batch, &eb->ww, 0, 0, PIN_USER(1ULL << (11)) | PIN_NONBLOCK(1ULL << (2)));
if (err)
goto err_unmap;

if (engine == eb->context->engine) {
rq = i915_request_create(eb->context);
} else {
struct intel_context *ce = eb->reloc_context;

if (!ce) {
	ce = intel_context_create(engine);
	if (IS_ERR(ce)) {
		err = PTR_ERR(ce);
		goto err_unpin;
	}

	i915_vm_put(ce->vm);
	ce->vm = i915_vm_get(eb->context->vm);
	eb->reloc_context = ce;
}

err = intel_context_pin_ww(ce, &eb->ww);
if (err)
	goto err_unpin;

rq = i915_request_create(ce);
intel_context_unpin(ce);
}
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
goto err_unpin;
}

err = intel_gt_buffer_pool_mark_active(pool, rq);
if (err)
goto err_request;

err = reloc_move_to_gpu(rq, vma);
if (err)
goto err_request;

err = eb->engine->emit_bb_start(rq,
			batch->node.start, PAGE_SIZE(1 << 12),
			cache->gen > 5 ? 0 : I915_DISPATCH_SECURE(1UL << (0)));
if (err)
goto skip_request;

assert_vma_held(batch)do { (void)(&((batch)->resv)->lock.base); } while(0
);
err = i915_request_await_object(rq, batch->obj, false0);
if (err == 0)
err = i915_vma_move_to_active(batch, rq, 0);
if (err)
goto skip_request;

rq->batch = batch;
i915_vma_unpin(batch);

cache->rq = rq;
cache->rq_cmd = cmd;
cache->rq_size = 0;
cache->pool = pool;

/* Return with batch mapping (cmd) still pinned */
return 0;

1395skip_request:
i915_request_set_error_once(rq, err);
1397err_request:
i915_request_add(rq);
1399err_unpin:
i915_vma_unpin(batch);
1401err_unmap:
i915_gem_object_unpin_map(pool->obj);
1403err_pool:
eb->reloc_pool = pool;
return err;
1406}

1408static bool_Bool reloc_can_use_engine(const struct intel_engine_cs *engine)
1409{
return engine->class != VIDEO_DECODE_CLASS1 || !IS_GEN(engine->i915, 6)(0 + (&(engine->i915)->__info)->gen == (6));
1411}

1413static u32 *reloc_gpu(struct i915_execbuffer *eb,
      struct i915_vma *vma,
      unsigned int len)
1416{
struct reloc_cache *cache = &eb->reloc_cache;
u32 *cmd;

if (cache->rq_size > PAGE_SIZE(1 << 12)/sizeof(u32) - (len + 1))
reloc_gpu_flush(eb, cache);

if (unlikely(!cache->rq)__builtin_expect(!!(!cache->rq), 0)) {
int err;
struct intel_engine_cs *engine = eb->engine;

/* If we need to copy for the cmdparser, we will stall anyway */
if (eb_use_cmdparser(eb))
	return ERR_PTR(-EWOULDBLOCK35);

if (!reloc_can_use_engine(engine)) {
	engine = engine->gt->engine_class[COPY_ENGINE_CLASS3][0];
	if (!engine)
		return ERR_PTR(-ENODEV19);
}

err = __reloc_gpu_alloc(eb, engine, vma, len);
if (unlikely(err)__builtin_expect(!!(err), 0))
	return ERR_PTR(err);
}

cmd = cache->rq_cmd + cache->rq_size;
cache->rq_size += len;

return cmd;
1446}

1448static inline bool_Bool use_reloc_gpu(struct i915_vma *vma)
1449{
if (DBG_FORCE_RELOC0 == FORCE_GPU_RELOC)
return true1;

if (DBG_FORCE_RELOC0)
return false0;

return !dma_resv_test_signaled_rcu(vma->resv, true1);
1457}

1459static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset)
1460{
struct vm_page *page;
unsigned long addr;

GEM_BUG_ON(vma->pages != vma->obj->mm.pages)((void)0);

page = i915_gem_object_get_page(vma->obj, offset >> PAGE_SHIFT12);
addr = PFN_PHYS(page_to_pfn(page))(((((page)->phys_addr) / (1 << 12))) << 12);
GEM_BUG_ON(overflows_type(addr, u32))((void)0); /* expected dma32 */

return addr + offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1));
1471}

1473static int __reloc_entry_gpu(struct i915_execbuffer *eb,
	      struct i915_vma *vma,
	      u64 offset,
	      u64 target_addr)
1477{
const unsigned int gen = eb->reloc_cache.gen;
unsigned int len;
u32 *batch;
u64 addr;

if (gen >= 8)
len = offset & 7 ? 8 : 5;
else if (gen >= 4)
len = 4;
else
len = 3;

batch = reloc_gpu(eb, vma, len);
if (batch == ERR_PTR(-EDEADLK11))
return -EDEADLK11;
else if (IS_ERR(batch))
return false0;

addr = gen8_canonical_addr(vma->node.start + offset);
if (gen >= 8) {
if (offset & 7) {
	*batch++ = MI_STORE_DWORD_IMM_GEN4(((0x20) << 23) | (2));
	*batch++ = lower_32_bits(addr)((u32)(addr));
	*batch++ = upper_32_bits(addr)((u32)(((addr) >> 16) >> 16));
	*batch++ = lower_32_bits(target_addr)((u32)(target_addr));

	addr = gen8_canonical_addr(addr + 4);

	*batch++ = MI_STORE_DWORD_IMM_GEN4(((0x20) << 23) | (2));
	*batch++ = lower_32_bits(addr)((u32)(addr));
	*batch++ = upper_32_bits(addr)((u32)(((addr) >> 16) >> 16));
	*batch++ = upper_32_bits(target_addr)((u32)(((target_addr) >> 16) >> 16));
} else {
	*batch++ = (MI_STORE_DWORD_IMM_GEN4(((0x20) << 23) | (2)) | (1 << 21)) + 1;
	*batch++ = lower_32_bits(addr)((u32)(addr));
	*batch++ = upper_32_bits(addr)((u32)(((addr) >> 16) >> 16));
	*batch++ = lower_32_bits(target_addr)((u32)(target_addr));
	*batch++ = upper_32_bits(target_addr)((u32)(((target_addr) >> 16) >> 16));
}
} else if (gen >= 6) {
*batch++ = MI_STORE_DWORD_IMM_GEN4(((0x20) << 23) | (2));
*batch++ = 0;
*batch++ = addr;
*batch++ = target_addr;
} else if (IS_I965G(eb->i915)IS_PLATFORM(eb->i915, INTEL_I965G)) {
*batch++ = MI_STORE_DWORD_IMM_GEN4(((0x20) << 23) | (2));
*batch++ = 0;
*batch++ = vma_phys_addr(vma, offset);
*batch++ = target_addr;
} else if (gen >= 4) {
*batch++ = MI_STORE_DWORD_IMM_GEN4(((0x20) << 23) | (2)) | MI_USE_GGTT(1 << 22);
*batch++ = 0;
*batch++ = addr;
*batch++ = target_addr;
} else if (gen >= 3 &&
   !(IS_I915G(eb->i915)IS_PLATFORM(eb->i915, INTEL_I915G) || IS_I915GM(eb->i915)IS_PLATFORM(eb->i915, INTEL_I915GM))) {
*batch++ = MI_STORE_DWORD_IMM(((0x20) << 23) | (1)) | MI_MEM_VIRTUAL(1 << 22);
*batch++ = addr;
*batch++ = target_addr;
} else {
*batch++ = MI_STORE_DWORD_IMM(((0x20) << 23) | (1));
*batch++ = vma_phys_addr(vma, offset);
*batch++ = target_addr;
}

return true1;
1544}

1546static int reloc_entry_gpu(struct i915_execbuffer *eb,
	    struct i915_vma *vma,
	    u64 offset,
	    u64 target_addr)
1550{
if (eb->reloc_cache.vaddr)
return false0;

if (!use_reloc_gpu(vma))
return false0;

return __reloc_entry_gpu(eb, vma, offset, target_addr);
1558}

1560static u64
1561relocate_entry(struct i915_vma *vma,
      const struct drm_i915_gem_relocation_entry *reloc,
      struct i915_execbuffer *eb,
      const struct i915_vma *target)
1565{
u64 target_addr = relocation_target(reloc, target);
u64 offset = reloc->offset;
int reloc_gpu = reloc_entry_gpu(eb, vma, offset, target_addr);

if (reloc_gpu < 0)
return reloc_gpu;

if (!reloc_gpu) {
bool_Bool wide = eb->reloc_cache.use_64bit_reloc;
void *vaddr;

1577repeat:
vaddr = reloc_vaddr(vma->obj, eb,
		    offset >> PAGE_SHIFT12);
if (IS_ERR(vaddr))
	return PTR_ERR(vaddr);

GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32)))((void)0);
clflush_write32(vaddr + offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1)),
		lower_32_bits(target_addr)((u32)(target_addr)),
		eb->reloc_cache.vaddr);

if (wide) {
	offset += sizeof(u32);
	target_addr >>= 32;
	wide = false0;
	goto repeat;
}
}

return target->node.start | UPDATE(1ULL << (7));
1597}

1599static u64
1600eb_relocate_entry(struct i915_execbuffer *eb,
  struct eb_vma *ev,
  const struct drm_i915_gem_relocation_entry *reloc)
1603{
struct drm_i915_privateinteldrm_softc *i915 = eb->i915;
struct eb_vma *target;
int err;

/* we've already hold a reference to all valid objects */
target = eb_get_vma(eb, reloc->target_handle);
if (unlikely(!target)__builtin_expect(!!(!target), 0))
return -ENOENT2;

/* Validate that the target is in a valid r/w GPU domain */
if (unlikely(reloc->write_domain & (reloc->write_domain - 1))__builtin_expect(!!(reloc->write_domain & (reloc->write_domain
 - 1)), 0)) {
drm_dbg(&i915->drm, "reloc with multiple write domains: "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  "target %d offset %d "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  "read %08x write %08x",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  reloc->target_handle,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  (int) reloc->offset,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  reloc->read_domains,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  reloc->write_domain)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with multiple write domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
);
return -EINVAL22;
}
if (unlikely((reloc->write_domain | reloc->read_domains)__builtin_expect(!!((reloc->write_domain | reloc->read_domains
) & ~(0x00000002 | 0x00000004 | 0x00000008 | 0x00000010 |
 0x00000020)), 0)
     & ~I915_GEM_GPU_DOMAINS)__builtin_expect(!!((reloc->write_domain | reloc->read_domains
) & ~(0x00000002 | 0x00000004 | 0x00000008 | 0x00000010 |
 0x00000020)), 0)) {
drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  "target %d offset %d "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  "read %08x write %08x",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  reloc->target_handle,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  (int) reloc->offset,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  reloc->read_domains,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
)
	  reloc->write_domain)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "reloc with read/write non-GPU domains: "
 "target %d offset %d " "read %08x write %08x", reloc->target_handle
, (int) reloc->offset, reloc->read_domains, reloc->write_domain
);
return -EINVAL22;
}

if (reloc->write_domain) {
target->flags |= EXEC_OBJECT_WRITE(1<<2);

/*
 * Sandybridge PPGTT errata: We need a global gtt mapping
 * for MI and pipe_control writes because the gpu doesn't
 * properly redirect them through the ppgtt for non_secure
 * batchbuffers.
 */
if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION0x00000010 &&
    IS_GEN(eb->i915, 6)(0 + (&(eb->i915)->__info)->gen == (6))) {
	err = i915_vma_bind(target->vma,
			    target->vma->obj->cache_level,
			    PIN_GLOBAL(1ULL << (10)), NULL((void *)0));
	if (err)
		return err;
}
}

/*
* If the relocation already has the right value in it, no
* more work needs to be done.
*/
if (!DBG_FORCE_RELOC0 &&
   gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset)
return 0;

/* Check that the relocation address is valid... */
if (unlikely(reloc->offset >__builtin_expect(!!(reloc->offset > ev->vma->size
 - (eb->reloc_cache.use_64bit_reloc ? 8 : 4)), 0)
     ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))__builtin_expect(!!(reloc->offset > ev->vma->size
 - (eb->reloc_cache.use_64bit_reloc ? 8 : 4)), 0)) {
drm_dbg(&i915->drm, "Relocation beyond object bounds: "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation beyond object bounds: "
 "target %d offset %d size %d.\n", reloc->target_handle, (
int)reloc->offset, (int)ev->vma->size)
	  "target %d offset %d size %d.\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation beyond object bounds: "
 "target %d offset %d size %d.\n", reloc->target_handle, (
int)reloc->offset, (int)ev->vma->size)
	  reloc->target_handle,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation beyond object bounds: "
 "target %d offset %d size %d.\n", reloc->target_handle, (
int)reloc->offset, (int)ev->vma->size)
	  (int)reloc->offset,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation beyond object bounds: "
 "target %d offset %d size %d.\n", reloc->target_handle, (
int)reloc->offset, (int)ev->vma->size)
	  (int)ev->vma->size)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation beyond object bounds: "
 "target %d offset %d size %d.\n", reloc->target_handle, (
int)reloc->offset, (int)ev->vma->size);
return -EINVAL22;
}
if (unlikely(reloc->offset & 3)__builtin_expect(!!(reloc->offset & 3), 0)) {
drm_dbg(&i915->drm, "Relocation not 4-byte aligned: "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation not 4-byte aligned: "
 "target %d offset %d.\n", reloc->target_handle, (int)reloc
->offset)
	  "target %d offset %d.\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation not 4-byte aligned: "
 "target %d offset %d.\n", reloc->target_handle, (int)reloc
->offset)
	  reloc->target_handle,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation not 4-byte aligned: "
 "target %d offset %d.\n", reloc->target_handle, (int)reloc
->offset)
	  (int)reloc->offset)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Relocation not 4-byte aligned: "
 "target %d offset %d.\n", reloc->target_handle, (int)reloc
->offset);
return -EINVAL22;
}

/*
* If we write into the object, we need to force the synchronisation
* barrier, either with an asynchronous clflush or if we executed the
* patching using the GPU (though that should be serialised by the
* timeline). To be completely sure, and since we are required to
* do relocations we are already stalling, disable the user's opt
* out of our synchronisation.
*/
ev->flags &= ~EXEC_OBJECT_ASYNC(1<<6);

/* and update the user's relocation entry */
return relocate_entry(ev->vma, reloc, eb, target->vma);
1693}

1695static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev)
1696{
1697#define N_RELOC(x)((x) / sizeof(struct drm_i915_gem_relocation_entry)) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
struct drm_i915_gem_relocation_entry stack[N_RELOC(512)((512) / sizeof(struct drm_i915_gem_relocation_entry))];
const struct drm_i915_gem_exec_object2 *entry = ev->exec;
struct drm_i915_gem_relocation_entry __user *urelocs =
u64_to_user_ptr(entry->relocs_ptr)((void *)(uintptr_t)(entry->relocs_ptr));
unsigned long remain = entry->relocation_count;

if (unlikely(remain > N_RELOC(ULONG_MAX))__builtin_expect(!!(remain > ((0xffffffffffffffffUL) / sizeof
(struct drm_i915_gem_relocation_entry))), 0))
return -EINVAL22;

/*
* We must check that the entire relocation array is safe
* to read. However, if the array is not writable the user loses
* the updated relocation values.
*/
if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs)))__builtin_expect(!!(!access_ok(urelocs, remain * sizeof(*urelocs
))), 0))
return -EFAULT14;

do {
struct drm_i915_gem_relocation_entry *r = stack;
unsigned int count =
	min_t(unsigned long, remain, ARRAY_SIZE(stack))({ unsigned long __min_a = (remain); unsigned long __min_b = (
(sizeof((stack)) / sizeof((stack)[0]))); __min_a < __min_b
 ? __min_a : __min_b; });
unsigned int copied;

/*
 * This is the fast path and we cannot handle a pagefault
 * whilst holding the struct mutex lest the user pass in the
 * relocations contained within a mmaped bo. For in such a case
 * we, the page fault handler would call i915_gem_fault() and
 * we would try to acquire the struct mutex again. Obviously
 * this is bad and so lockdep complains vehemently.
 */
pagefault_disable();
copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
pagefault_enable();
if (unlikely(copied)__builtin_expect(!!(copied), 0)) {
	remain = -EFAULT14;
	goto out;
}

remain -= count;
do {
	u64 offset = eb_relocate_entry(eb, ev, r);

	if (likely(offset == 0)__builtin_expect(!!(offset == 0), 1)) {
	} else if ((s64)offset < 0) {
		remain = (int)offset;
		goto out;
	} else {
		/*
		 * Note that reporting an error now
		 * leaves everything in an inconsistent
		 * state as we have *already* changed
		 * the relocation value inside the
		 * object. As we have not changed the
		 * reloc.presumed_offset or will not
		 * change the execobject.offset, on the
		 * call we may not rewrite the value
		 * inside the object, leaving it
		 * dangling and causing a GPU hang. Unless
		 * userspace dynamically rebuilds the
		 * relocations on each execbuf rather than
		 * presume a static tree.
		 *
		 * We did previously check if the relocations
		 * were writable (access_ok), an error now
		 * would be a strange race with mprotect,
		 * having already demonstrated that we
		 * can read from this userspace address.
		 */
		offset = gen8_canonical_addr(offset & ~UPDATE(1ULL << (7)));
		__put_user(offset,({ __typeof(((offset))) __tmp = ((offset)); -copyout(&(__tmp
), (&urelocs[r - stack].presumed_offset), sizeof(__tmp));
 })
			   &urelocs[r - stack].presumed_offset)({ __typeof(((offset))) __tmp = ((offset)); -copyout(&(__tmp
), (&urelocs[r - stack].presumed_offset), sizeof(__tmp));
 });
	}
} while (r++, --count);
urelocs += ARRAY_SIZE(stack)(sizeof((stack)) / sizeof((stack)[0]));
} while (remain);
1774out:
reloc_cache_reset(&eb->reloc_cache, eb);
return remain;
1777}

1779static int
1780eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev)
1781{
const struct drm_i915_gem_exec_object2 *entry = ev->exec;
struct drm_i915_gem_relocation_entry *relocs =
u64_to_ptr(typeof(*relocs), entry->relocs_ptr)({ 1; (typeof(*relocs) *)(uintptr_t)(entry->relocs_ptr); }
);
unsigned int i;
int err;

for (i = 0; i < entry->relocation_count; i++) {
u64 offset = eb_relocate_entry(eb, ev, &relocs[i]);

if ((s64)offset < 0) {
	err = (int)offset;
	goto err;
}
}
err = 0;
1797err:
reloc_cache_reset(&eb->reloc_cache, eb);
return err;
1800}

1802static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1803{
const char __user *addr, *end;
unsigned long size;
char __maybe_unused__attribute__((__unused__)) c;

size = entry->relocation_count;
if (size == 0)
return 0;

if (size > N_RELOC(ULONG_MAX)((0xffffffffffffffffUL) / sizeof(struct drm_i915_gem_relocation_entry
)))
return -EINVAL22;

addr = u64_to_user_ptr(entry->relocs_ptr)((void *)(uintptr_t)(entry->relocs_ptr));
size *= sizeof(struct drm_i915_gem_relocation_entry);
if (!access_ok(addr, size))
return -EFAULT14;

end = addr + size;
for (; addr < end; addr += PAGE_SIZE(1 << 12)) {
int err = __get_user(c, addr)-copyin((addr), &((c)), sizeof((c)));
if (err)
	return err;
}
return __get_user(c, end - 1)-copyin((end - 1), &((c)), sizeof((c)));
1827}

1829static int eb_copy_relocations(const struct i915_execbuffer *eb)
1830{
struct drm_i915_gem_relocation_entry *relocs;
const unsigned int count = eb->buffer_count;
unsigned int i;
int err;

for (i = 0; i < count; i++) {
const unsigned int nreloc = eb->exec[i].relocation_count;
struct drm_i915_gem_relocation_entry __user *urelocs;
unsigned long size;
unsigned long copied;

if (nreloc == 0)
	continue;

err = check_relocations(&eb->exec[i]);
if (err)
	goto err;

urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr)((void *)(uintptr_t)(eb->exec[i].relocs_ptr));
size = nreloc * sizeof(*relocs);

relocs = kvmalloc_array(size, 1, GFP_KERNEL(0x0001 | 0x0004));
if (!relocs) {
	err = -ENOMEM12;
	goto err;
}

/* copy_from_user is limited to < 4GiB */
copied = 0;
do {
	unsigned int len =
		min_t(u64, BIT_ULL(31), size - copied)({ u64 __min_a = ((1ULL << (31))); u64 __min_b = (size -
 copied); __min_a < __min_b ? __min_a : __min_b; });

	if (__copy_from_user((char *)relocs + copied,
			     (char __user *)urelocs + copied,
			     len))
		goto end;

	copied += len;
} while (copied < size);

/*
 * As we do not update the known relocation offsets after
 * relocating (due to the complexities in lock handling),
 * we need to mark them as invalid now so that we force the
 * relocation processing next time. Just in case the target
 * object is evicted and then rebound into its old
 * presumed_offset before the next execbuffer - if that
 * happened we would make the mistake of assuming that the
 * relocations were valid.
 */
if (!user_access_begin(urelocs, size)access_ok(urelocs, size))
	goto end;

for (copied = 0; copied < nreloc; copied++)
	unsafe_put_user(-1,({ __typeof((-1)) __tmp = (-1); if (copyout(&(__tmp), &
urelocs[copied].presumed_offset, sizeof(__tmp)) != 0) goto end_user
; })
			&urelocs[copied].presumed_offset,({ __typeof((-1)) __tmp = (-1); if (copyout(&(__tmp), &
urelocs[copied].presumed_offset, sizeof(__tmp)) != 0) goto end_user
; })
			end_user)({ __typeof((-1)) __tmp = (-1); if (copyout(&(__tmp), &
urelocs[copied].presumed_offset, sizeof(__tmp)) != 0) goto end_user
; });
user_access_end();

eb->exec[i].relocs_ptr = (uintptr_t)relocs;
}

return 0;

1896end_user:
user_access_end();
1898end:
kvfree(relocs);
err = -EFAULT14;
1901err:
while (i--) {
relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr)({ 1; (typeof(*relocs) *)(uintptr_t)(eb->exec[i].relocs_ptr
); });
if (eb->exec[i].relocation_count)
	kvfree(relocs);
}
return err;
1908}

1910static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1911{
const unsigned int count = eb->buffer_count;
unsigned int i;

for (i = 0; i < count; i++) {
int err;

err = check_relocations(&eb->exec[i]);
if (err)
	return err;
}

return 0;
1924}

1926static noinline__attribute__((__noinline__)) int eb_relocate_parse_slow(struct i915_execbuffer *eb,
			   struct i915_request *rq)
1928{
bool_Bool have_copy = false0;
struct eb_vma *ev;
int err = 0;

1933repeat:
if (signal_pending(current)(((({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" :
 "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self
))); __ci;})->ci_curproc)->p_siglist | (({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})->ci_curproc)->p_p
->ps_siglist) & ~(({struct cpu_info *__ci; asm volatile
("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct
 cpu_info, ci_self))); __ci;})->ci_curproc)->p_sigmask)) {
err = -ERESTARTSYS4;
goto out;
}

/* We may process another execbuffer during the unlock... */
eb_release_vmas(eb, false0);
i915_gem_ww_ctx_fini(&eb->ww);

if (rq) {
/* nonblocking is always false */
if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE(1UL << (0)),
		      MAX_SCHEDULE_TIMEOUT(0x7fffffff)) < 0) {
	i915_request_put(rq);
	rq = NULL((void *)0);

	err = -EINTR4;
	goto err_relock;
}

i915_request_put(rq);
rq = NULL((void *)0);
}

/*
* We take 3 passes through the slowpatch.
*
* 1 - we try to just prefault all the user relocation entries and
* then attempt to reuse the atomic pagefault disabled fast path again.
*
* 2 - we copy the user entries to a local buffer here outside of the
* local and allow ourselves to wait upon any rendering before
* relocations
*
* 3 - we already have a local copy of the relocation entries, but
* were interrupted (EAGAIN) whilst waiting for the objects, try again.
*/
if (!err) {
err = eb_prefault_relocations(eb);
} else if (!have_copy) {
err = eb_copy_relocations(eb);
have_copy = err == 0;
} else {
cond_resched()do { if (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0"
 : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self
))); __ci;})->ci_schedstate.spc_schedflags & 0x0002) yield
(); } while (0);
err = 0;
}

if (!err)
flush_workqueue(eb->i915->mm.userptr_wq);

1984err_relock:
i915_gem_ww_ctx_init(&eb->ww, true1);
if (err)
goto out;

/* reacquire the objects */
1990repeat_validate:
rq = eb_pin_engine(eb, false0);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
rq = NULL((void *)0);
goto err;
}

/* We didn't throttle, should be NULL */
GEM_WARN_ON(rq)({ __builtin_expect(!!(!!(rq)), 0); });

err = eb_validate_vmas(eb);
if (err)
goto err;

GEM_BUG_ON(!eb->batch)((void)0);

list_for_each_entry(ev, &eb->relocs, reloc_link)for (ev = ({ const __typeof( ((__typeof(*ev) *)0)->reloc_link
 ) *__mptr = ((&eb->relocs)->next); (__typeof(*ev) *
)( (char *)__mptr - __builtin_offsetof(__typeof(*ev), reloc_link
) );}); &ev->reloc_link != (&eb->relocs); ev = (
{ const __typeof( ((__typeof(*ev) *)0)->reloc_link ) *__mptr
 = (ev->reloc_link.next); (__typeof(*ev) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*ev), reloc_link) );})) {
if (!have_copy) {
	pagefault_disable();
	err = eb_relocate_vma(eb, ev);
	pagefault_enable();
	if (err)
		break;
} else {
	err = eb_relocate_vma_slow(eb, ev);
	if (err)
		break;
}
}

if (err == -EDEADLK11)
goto err;

if (err && !have_copy)
goto repeat;

if (err)
goto err;

/* as last step, parse the command buffer */
err = eb_parse(eb);
if (err)
goto err;

/*
* Leave the user relocations as are, this is the painfully slow path,
* and we want to avoid the complication of dropping the lock whilst
* having buffers reserved in the aperture and so causing spurious
* ENOSPC for random operations.
*/

2042err:
if (err == -EDEADLK11) {
eb_release_vmas(eb, false0);
err = i915_gem_ww_ctx_backoff(&eb->ww);
if (!err)
	goto repeat_validate;
}

if (err == -EAGAIN35)
goto repeat;

2053out:
if (have_copy) {
const unsigned int count = eb->buffer_count;
unsigned int i;

for (i = 0; i < count; i++) {
	const struct drm_i915_gem_exec_object2 *entry =
		&eb->exec[i];
	struct drm_i915_gem_relocation_entry *relocs;

	if (!entry->relocation_count)
		continue;

	relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr)({ 1; (typeof(*relocs) *)(uintptr_t)(entry->relocs_ptr); }
);
	kvfree(relocs);
}
}

if (rq)
i915_request_put(rq);

return err;
2075}

2077static int eb_relocate_parse(struct i915_execbuffer *eb)
2078{
int err;
struct i915_request *rq = NULL((void *)0);
bool_Bool throttle = true1;

2083retry:
rq = eb_pin_engine(eb, throttle);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
rq = NULL((void *)0);
if (err != -EDEADLK11)
	return err;

goto err;
}

if (rq) {
2095#ifdef __linux__
bool_Bool nonblock = eb->file->filp->f_flags & O_NONBLOCK0x0004;
2097#else
bool_Bool nonblock = eb->file->filp->f_flag & FNONBLOCK0x0004;
2099#endif

/* Need to drop all locks now for throttling, take slowpath */
err = i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE(1UL << (0)), 0);
if (err == -ETIME60) {
	if (nonblock) {
		err = -EWOULDBLOCK35;
		i915_request_put(rq);
		goto err;
	}
	goto slow;
}
i915_request_put(rq);
rq = NULL((void *)0);
}

/* only throttle once, even if we didn't need to throttle */
throttle = false0;

err = eb_validate_vmas(eb);
if (err == -EAGAIN35)
goto slow;
else if (err)
goto err;

/* The objects are in their final locations, apply the relocations. */
if (eb->args->flags & __EXEC_HAS_RELOC(1UL << (31))) {
struct eb_vma *ev;

list_for_each_entry(ev, &eb->relocs, reloc_link)for (ev = ({ const __typeof( ((__typeof(*ev) *)0)->reloc_link
 ) *__mptr = ((&eb->relocs)->next); (__typeof(*ev) *
)( (char *)__mptr - __builtin_offsetof(__typeof(*ev), reloc_link
) );}); &ev->reloc_link != (&eb->relocs); ev = (
{ const __typeof( ((__typeof(*ev) *)0)->reloc_link ) *__mptr
 = (ev->reloc_link.next); (__typeof(*ev) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*ev), reloc_link) );})) {
	err = eb_relocate_vma(eb, ev);
	if (err)
		break;
}

if (err == -EDEADLK11)
	goto err;
else if (err)
	goto slow;
}

if (!err)
err = eb_parse(eb);

2143err:
if (err == -EDEADLK11) {
eb_release_vmas(eb, false0);
err = i915_gem_ww_ctx_backoff(&eb->ww);
if (!err)
	goto retry;
}

return err;

2153slow:
err = eb_relocate_parse_slow(eb, rq);
if (err)
/*
 * If the user expects the execobject.offset and
 * reloc.presumed_offset to be an exact match,
 * as for using NO_RELOC, then we cannot update
 * the execobject.offset until we have completed
 * relocation.
 */
eb->args->flags &= ~__EXEC_HAS_RELOC(1UL << (31));

return err;
2166}

2168static int eb_move_to_gpu(struct i915_execbuffer *eb)
2169{
const unsigned int count = eb->buffer_count;
unsigned int i = count;
int err = 0;

while (i--) {
struct eb_vma *ev = &eb->vma[i];
struct i915_vma *vma = ev->vma;
unsigned int flags = ev->flags;
struct drm_i915_gem_object *obj = vma->obj;

assert_vma_held(vma)do { (void)(&((vma)->resv)->lock.base); } while(0);

if (flags & EXEC_OBJECT_CAPTURE(1<<7)) {
	struct i915_capture_list *capture;

	capture = kmalloc(sizeof(*capture), GFP_KERNEL(0x0001 | 0x0004));
	if (capture) {
		capture->next = eb->request->capture_list;
		capture->vma = vma;
		eb->request->capture_list = capture;
	}
}

/*
 * If the GPU is not _reading_ through the CPU cache, we need
 * to make sure that any writes (both previous GPU writes from
 * before a change in snooping levels and normal CPU writes)
 * caught in that cache are flushed to main memory.
 *
 * We want to say
 *   obj->cache_dirty &&
 *   !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
 * but gcc's optimiser doesn't handle that as well and emits
 * two jumps instead of one. Maybe one day...
 */
if (unlikely(obj->cache_dirty & ~obj->cache_coherent)__builtin_expect(!!(obj->cache_dirty & ~obj->cache_coherent
), 0)) {
	if (i915_gem_clflush_object(obj, 0))
		flags &= ~EXEC_OBJECT_ASYNC(1<<6);
}

if (err == 0 && !(flags & EXEC_OBJECT_ASYNC(1<<6))) {
	err = i915_request_await_object
		(eb->request, obj, flags & EXEC_OBJECT_WRITE(1<<2));
}

if (err == 0)
	err = i915_vma_move_to_active(vma, eb->request, flags);
}

if (unlikely(err)__builtin_expect(!!(err), 0))
goto err_skip;

/* Unconditionally flush any chipset caches (for streaming writes). */
intel_gt_chipset_flush(eb->engine->gt);
return 0;

2226err_skip:
i915_request_set_error_once(eb->request, err);
return err;
2229}

2231static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
2232{
if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS((-((1 << 21) << 1)) | (3<<6) | (1<<15
)))
return -EINVAL22;

/* Kernel clipping was a DRI1 misfeature */
if (!(exec->flags & (I915_EXEC_FENCE_ARRAY(1<<19) |
	     I915_EXEC_USE_EXTENSIONS(1 << 21)))) {
if (exec->num_cliprects || exec->cliprects_ptr)
	return -EINVAL22;
}

if (exec->DR4 == 0xffffffff) {
DRM_DEBUG("UXA submitting garbage DR4, fixing up\n")__drm_dbg(DRM_UT_CORE, "UXA submitting garbage DR4, fixing up\n"
);
exec->DR4 = 0;
}
if (exec->DR1 || exec->DR4)
return -EINVAL22;

if ((exec->batch_start_offset | exec->batch_len) & 0x7)
return -EINVAL22;

return 0;
2254}

2256static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
2257{
u32 *cs;
int i;

if (!IS_GEN(rq->engine->i915, 7)(0 + (&(rq->engine->i915)->__info)->gen == (7
)) || rq->engine->id != RCS0) {
drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n")drm_dev_dbg((&rq->engine->i915->drm)->dev, DRM_UT_DRIVER
, "sol reset is gen7/rcs only\n");
return -EINVAL22;
}

cs = intel_ring_begin(rq, 4 * 2 + 2);
if (IS_ERR(cs))
return PTR_ERR(cs);

*cs++ = MI_LOAD_REGISTER_IMM(4)(((0x22) << 23) | (2*(4)-1));
for (i = 0; i < 4; i++) {
*cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i)((const i915_reg_t){ .reg = (0x5280 + (i) * 4) }));
*cs++ = 0;
}
*cs++ = MI_NOOP(((0) << 23) | (0));
intel_ring_advance(rq, cs);

return 0;
2279}

2281static struct i915_vma *
2282shadow_batch_pin(struct i915_execbuffer *eb,
 struct drm_i915_gem_object *obj,
 struct i915_address_space *vm,
 unsigned int flags)
2286{
struct i915_vma *vma;
int err;

vma = i915_vma_instance(obj, vm, NULL((void *)0));
if (IS_ERR(vma))
return vma;

err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags);
if (err)
return ERR_PTR(err);

return vma;
2299}

2301static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma)
2302{
/*
* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
* batch" bit. Hence we need to pin secure batches into the global gtt.
* hsw should have this fixed, but bdw mucks it up again. */
if (eb->batch_flags & I915_DISPATCH_SECURE(1UL << (0)))
return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL((void *)0), 0, 0, 0);

return NULL((void *)0);
2311}

2313static int eb_parse(struct i915_execbuffer *eb)
2314{
struct drm_i915_privateinteldrm_softc *i915 = eb->i915;
struct intel_gt_buffer_pool_node *pool = eb->batch_pool;
struct i915_vma *shadow, *trampoline, *batch;
unsigned long len;
int err;

if (!eb_use_cmdparser(eb)) {
batch = eb_dispatch_secure(eb, eb->batch->vma);
if (IS_ERR(batch))
	return PTR_ERR(batch);

goto secure_batch;
}

len = eb->batch_len;
if (!CMDPARSER_USES_GGTT(eb->i915)(0 + (&(eb->i915)->__info)->gen == (7))) {
/*
 * ppGTT backed shadow buffers must be mapped RO, to prevent
 * post-scan tampering
 */
if (!eb->context->vm->has_read_only) {
	drm_dbg(&i915->drm,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Cannot prevent post-scan tampering without RO capable vm\n"
)
		"Cannot prevent post-scan tampering without RO capable vm\n")drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Cannot prevent post-scan tampering without RO capable vm\n"
);
	return -EINVAL22;
}
} else {
len += I915_CMD_PARSER_TRAMPOLINE_SIZE8;
}
if (unlikely(len < eb->batch_len)__builtin_expect(!!(len < eb->batch_len), 0)) /* last paranoid check of overflow */
return -EINVAL22;

if (!pool) {
pool = intel_gt_get_buffer_pool(eb->engine->gt, len);
if (IS_ERR(pool))
	return PTR_ERR(pool);
eb->batch_pool = pool;
}

err = i915_gem_object_lock(pool->obj, &eb->ww);
if (err)
goto err;

shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER(1ULL << (11)));
if (IS_ERR(shadow)) {
err = PTR_ERR(shadow);
goto err;
}
i915_gem_object_set_readonly(shadow->obj);
shadow->private = pool;

trampoline = NULL((void *)0);
if (CMDPARSER_USES_GGTT(eb->i915)(0 + (&(eb->i915)->__info)->gen == (7))) {
trampoline = shadow;

shadow = shadow_batch_pin(eb, pool->obj,
			  &eb->engine->gt->ggtt->vm,
			  PIN_GLOBAL(1ULL << (10)));
if (IS_ERR(shadow)) {
	err = PTR_ERR(shadow);
	shadow = trampoline;
	goto err_shadow;
}
shadow->private = pool;

eb->batch_flags |= I915_DISPATCH_SECURE(1UL << (0));
}

batch = eb_dispatch_secure(eb, shadow);
if (IS_ERR(batch)) {
err = PTR_ERR(batch);
goto err_trampoline;
}

err = intel_engine_cmd_parser(eb->engine,
		      eb->batch->vma,
		      eb->batch_start_offset,
		      eb->batch_len,
		      shadow, trampoline);
if (err)
goto err_unpin_batch;

eb->batch = &eb->vma[eb->buffer_count++];
eb->batch->vma = i915_vma_get(shadow);
eb->batch->flags = __EXEC_OBJECT_HAS_PIN(1UL << (31));

eb->trampoline = trampoline;
eb->batch_start_offset = 0;

2403secure_batch:
if (batch) {
eb->batch = &eb->vma[eb->buffer_count++];
eb->batch->flags = __EXEC_OBJECT_HAS_PIN(1UL << (31));
eb->batch->vma = i915_vma_get(batch);
}
return 0;

2411err_unpin_batch:
if (batch)
i915_vma_unpin(batch);
2414err_trampoline:
if (trampoline)
i915_vma_unpin(trampoline);
2417err_shadow:
i915_vma_unpin(shadow);
2419err:
return err;
2421}

2423static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch)
2424{
int err;

err = eb_move_to_gpu(eb);
if (err)
return err;

if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET(1<<8)) {
err = i915_reset_gen7_sol_offsets(eb->request);
if (err)
	return err;
}

/*
* After we completed waiting for other engines (using HW semaphores)
* then we can signal that this request/batch is ready to run. This
* allows us to determine if the batch is still waiting on the GPU
* or actually running by checking the breadcrumb.
*/
if (eb->engine->emit_init_breadcrumb) {
err = eb->engine->emit_init_breadcrumb(eb->request);
if (err)
	return err;
}

err = eb->engine->emit_bb_start(eb->request,
			batch->node.start +
			eb->batch_start_offset,
			eb->batch_len,
			eb->batch_flags);
if (err)
return err;

if (eb->trampoline) {
GEM_BUG_ON(eb->batch_start_offset)((void)0);
err = eb->engine->emit_bb_start(eb->request,
				eb->trampoline->node.start +
				eb->batch_len,
				0, 0);
if (err)
	return err;
}

if (intel_context_nopreempt(eb->context))
__set_bit(I915_FENCE_FLAG_NOPREEMPT, &eb->request->fence.flags);

return 0;
2471}

2473static int num_vcs_engines(const struct drm_i915_privateinteldrm_softc *i915)
2474{
return hweight64(VDBOX_MASK(&i915->gt)({ unsigned int first__ = (VCS0); unsigned int count__ = (4);
 ((&i915->gt)->info.engine_mask & (((~0UL) >>
 (64 - (first__ + count__ - 1) - 1)) & ((~0UL) << (
first__)))) >> first__; }));
2476}

2478/*
* Find one BSD ring to dispatch the corresponding BSD command.
* The engine index is returned.
*/
2482static unsigned int
2483gen8_dispatch_bsd_engine(struct drm_i915_privateinteldrm_softc *dev_priv,
	 struct drm_file *file)
2485{
struct drm_i915_file_private *file_priv = file->driver_priv;

/* Check whether the file_priv has already selected one ring. */
if ((int)file_priv->bsd_engine < 0)
file_priv->bsd_engine =
	get_random_int()arc4random() % num_vcs_engines(dev_priv);

return file_priv->bsd_engine;
2494}

2496static const enum intel_engine_id user_ring_map[] = {
[I915_EXEC_DEFAULT(0<<0)]	= RCS0,
[I915_EXEC_RENDER(1<<0)]	= RCS0,
[I915_EXEC_BLT(3<<0)]		= BCS0,
[I915_EXEC_BSD(2<<0)]		= VCS0,
[I915_EXEC_VEBOX(4<<0)]	= VECS0
2502};

2504static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce)
2505{
struct intel_ring *ring = ce->ring;
struct intel_timeline *tl = ce->timeline;
struct i915_request *rq;

/*
* Completely unscientific finger-in-the-air estimates for suitable
* maximum user request size (to avoid blocking) and then backoff.
*/
if (intel_ring_update_space(ring) >= PAGE_SIZE(1 << 12))
return NULL((void *)0);

/*
* Find a request that after waiting upon, there will be at least half
* the ring available. The hysteresis allows us to compete for the
* shared ring and should mean that we sleep less often prior to
* claiming our resources, but not so long that the ring completely
* drains before we can submit our next request.
*/
list_for_each_entry(rq, &tl->requests, link)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->link ) *
__mptr = ((&tl->requests)->next); (__typeof(*rq) *)
( (char *)__mptr - __builtin_offsetof(__typeof(*rq), link) );
}); &rq->link != (&tl->requests); rq = ({ const
 __typeof( ((__typeof(*rq) *)0)->link ) *__mptr = (rq->
link.next); (__typeof(*rq) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*rq), link) );})) {
if (rq->ring != ring)
	continue;

if (__intel_ring_space(rq->postfix,
		       ring->emit, ring->size) > ring->size / 2)
	break;
}
if (&rq->link == &tl->requests)
return NULL((void *)0); /* weird, we will check again later for real */

return i915_request_get(rq);
2536}

2538static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, bool_Bool throttle)
2539{
struct intel_context *ce = eb->context;
struct intel_timeline *tl;
struct i915_request *rq = NULL((void *)0);
int err;

GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED)((void)0);

if (unlikely(intel_context_is_banned(ce))__builtin_expect(!!(intel_context_is_banned(ce)), 0))
return ERR_PTR(-EIO5);

/*
* Pinning the contexts may generate requests in order to acquire
* GGTT space, so do this first before we reserve a seqno for
* ourselves.
*/
err = intel_context_pin_ww(ce, &eb->ww);
if (err)
return ERR_PTR(err);

/*
* Take a local wakeref for preparing to dispatch the execbuf as
* we expect to access the hardware fairly frequently in the
* process, and require the engine to be kept awake between accesses.
* Upon dispatch, we acquire another prolonged wakeref that we hold
* until the timeline is idle, which in turn releases the wakeref
* taken on the engine, and the parent device.
*/
tl = intel_context_timeline_lock(ce);
if (IS_ERR(tl)) {
intel_context_unpin(ce);
return ERR_CAST(tl);
}

intel_context_enter(ce);
if (throttle)
rq = eb_throttle(eb, ce);
intel_context_timeline_unlock(tl);

eb->args->flags |= __EXEC_ENGINE_PINNED(1UL << (30));
return rq;
2580}

2582static void eb_unpin_engine(struct i915_execbuffer *eb)
2583{
struct intel_context *ce = eb->context;
struct intel_timeline *tl = ce->timeline;

if (!(eb->args->flags & __EXEC_ENGINE_PINNED(1UL << (30))))
return;

eb->args->flags &= ~__EXEC_ENGINE_PINNED(1UL << (30));

mutex_lock(&tl->mutex)rw_enter_write(&tl->mutex);
intel_context_exit(ce);
mutex_unlock(&tl->mutex)rw_exit_write(&tl->mutex);

intel_context_unpin(ce);
2597}

2599static unsigned int
2600eb_select_legacy_ring(struct i915_execbuffer *eb)
2601{
struct drm_i915_privateinteldrm_softc *i915 = eb->i915;
struct drm_i915_gem_execbuffer2 *args = eb->args;
unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK(0x3f);

if (user_ring_id != I915_EXEC_BSD(2<<0) &&
   (args->flags & I915_EXEC_BSD_MASK(3 << (13)))) {
drm_dbg(&i915->drm,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with non bsd ring but with invalid "
 "bsd dispatch flags: %d\n", (int)(args->flags))
	"execbuf with non bsd ring but with invalid "drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with non bsd ring but with invalid "
 "bsd dispatch flags: %d\n", (int)(args->flags))
	"bsd dispatch flags: %d\n", (int)(args->flags))drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with non bsd ring but with invalid "
 "bsd dispatch flags: %d\n", (int)(args->flags));
return -1;
}

if (user_ring_id == I915_EXEC_BSD(2<<0) && num_vcs_engines(i915) > 1) {
unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK(3 << (13));

if (bsd_idx == I915_EXEC_BSD_DEFAULT(0 << (13))) {
	bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file);
} else if (bsd_idx >= I915_EXEC_BSD_RING1(1 << (13)) &&
	   bsd_idx <= I915_EXEC_BSD_RING2(2 << (13))) {
	bsd_idx >>= I915_EXEC_BSD_SHIFT(13);
	bsd_idx--;
} else {
	drm_dbg(&i915->drm,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with unknown bsd ring: %u\n"
, bsd_idx)
		"execbuf with unknown bsd ring: %u\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with unknown bsd ring: %u\n"
, bsd_idx)
		bsd_idx)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with unknown bsd ring: %u\n"
, bsd_idx);
	return -1;
}

return _VCS(bsd_idx)(VCS0 + (bsd_idx));
}

if (user_ring_id >= ARRAY_SIZE(user_ring_map)(sizeof((user_ring_map)) / sizeof((user_ring_map)[0]))) {
drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with unknown ring: %u\n"
, user_ring_id)
	user_ring_id)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf with unknown ring: %u\n"
, user_ring_id);
return -1;
}

return user_ring_map[user_ring_id];
2640}

2642static int
2643eb_select_engine(struct i915_execbuffer *eb)
2644{
struct intel_context *ce;
unsigned int idx;
int err;

if (i915_gem_context_user_engines(eb->gem_context))
idx = eb->args->flags & I915_EXEC_RING_MASK(0x3f);
else
idx = eb_select_legacy_ring(eb);

ce = i915_gem_context_get_engine(eb->gem_context, idx);
if (IS_ERR(ce))
return PTR_ERR(ce);

intel_gt_pm_get(ce->engine->gt);

if (!test_bit(CONTEXT_ALLOC_BIT1, &ce->flags)) {
err = intel_context_alloc_state(ce);
if (err)
	goto err;
}

/*
* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
* EIO if the GPU is already wedged.
*/
err = intel_gt_terminally_wedged(ce->engine->gt);
if (err)
goto err;

eb->context = ce;
eb->engine = ce->engine;

/*
* Make sure engine pool stays alive even if we call intel_context_put
* during ww handling. The pool is destroyed when last pm reference
* is dropped, which breaks our -EDEADLK handling.
*/
return err;

2684err:
intel_gt_pm_put(ce->engine->gt);
intel_context_put(ce);
return err;
2688}

2690static void
2691eb_put_engine(struct i915_execbuffer *eb)
2692{
intel_gt_pm_put(eb->engine->gt);
intel_context_put(eb->context);
2695}

2697static void
2698__free_fence_array(struct eb_fence *fences, unsigned int n)
2699{
while (n--) {
drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2)({ unsigned long __v = (unsigned long)(fences[n].syncobj); (typeof
(fences[n].syncobj))(__v & -(1UL << (2))); }));
dma_fence_put(fences[n].dma_fence);
kfree(fences[n].chain_fence);
}
kvfree(fences);
2706}

2708static int
2709add_timeline_fence_array(struct i915_execbuffer *eb,
	 const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences)
2711{
struct drm_i915_gem_exec_fence __user *user_fences;
u64 __user *user_values;
struct eb_fence *f;
u64 nfences;
int err = 0;

nfences = timeline_fences->fence_count;
if (!nfences)
return 0;

/* Check multiplication overflow for access_ok() and kvmalloc_array() */
BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long))extern char _ctassert[(!(sizeof(size_t) > sizeof(unsigned long
))) ? 1 : -1 ] __attribute__((__unused__));
if (nfences > min_t(unsigned long,({ unsigned long __min_a = (0xffffffffffffffffUL / sizeof(*user_fences
)); unsigned long __min_b = (0xffffffffffffffffUL / sizeof(*f
)); __min_a < __min_b ? __min_a : __min_b; })
	    ULONG_MAX / sizeof(*user_fences),({ unsigned long __min_a = (0xffffffffffffffffUL / sizeof(*user_fences
)); unsigned long __min_b = (0xffffffffffffffffUL / sizeof(*f
)); __min_a < __min_b ? __min_a : __min_b; })
	    SIZE_MAX / sizeof(*f))({ unsigned long __min_a = (0xffffffffffffffffUL / sizeof(*user_fences
)); unsigned long __min_b = (0xffffffffffffffffUL / sizeof(*f
)); __min_a < __min_b ? __min_a : __min_b; }) - eb->num_fences)
return -EINVAL22;

user_fences = u64_to_user_ptr(timeline_fences->handles_ptr)((void *)(uintptr_t)(timeline_fences->handles_ptr));
if (!access_ok(user_fences, nfences * sizeof(*user_fences)))
return -EFAULT14;

user_values = u64_to_user_ptr(timeline_fences->values_ptr)((void *)(uintptr_t)(timeline_fences->values_ptr));
if (!access_ok(user_values, nfences * sizeof(*user_values)))
return -EFAULT14;

2737#ifdef __linux__
f = krealloc(eb->fences,
     (eb->num_fences + nfences) * sizeof(*f),
     __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));
if (!f)
return -ENOMEM12;
2743#else
f = kmalloc((eb->num_fences + nfences) * sizeof(*f),
     __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));
if (!f)
return -ENOMEM12;
memcpy(f, eb->fences, eb->num_fences * sizeof(*f))__builtin_memcpy((f), (eb->fences), (eb->num_fences * sizeof
(*f)));
kfree(eb->fences);
2750#endif

eb->fences = f;
f += eb->num_fences;

2755#ifdef notyet
BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &extern char _ctassert[(!(~(ARCH_KMALLOC_MINALIGN - 1) & ~
(-((1<<1) << 1)))) ? 1 : -1 ] __attribute__((__unused__
))
     ~__I915_EXEC_FENCE_UNKNOWN_FLAGS)extern char _ctassert[(!(~(ARCH_KMALLOC_MINALIGN - 1) & ~
(-((1<<1) << 1)))) ? 1 : -1 ] __attribute__((__unused__
));
2758#endif

while (nfences--) {
struct drm_i915_gem_exec_fence user_fence;
struct drm_syncobj *syncobj;
struct dma_fence *fence = NULL((void *)0);
u64 point;

if (__copy_from_user(&user_fence,
		     user_fences++,
		     sizeof(user_fence)))
	return -EFAULT14;

if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS(-((1<<1) << 1)))
	return -EINVAL22;

if (__get_user(point, user_values++)-copyin((user_values++), &((point)), sizeof((point))))
	return -EFAULT14;

syncobj = drm_syncobj_find(eb->file, user_fence.handle);
if (!syncobj) {
	DRM_DEBUG("Invalid syncobj handle provided\n")__drm_dbg(DRM_UT_CORE, "Invalid syncobj handle provided\n");
	return -ENOENT2;
}

fence = drm_syncobj_fence_get(syncobj);

if (!fence && user_fence.flags &&
    !(user_fence.flags & I915_EXEC_FENCE_SIGNAL(1<<1))) {
	DRM_DEBUG("Syncobj handle has no fence\n")__drm_dbg(DRM_UT_CORE, "Syncobj handle has no fence\n");
	drm_syncobj_put(syncobj);
	return -EINVAL22;
}

if (fence)
	err = dma_fence_chain_find_seqno(&fence, point);

if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL(1<<1))) {
	DRM_DEBUG("Syncobj handle missing requested point %llu\n", point)__drm_dbg(DRM_UT_CORE, "Syncobj handle missing requested point %llu\n"
, point);
	dma_fence_put(fence);
	drm_syncobj_put(syncobj);
	return err;
}

/*
 * A point might have been signaled already and
 * garbage collected from the timeline. In this case
 * just ignore the point and carry on.
 */
if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL(1<<1))) {
	drm_syncobj_put(syncobj);
	continue;
}

/*
 * For timeline syncobjs we need to preallocate chains for
 * later signaling.
 */
if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL(1<<1)) {
	/*
	 * Waiting and signaling the same point (when point !=
	 * 0) would break the timeline.
	 */
	if (user_fence.flags & I915_EXEC_FENCE_WAIT(1<<0)) {
		DRM_DEBUG("Trying to wait & signal the same timeline point.\n")__drm_dbg(DRM_UT_CORE, "Trying to wait & signal the same timeline point.\n"
);
		dma_fence_put(fence);
		drm_syncobj_put(syncobj);
		return -EINVAL22;
	}

	f->chain_fence =
		kmalloc(sizeof(*f->chain_fence),
			GFP_KERNEL(0x0001 | 0x0004));
	if (!f->chain_fence) {
		drm_syncobj_put(syncobj);
		dma_fence_put(fence);
		return -ENOMEM12;
	}
} else {
	f->chain_fence = NULL((void *)0);
}

f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2)({ unsigned long __bits = (user_fence.flags); ((void)0); ((typeof
(syncobj))((unsigned long)(syncobj) | __bits)); });
f->dma_fence = fence;
f->value = point;
f++;
eb->num_fences++;
}

return 0;
2848}

2850static int add_fence_array(struct i915_execbuffer *eb)
2851{
struct drm_i915_gem_execbuffer2 *args = eb->args;
struct drm_i915_gem_exec_fence __user *user;
unsigned long num_fences = args->num_cliprects;
struct eb_fence *f;

if (!(args->flags & I915_EXEC_FENCE_ARRAY(1<<19)))
return 0;

if (!num_fences)
return 0;

/* Check multiplication overflow for access_ok() and kvmalloc_array() */
BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long))extern char _ctassert[(!(sizeof(size_t) > sizeof(unsigned long
))) ? 1 : -1 ] __attribute__((__unused__));
if (num_fences > min_t(unsigned long,({ unsigned long __min_a = (0xffffffffffffffffUL / sizeof(*user
)); unsigned long __min_b = (0xffffffffffffffffUL / sizeof(*f
) - eb->num_fences); __min_a < __min_b ? __min_a : __min_b
; })
	       ULONG_MAX / sizeof(*user),({ unsigned long __min_a = (0xffffffffffffffffUL / sizeof(*user
)); unsigned long __min_b = (0xffffffffffffffffUL / sizeof(*f
) - eb->num_fences); __min_a < __min_b ? __min_a : __min_b
; })
	       SIZE_MAX / sizeof(*f) - eb->num_fences)({ unsigned long __min_a = (0xffffffffffffffffUL / sizeof(*user
)); unsigned long __min_b = (0xffffffffffffffffUL / sizeof(*f
) - eb->num_fences); __min_a < __min_b ? __min_a : __min_b
; }))
return -EINVAL22;

user = u64_to_user_ptr(args->cliprects_ptr)((void *)(uintptr_t)(args->cliprects_ptr));
if (!access_ok(user, num_fences * sizeof(*user)))
return -EFAULT14;

2874#ifdef __linux__
f = krealloc(eb->fences,
     (eb->num_fences + num_fences) * sizeof(*f),
     __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));
if (!f)
return -ENOMEM12;
2880#else
f = kmalloc((eb->num_fences + num_fences) * sizeof(*f),
     __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));
if (!f)
return -ENOMEM12;
memcpy(f, eb->fences, eb->num_fences * sizeof(*f))__builtin_memcpy((f), (eb->fences), (eb->num_fences * sizeof
(*f)));
kfree(eb->fences);
2887#endif

eb->fences = f;
f += eb->num_fences;
while (num_fences--) {
struct drm_i915_gem_exec_fence user_fence;
struct drm_syncobj *syncobj;
struct dma_fence *fence = NULL((void *)0);

if (__copy_from_user(&user_fence, user++, sizeof(user_fence)))
	return -EFAULT14;

if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS(-((1<<1) << 1)))
	return -EINVAL22;

syncobj = drm_syncobj_find(eb->file, user_fence.handle);
if (!syncobj) {
	DRM_DEBUG("Invalid syncobj handle provided\n")__drm_dbg(DRM_UT_CORE, "Invalid syncobj handle provided\n");
	return -ENOENT2;
}

if (user_fence.flags & I915_EXEC_FENCE_WAIT(1<<0)) {
	fence = drm_syncobj_fence_get(syncobj);
	if (!fence) {
		DRM_DEBUG("Syncobj handle has no fence\n")__drm_dbg(DRM_UT_CORE, "Syncobj handle has no fence\n");
		drm_syncobj_put(syncobj);
		return -EINVAL22;
	}
}

2917#ifdef notyet
BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &extern char _ctassert[(!(~(ARCH_KMALLOC_MINALIGN - 1) & ~
(-((1<<1) << 1)))) ? 1 : -1 ] __attribute__((__unused__
))
	     ~__I915_EXEC_FENCE_UNKNOWN_FLAGS)extern char _ctassert[(!(~(ARCH_KMALLOC_MINALIGN - 1) & ~
(-((1<<1) << 1)))) ? 1 : -1 ] __attribute__((__unused__
));
2920#endif

f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2)({ unsigned long __bits = (user_fence.flags); ((void)0); ((typeof
(syncobj))((unsigned long)(syncobj) | __bits)); });
f->dma_fence = fence;
f->value = 0;
f->chain_fence = NULL((void *)0);
f++;
eb->num_fences++;
}

return 0;
2931}

2933static void put_fence_array(struct eb_fence *fences, int num_fences)
2934{
if (fences)
__free_fence_array(fences, num_fences);
2937}

2939static int
2940await_fence_array(struct i915_execbuffer *eb)
2941{
unsigned int n;
int err;

for (n = 0; n < eb->num_fences; n++) {
struct drm_syncobj *syncobj;
unsigned int flags;

syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2)({ unsigned long __v = (unsigned long)(eb->fences[n].syncobj
); *(&flags) = __v & ((1UL << (2)) - 1); (typeof
(eb->fences[n].syncobj))(__v & -(1UL << (2))); }
);

if (!eb->fences[n].dma_fence)
	continue;

err = i915_request_await_dma_fence(eb->request,
				   eb->fences[n].dma_fence);
if (err < 0)
	return err;
}

return 0;
2961}

2963static void signal_fence_array(const struct i915_execbuffer *eb)
2964{
struct dma_fence * const fence = &eb->request->fence;
unsigned int n;

for (n = 0; n < eb->num_fences; n++) {
struct drm_syncobj *syncobj;
unsigned int flags;

syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2)({ unsigned long __v = (unsigned long)(eb->fences[n].syncobj
); *(&flags) = __v & ((1UL << (2)) - 1); (typeof
(eb->fences[n].syncobj))(__v & -(1UL << (2))); }
);
if (!(flags & I915_EXEC_FENCE_SIGNAL(1<<1)))
	continue;

if (eb->fences[n].chain_fence) {
	drm_syncobj_add_point(syncobj,
			      eb->fences[n].chain_fence,
			      fence,
			      eb->fences[n].value);
	/*
	 * The chain's ownership is transferred to the
	 * timeline.
	 */
	eb->fences[n].chain_fence = NULL((void *)0);
} else {
	drm_syncobj_replace_fence(syncobj, fence);
}
}
2990}

2992static int
2993parse_timeline_fences(struct i915_user_extension __user *ext, void *data)
2994{
struct i915_execbuffer *eb = data;
struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences;

if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences)))
return -EFAULT14;

return add_timeline_fence_array(eb, &timeline_fences);
3002}

3004static void retire_requests(struct intel_timeline *tl, struct i915_request *end)
3005{
struct i915_request *rq, *rn;

list_for_each_entry_safe(rq, rn, &tl->requests, link)for (rq = ({ const __typeof( ((__typeof(*rq) *)0)->link ) *
__mptr = ((&tl->requests)->next); (__typeof(*rq) *)
( (char *)__mptr - __builtin_offsetof(__typeof(*rq), link) );
}), rn = ({ const __typeof( ((__typeof(*rq) *)0)->link ) *
__mptr = (rq->link.next); (__typeof(*rq) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*rq), link) );}); &rq->
link != (&tl->requests); rq = rn, rn = ({ const __typeof
( ((__typeof(*rn) *)0)->link ) *__mptr = (rn->link.next
); (__typeof(*rn) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*rn), link) );}))
if (rq == end || !i915_request_retire(rq))
	break;
3011}

3013static int eb_request_add(struct i915_execbuffer *eb, int err)
3014{
struct i915_request *rq = eb->request;
struct intel_timeline * const tl = i915_request_timeline(rq);
struct i915_sched_attr attr = {};
struct i915_request *prev;

lockdep_assert_held(&tl->mutex)do { (void)(&tl->mutex); } while(0);
lockdep_unpin_lock(&tl->mutex, rq->cookie);

trace_i915_request_add(rq);

prev = __i915_request_commit(rq);

/* Check that the context wasn't destroyed before submission */
if (likely(!intel_context_is_closed(eb->context))__builtin_expect(!!(!intel_context_is_closed(eb->context))
, 1)) {
attr = eb->gem_context->sched;
} else {
/* Serialise with context_close via the add_to_timeline */
i915_request_set_error_once(rq, -ENOENT2);
__i915_request_skip(rq);
err = -ENOENT2; /* override any transient errors */
}

__i915_request_queue(rq, &attr);

/* Try to clean up the client's timeline after submitting the request */
if (prev)
retire_requests(tl, prev);

mutex_unlock(&tl->mutex)rw_exit_write(&tl->mutex);

return err;
3046}

3048static const i915_user_extension_fn execbuf_extensions[] = {
[DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES0] = parse_timeline_fences,
3050};

3052static int
3053parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args,
	  struct i915_execbuffer *eb)
3055{
if (!(args->flags & I915_EXEC_USE_EXTENSIONS(1 << 21)))
17
←
Taking true branch→
return 0;
18
←
Returning without writing to 'eb->buckets', which participates in a condition later→

/* The execbuf2 extension mechanism reuses cliprects_ptr. So we cannot
* have another flag also using it at the same time.
*/
if (eb->args->flags & I915_EXEC_FENCE_ARRAY(1<<19))
return -EINVAL22;

if (args->num_cliprects != 0)
return -EINVAL22;

return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr)((void *)(uintptr_t)(args->cliprects_ptr)),
		    execbuf_extensions,
		    ARRAY_SIZE(execbuf_extensions)(sizeof((execbuf_extensions)) / sizeof((execbuf_extensions)[0
])),
		    eb);
3072}

3074static int
3075i915_gem_do_execbuffer(struct drm_device *dev,
       struct drm_file *file,
       struct drm_i915_gem_execbuffer2 *args,
       struct drm_i915_gem_exec_object2 *exec)
3079{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct i915_execbuffer eb;
struct dma_fence *in_fence = NULL((void *)0);
struct sync_file *out_fence = NULL((void *)0);
struct i915_vma *batch;
int out_fence_fd = -1;
int err;

BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS)extern char _ctassert[(!((~0u << 30) & ~((-((1 <<
 21) << 1)) | (3<<6) | (1<<15)))) ? 1 : -1 ]
 __attribute__((__unused__));
BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &extern char _ctassert[(!((~0u << 28) & ~-((1<<
7)<<1))) ? 1 : -1 ] __attribute__((__unused__))
     ~__EXEC_OBJECT_UNKNOWN_FLAGS)extern char _ctassert[(!((~0u << 28) & ~-((1<<
7)<<1))) ? 1 : -1 ] __attribute__((__unused__));

eb.i915 = i915;
eb.file = file;
eb.args = args;
if (DBG_FORCE_RELOC0 || !(args->flags & I915_EXEC_NO_RELOC(1<<11)))
13
←
Taking true branch→
args->flags |= __EXEC_HAS_RELOC(1UL << (31));

eb.exec = exec;
eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1);
eb.vma[0].vma = NULL((void *)0);
eb.reloc_pool = eb.batch_pool = NULL((void *)0);
eb.reloc_context = NULL((void *)0);

eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS-((1<<7)<<1);
reloc_cache_init(&eb.reloc_cache, eb.i915);

eb.buffer_count = args->buffer_count;
eb.batch_start_offset = args->batch_start_offset;
eb.batch_len = args->batch_len;
eb.trampoline = NULL((void *)0);

eb.fences = NULL((void *)0);
eb.num_fences = 0;

eb.batch_flags = 0;
if (args->flags & I915_EXEC_SECURE(1<<9)) {
14
←
Taking false branch→
if (INTEL_GEN(i915)((&(i915)->__info)->gen) >= 11)
	return -ENODEV19;

/* Return -EPERM to trigger fallback code on old binaries. */
if (!HAS_SECURE_BATCHES(i915)(((&(i915)->__info)->gen) < 6))
	return -EPERM1;

if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN0x1))
	return -EPERM1;

eb.batch_flags |= I915_DISPATCH_SECURE(1UL << (0));
}
if (args->flags & I915_EXEC_IS_PINNED(1<<10))
15
←
Taking false branch→
eb.batch_flags |= I915_DISPATCH_PINNED(1UL << (1));

err = parse_execbuf2_extensions(args, &eb);
16
←
Calling 'parse_execbuf2_extensions'→
19
←
Returning from 'parse_execbuf2_extensions'→
if (err19.1
'err' is 0
)
20
←
Taking false branch→
goto err_ext;

err = add_fence_array(&eb);
if (err)
21
←
Assuming 'err' is 0→
22
←
Taking false branch→
goto err_ext;

3140#define IN_FENCES (I915_EXEC_FENCE_IN(1<<16) | I915_EXEC_FENCE_SUBMIT(1 << 20))
if (args->flags & IN_FENCES) {
23
←
Assuming the condition is false→
24
←
Taking false branch→
if ((args->flags & IN_FENCES) == IN_FENCES)
	return -EINVAL22;

in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)((u32)(args->rsvd2)));
if (!in_fence) {
	err = -EINVAL22;
	goto err_ext;
}
}
3151#undef IN_FENCES

if (args->flags & I915_EXEC_FENCE_OUT(1<<17)) {
25
←
Assuming the condition is false→
26
←
Taking false branch→
out_fence_fd = get_unused_fd_flags(O_CLOEXEC0x10000);
if (out_fence_fd < 0) {
	err = out_fence_fd;
	goto err_in_fence;
}
}

err = eb_create(&eb);
27
←
Calling 'eb_create'→
if (err)
goto err_out_fence;

GEM_BUG_ON(!eb.lut_size)((void)0);

err = eb_select_context(&eb);
if (unlikely(err)__builtin_expect(!!(err), 0))
goto err_destroy;

err = eb_select_engine(&eb);
if (unlikely(err)__builtin_expect(!!(err), 0))
goto err_context;

err = eb_lookup_vmas(&eb);
if (err) {
eb_release_vmas(&eb, true1);
goto err_engine;
}

i915_gem_ww_ctx_init(&eb.ww, true1);

err = eb_relocate_parse(&eb);
if (err) {
/*
 * If the user expects the execobject.offset and
 * reloc.presumed_offset to be an exact match,
 * as for using NO_RELOC, then we cannot update
 * the execobject.offset until we have completed
 * relocation.
 */
args->flags &= ~__EXEC_HAS_RELOC(1UL << (31));
goto err_vma;
}

ww_acquire_done(&eb.ww.ctx);

batch = eb.batch->vma;

/* All GPU relocation batches must be submitted prior to the user rq */
GEM_BUG_ON(eb.reloc_cache.rq)((void)0);

/* Allocate a request for this batch buffer nice and early. */
eb.request = i915_request_create(eb.context);
if (IS_ERR(eb.request)) {
err = PTR_ERR(eb.request);
goto err_vma;
}

if (in_fence) {
if (args->flags & I915_EXEC_FENCE_SUBMIT(1 << 20))
	err = i915_request_await_execution(eb.request,
					   in_fence,
					   eb.engine->bond_execute);
else
	err = i915_request_await_dma_fence(eb.request,
					   in_fence);
if (err < 0)
	goto err_request;
}

if (eb.fences) {
err = await_fence_array(&eb);
if (err)
	goto err_request;
}

if (out_fence_fd != -1) {
out_fence = sync_file_create(&eb.request->fence);
if (!out_fence) {
	err = -ENOMEM12;
	goto err_request;
}
}

/*
* Whilst this request exists, batch_obj will be on the
* active_list, and so will hold the active reference. Only when this
* request is retired will the the batch_obj be moved onto the
* inactive_list and lose its active reference. Hence we do not need
* to explicitly hold another reference here.
*/
eb.request->batch = batch;
if (eb.batch_pool)
intel_gt_buffer_pool_mark_active(eb.batch_pool, eb.request);

trace_i915_request_queue(eb.request, eb.batch_flags);
err = eb_submit(&eb, batch);
3249err_request:
i915_request_get(eb.request);
err = eb_request_add(&eb, err);

if (eb.fences)
signal_fence_array(&eb);

if (out_fence) {
if (err == 0) {
	fd_install(out_fence_fd, out_fence->file);
	args->rsvd2 &= GENMASK_ULL(31, 0)(((~0ULL) >> (64 - (31) - 1)) & ((~0ULL) << (
0))); /* keep in-fence */
	args->rsvd2 |= (u64)out_fence_fd << 32;
	out_fence_fd = -1;
} else {
	fput(out_fence->file);
}
}
i915_request_put(eb.request);

3268err_vma:
eb_release_vmas(&eb, true1);
if (eb.trampoline)
i915_vma_unpin(eb.trampoline);
WARN_ON(err == -EDEADLK)({ int __ret = !!((err == -11)); if (__ret) printf("%s", "WARN_ON("
 "err == -11" ")"); __builtin_expect(!!(__ret), 0); });
i915_gem_ww_ctx_fini(&eb.ww);

if (eb.batch_pool)
intel_gt_buffer_pool_put(eb.batch_pool);
if (eb.reloc_pool)
intel_gt_buffer_pool_put(eb.reloc_pool);
if (eb.reloc_context)
intel_context_put(eb.reloc_context);
3281err_engine:
eb_put_engine(&eb);
3283err_context:
i915_gem_context_put(eb.gem_context);
3285err_destroy:
eb_destroy(&eb);
3287err_out_fence:
if (out_fence_fd != -1)
put_unused_fd(out_fence_fd);
3290err_in_fence:
dma_fence_put(in_fence);
3292err_ext:
put_fence_array(eb.fences, eb.num_fences);
return err;
3295}

3297static size_t eb_element_size(void)
3298{
return sizeof(struct drm_i915_gem_exec_object2) + sizeof(struct eb_vma);
3300}

3302static bool_Bool check_buffer_count(size_t count)
3303{
const size_t sz = eb_element_size();

/*
* When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup
* array size (see eb_create()). Otherwise, we can accept an array as
* large as can be addressed (though use large arrays at your peril)!
*/

return !(count < 1 || count > INT_MAX0x7fffffff || count > SIZE_MAX0xffffffffffffffffUL / sz - 1);
3313}

3315/*
* Legacy execbuffer just creates an exec2 list from the original exec object
* list array and passes it to the real function.
*/
3319int
3320i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
	  struct drm_file *file)
3322{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct drm_i915_gem_execbuffer *args = data;
struct drm_i915_gem_execbuffer2 exec2;
struct drm_i915_gem_exec_object *exec_list = NULL((void *)0);
struct drm_i915_gem_exec_object2 *exec2_list = NULL((void *)0);
const size_t count = args->buffer_count;
unsigned int i;
int err;

if (!check_buffer_count(count)) {
1
Taking false branch→
drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf2 with %zd buffers\n"
, count);
return -EINVAL22;
}

exec2.buffers_ptr = args->buffers_ptr;
exec2.buffer_count = args->buffer_count;
exec2.batch_start_offset = args->batch_start_offset;
exec2.batch_len = args->batch_len;
exec2.DR1 = args->DR1;
exec2.DR4 = args->DR4;
exec2.num_cliprects = args->num_cliprects;
exec2.cliprects_ptr = args->cliprects_ptr;
exec2.flags = I915_EXEC_RENDER(1<<0);
i915_execbuffer2_set_context_id(exec2, 0)(exec2).rsvd1 = 0 & (0xffffffff);

err = i915_gem_check_execbuffer(&exec2);
if (err1.1
'err' is 0
)
2
←
Taking false branch→
return err;

/* Copy in the exec list from userland */
exec_list = kvmalloc_array(count, sizeof(*exec_list),
		   __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));

/* Allocate extra slots for use by the command parser */
exec2_list = kvmalloc_array(count + 2, eb_element_size(),
		    __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));
if (exec_list == NULL((void *)0) || exec2_list == NULL((void *)0)) {
3
←
Assuming 'exec_list' is not equal to NULL→
4
←
Assuming 'exec2_list' is not equal to NULL→
5
←
Taking false branch→
drm_dbg(&i915->drm,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Failed to allocate exec list for %d buffers\n"
, args->buffer_count)
	"Failed to allocate exec list for %d buffers\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Failed to allocate exec list for %d buffers\n"
, args->buffer_count)
	args->buffer_count)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Failed to allocate exec list for %d buffers\n"
, args->buffer_count);
kvfree(exec_list);
kvfree(exec2_list);
return -ENOMEM12;
}
err = copy_from_user(exec_list,
	     u64_to_user_ptr(args->buffers_ptr)((void *)(uintptr_t)(args->buffers_ptr)),
	     sizeof(*exec_list) * count);
if (err5.1
'err' is 0
) {
6
←
Taking false branch→
drm_dbg(&i915->drm, "copy %d exec entries failed %d\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "copy %d exec entries failed %d\n"
, args->buffer_count, err)
	args->buffer_count, err)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "copy %d exec entries failed %d\n"
, args->buffer_count, err);
kvfree(exec_list);
kvfree(exec2_list);
return -EFAULT14;
}

for (i = 0; i6.1
'i' is < field 'buffer_count'
 < args->buffer_count; i++) {
7
←
Loop condition is true.  Entering loop body→
10
←
Assuming 'i' is >= field 'buffer_count'→
11
←
Loop condition is false. Execution continues on line 3390→
exec2_list[i].handle = exec_list[i].handle;
exec2_list[i].relocation_count = exec_list[i].relocation_count;
exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
exec2_list[i].alignment = exec_list[i].alignment;
exec2_list[i].offset = exec_list[i].offset;
if (INTEL_GEN(to_i915(dev))((&(to_i915(dev))->__info)->gen) < 4)
8
←
Assuming field 'gen' is >= 4→
9
←
Taking false branch→
	exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE(1<<0);
else
	exec2_list[i].flags = 0;
}

err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
12
←
Calling 'i915_gem_do_execbuffer'→
if (exec2.flags & __EXEC_HAS_RELOC(1UL << (31))) {
struct drm_i915_gem_exec_object __user *user_exec_list =
	u64_to_user_ptr(args->buffers_ptr)((void *)(uintptr_t)(args->buffers_ptr));

/* Copy the new buffer offsets back to the user's exec list. */
for (i = 0; i < args->buffer_count; i++) {
	if (!(exec2_list[i].offset & UPDATE(1ULL << (7))))
		continue;

	exec2_list[i].offset =
		gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK-(1ULL << (12)));
	exec2_list[i].offset &= PIN_OFFSET_MASK-(1ULL << (12));
	if (__copy_to_user(&user_exec_list[i].offset,
			   &exec2_list[i].offset,
			   sizeof(user_exec_list[i].offset)))
		break;
}
}

kvfree(exec_list);
kvfree(exec2_list);
return err;
3413}

3415int
3416i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
	   struct drm_file *file)
3418{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct drm_i915_gem_execbuffer2 *args = data;
struct drm_i915_gem_exec_object2 *exec2_list;
const size_t count = args->buffer_count;
int err;

if (!check_buffer_count(count)) {
drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "execbuf2 with %zd buffers\n"
, count);
return -EINVAL22;
}

err = i915_gem_check_execbuffer(args);
if (err)
return err;

/* Allocate extra slots for use by the command parser */
exec2_list = kvmalloc_array(count + 2, eb_element_size(),
		    __GFP_NOWARN0 | GFP_KERNEL(0x0001 | 0x0004));
if (exec2_list == NULL((void *)0)) {
drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n",drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Failed to allocate exec list for %zd buffers\n"
, count)
	count)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Failed to allocate exec list for %zd buffers\n"
, count);
return -ENOMEM12;
}
if (copy_from_user(exec2_list,
	   u64_to_user_ptr(args->buffers_ptr)((void *)(uintptr_t)(args->buffers_ptr)),
	   sizeof(*exec2_list) * count)) {
drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count)drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "copy %zd exec entries failed\n"
, count);
kvfree(exec2_list);
return -EFAULT14;
}

err = i915_gem_do_execbuffer(dev, file, args, exec2_list);

/*
* Now that we have begun execution of the batchbuffer, we ignore
* any new error after this point. Also given that we have already
* updated the associated relocations, we try to write out the current
* object locations irrespective of any error.
*/
if (args->flags & __EXEC_HAS_RELOC(1UL << (31))) {
struct drm_i915_gem_exec_object2 __user *user_exec_list =
	u64_to_user_ptr(args->buffers_ptr)((void *)(uintptr_t)(args->buffers_ptr));
unsigned int i;

/* Copy the new buffer offsets back to the user's exec list. */
/*
 * Note: count * sizeof(*user_exec_list) does not overflow,
 * because we checked 'count' in check_buffer_count().
 *
 * And this range already got effectively checked earlier
 * when we did the "copy_from_user()" above.
 */
if (!user_write_access_begin(user_exec_list,access_ok(user_exec_list, count * sizeof(*user_exec_list))
			     count * sizeof(*user_exec_list))access_ok(user_exec_list, count * sizeof(*user_exec_list)))
	goto end;

for (i = 0; i < args->buffer_count; i++) {
	if (!(exec2_list[i].offset & UPDATE(1ULL << (7))))
		continue;

	exec2_list[i].offset =
		gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK-(1ULL << (12)));
	unsafe_put_user(exec2_list[i].offset,({ __typeof((exec2_list[i].offset)) __tmp = (exec2_list[i].offset
); if (copyout(&(__tmp), &user_exec_list[i].offset, sizeof
(__tmp)) != 0) goto end_user; })
			&user_exec_list[i].offset,({ __typeof((exec2_list[i].offset)) __tmp = (exec2_list[i].offset
); if (copyout(&(__tmp), &user_exec_list[i].offset, sizeof
(__tmp)) != 0) goto end_user; })
			end_user)({ __typeof((exec2_list[i].offset)) __tmp = (exec2_list[i].offset
); if (copyout(&(__tmp), &user_exec_list[i].offset, sizeof
(__tmp)) != 0) goto end_user; });
}
3485end_user:
user_write_access_end();
3487end:;
}

args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS(-((1 << 21) << 1));
kvfree(exec2_list);
return err;
3493}

3495#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
3496#include "selftests/i915_gem_execbuffer.c"
3497#endif