Bug Summary

File:dev/pci/drm/i915/gem/i915_gem_shmem.c
Warning:line 670, column 3
Value stored to 'mask' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name i915_gem_shmem.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/gem/i915_gem_shmem.c
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7#include <linux/pagevec.h>
8#include <linux/shmem_fs.h>
9#include <linux/swap.h>
10
11#include <drm/drm_cache.h>
12
13#include "gem/i915_gem_region.h"
14#include "i915_drv.h"
15#include "i915_gem_object.h"
16#include "i915_gem_tiling.h"
17#include "i915_gemfs.h"
18#include "i915_scatterlist.h"
19#include "i915_trace.h"
20
21/*
22 * Move pages to appropriate lru and release the pagevec, decrementing the
23 * ref count of those pages.
24 */
25static void check_release_pagevec(struct pagevec *pvec)
26{
27 STUB()do { printf("%s: stub\n", __func__); } while(0);
28#ifdef notyet
29 check_move_unevictable_pages(pvec);
30#endif
31 __pagevec_release(pvec);
32 cond_resched()do { if (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0"
: "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self
))); __ci;})->ci_schedstate.spc_schedflags & 0x0002) yield
(); } while (0);
33}
34
35void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping,
36 bool_Bool dirty, bool_Bool backup,
37 struct drm_i915_gem_object *obj)
38{
39 struct sgt_iter sgt_iter;
40 struct pagevec pvec;
41 struct vm_page *page;
42
43#ifdef __linux__
44 mapping_clear_unevictable(mapping);
45#endif
46
47 pagevec_init(&pvec);
48 for_each_sgt_page(page, sgt_iter, st)for ((sgt_iter) = __sgt_iter((st)->sgl, 0); ((page) = (sgt_iter
).pfn == 0 ? ((void *)0) : (PHYS_TO_VM_PAGE(((paddr_t)((sgt_iter
).pfn + ((sgt_iter).curr >> 12)) << 12)))); (((sgt_iter
).curr += (1 << 12)) >= (sgt_iter).max) ? (sgt_iter)
= __sgt_iter(__sg_next((sgt_iter).sgp), 0), 0 : 0) {
49 if (dirty)
50 set_page_dirty(page)x86_atomic_clearbits_u32(&page->pg_flags, 0x00000008);
51
52#ifdef __linux__
53 if (backup)
54 mark_page_accessed(page);
55
56 if (!pagevec_add(&pvec, page))
57 check_release_pagevec(&pvec);
58#endif
59 }
60#ifdef __linux__
61 if (pagevec_count(&pvec))
62 check_release_pagevec(&pvec);
63#else
64 uvm_obj_unwire(obj->base.uao, 0, obj->base.size);
65#endif
66
67 sg_free_table(st);
68}
69
70int shmem_sg_alloc_table(struct drm_i915_privateinteldrm_softc *i915, struct sg_table *st,
71 size_t size, struct intel_memory_region *mr,
72 struct address_space *mapping,
73 unsigned int max_segment,
74 struct drm_i915_gem_object *obj)
75{
76 const unsigned long page_count = size / PAGE_SIZE(1 << 12);
77 unsigned long i;
78 struct scatterlist *sg;
79 struct vm_page *page;
80 unsigned long last_pfn = 0; /* suppress gcc warning */
81 gfp_t noreclaim;
82 int ret;
83 struct pglist plist;
84
85 /*
86 * If there's no chance of allocating enough pages for the whole
87 * object, bail early.
88 */
89 if (size > resource_size(&mr->region))
90 return -ENOMEM12;
91
92 if (sg_alloc_table(st, page_count, GFP_KERNEL(0x0001 | 0x0004) | __GFP_NOWARN0))
93 return -ENOMEM12;
94#ifdef __linux__
95
96 /*
97 * Get the list of pages out of our struct file. They'll be pinned
98 * at this point until we release them.
99 *
100 * Fail silently without starting the shrinker
101 */
102 mapping_set_unevictable(mapping);
103 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
104 noreclaim |= __GFP_NORETRY0 | __GFP_NOWARN0;
105
106 sg = st->sgl;
107 st->nents = 0;
108 for (i = 0; i < page_count; i++) {
109 const unsigned int shrink[] = {
110 I915_SHRINK_BOUND(1UL << (1)) | I915_SHRINK_UNBOUND(1UL << (0)),
111 0,
112 }, *s = shrink;
113 gfp_t gfp = noreclaim;
114
115 do {
116 cond_resched()do { if (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0"
: "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self
))); __ci;})->ci_schedstate.spc_schedflags & 0x0002) yield
(); } while (0);
117 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
118 if (!IS_ERR(page))
119 break;
120
121 if (!*s) {
122 ret = PTR_ERR(page);
123 goto err_sg;
124 }
125
126 i915_gem_shrink(NULL((void *)0), i915, 2 * page_count, NULL((void *)0), *s++);
127
128 /*
129 * We've tried hard to allocate the memory by reaping
130 * our own buffer, now let the real VM do its job and
131 * go down in flames if truly OOM.
132 *
133 * However, since graphics tend to be disposable,
134 * defer the oom here by reporting the ENOMEM back
135 * to userspace.
136 */
137 if (!*s) {
138 /* reclaim and warn, but no oom */
139 gfp = mapping_gfp_mask(mapping);
140
141 /*
142 * Our bo are always dirty and so we require
143 * kswapd to reclaim our pages (direct reclaim
144 * does not effectively begin pageout of our
145 * buffers on its own). However, direct reclaim
146 * only waits for kswapd when under allocation
147 * congestion. So as a result __GFP_RECLAIM is
148 * unreliable and fails to actually reclaim our
149 * dirty pages -- unless you try over and over
150 * again with !__GFP_NORETRY. However, we still
151 * want to fail this allocation rather than
152 * trigger the out-of-memory killer and for
153 * this we want __GFP_RETRY_MAYFAIL.
154 */
155 gfp |= __GFP_RETRY_MAYFAIL0 | __GFP_NOWARN0;
156 }
157 } while (1);
158
159 if (!i ||
160 sg->length >= max_segment ||
161 page_to_pfn(page)(((page)->phys_addr) / (1 << 12)) != last_pfn + 1) {
162 if (i)
163 sg = sg_next(sg);
164
165 st->nents++;
166 sg_set_page(sg, page, PAGE_SIZE(1 << 12), 0);
167 } else {
168 sg->length += PAGE_SIZE(1 << 12);
169 }
170 last_pfn = page_to_pfn(page)(((page)->phys_addr) / (1 << 12));
171
172 /* Check that the i965g/gm workaround works. */
173 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL)((void)0);
174 }
175#else
176 sg = st->sgl;
177 st->nents = 0;
178
179 TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0);
180 if (uvm_obj_wire(obj->base.uao, 0, obj->base.size, &plist)) {
181 sg_free_table(st);
182 kfree(st);
183 return -ENOMEM12;
184 }
185
186 i = 0;
187 TAILQ_FOREACH(page, &plist, pageq)for((page) = ((&plist)->tqh_first); (page) != ((void *
)0); (page) = ((page)->pageq.tqe_next)) {
188 if (i)
189 sg = sg_next(sg);
190 st->nents++;
191 sg_set_page(sg, page, PAGE_SIZE(1 << 12), 0);
192 i++;
193 }
194#endif
195 if (sg) /* loop terminated early; short sg table */
196 sg_mark_end(sg);
197
198 /* Trim unused sg entries to avoid wasting memory. */
199 i915_sg_trim(st);
200
201 return 0;
202#ifdef notyet
203err_sg:
204 sg_mark_end(sg);
205 if (sg != st->sgl) {
206 shmem_sg_free_table(st, mapping, false0, false0);
207 } else {
208 mapping_clear_unevictable(mapping);
209 sg_free_table(st);
210 }
211
212 /*
213 * shmemfs first checks if there is enough memory to allocate the page
214 * and reports ENOSPC should there be insufficient, along with the usual
215 * ENOMEM for a genuine allocation failure.
216 *
217 * We use ENOSPC in our driver to mean that we have run out of aperture
218 * space and so want to translate the error from shmemfs back to our
219 * usual understanding of ENOMEM.
220 */
221 if (ret == -ENOSPC28)
222 ret = -ENOMEM12;
223
224 return ret;
225#endif
226}
227
228static int shmem_get_pages(struct drm_i915_gem_object *obj)
229{
230 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
231 struct intel_memory_region *mem = obj->mm.region;
232#ifdef __linux__
233 struct address_space *mapping = obj->base.filp->f_mapping;
234#endif
235 const unsigned long page_count = obj->base.size / PAGE_SIZE(1 << 12);
236 unsigned int max_segment = i915_sg_segment_size(i915->drm.dev);
237 struct sg_table *st;
238 struct sgt_iter sgt_iter;
239 struct vm_page *page;
240 int ret;
241
242 /*
243 * Assert that the object is not currently in any GPU domain. As it
244 * wasn't in the GTT, there shouldn't be any way it could have been in
245 * a GPU cache
246 */
247 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS)((void)0);
248 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS)((void)0);
249
250rebuild_st:
251 st = kmalloc(sizeof(*st), GFP_KERNEL(0x0001 | 0x0004) | __GFP_NOWARN0);
252 if (!st)
253 return -ENOMEM12;
254
255#ifdef __linux__
256 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping,
257 max_segment);
258#else
259 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, NULL((void *)0),
260 max_segment, obj);
261#endif
262 if (ret)
263 goto err_st;
264
265 ret = i915_gem_gtt_prepare_pages(obj, st);
266 if (ret) {
267 /*
268 * DMA remapping failed? One possible cause is that
269 * it could not reserve enough large entries, asking
270 * for PAGE_SIZE chunks instead may be helpful.
271 */
272 if (max_segment > PAGE_SIZE(1 << 12)) {
273#ifdef __linux__
274 for_each_sgt_page(page, sgt_iter, st)for ((sgt_iter) = __sgt_iter((st)->sgl, 0); ((page) = (sgt_iter
).pfn == 0 ? ((void *)0) : (PHYS_TO_VM_PAGE(((paddr_t)((sgt_iter
).pfn + ((sgt_iter).curr >> 12)) << 12)))); (((sgt_iter
).curr += (1 << 12)) >= (sgt_iter).max) ? (sgt_iter)
= __sgt_iter(__sg_next((sgt_iter).sgp), 0), 0 : 0)
275 put_page(page);
276#else
277 uvm_obj_unwire(obj->base.uao, 0, obj->base.size);
278#endif
279 sg_free_table(st);
280 kfree(st);
281
282 max_segment = PAGE_SIZE(1 << 12);
283 goto rebuild_st;
284 } else {
285 dev_warn(i915->drm.dev,printf("drm:pid%d:%s *WARNING* " "Failed to DMA remap %lu pages\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , page_count
)
286 "Failed to DMA remap %lu pages\n",printf("drm:pid%d:%s *WARNING* " "Failed to DMA remap %lu pages\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , page_count
)
287 page_count)printf("drm:pid%d:%s *WARNING* " "Failed to DMA remap %lu pages\n"
, ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc->p_p->ps_pid, __func__ , page_count
);
288 goto err_pages;
289 }
290 }
291
292 if (i915_gem_object_needs_bit17_swizzle(obj))
293 i915_gem_object_do_bit_17_swizzle(obj, st);
294
295 if (i915_gem_object_can_bypass_llc(obj))
296 obj->cache_dirty = true1;
297
298 __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl));
299
300 return 0;
301
302err_pages:
303#ifdef __linux__
304 shmem_sg_free_table(st, mapping, false0, false0);
305#else
306 shmem_sg_free_table(st, NULL((void *)0), false0, false0, obj);
307#endif
308 /*
309 * shmemfs first checks if there is enough memory to allocate the page
310 * and reports ENOSPC should there be insufficient, along with the usual
311 * ENOMEM for a genuine allocation failure.
312 *
313 * We use ENOSPC in our driver to mean that we have run out of aperture
314 * space and so want to translate the error from shmemfs back to our
315 * usual understanding of ENOMEM.
316 */
317err_st:
318 if (ret == -ENOSPC28)
319 ret = -ENOMEM12;
320
321 kfree(st);
322
323 return ret;
324}
325
326static int
327shmem_truncate(struct drm_i915_gem_object *obj)
328{
329 /*
330 * Our goal here is to return as much of the memory as
331 * is possible back to the system as we are called from OOM.
332 * To do this we must instruct the shmfs to drop all of its
333 * backing pages, *now*.
334 */
335#ifdef __linux__
336 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
337#else
338 rw_enter(obj->base.uao->vmobjlock, RW_WRITE0x0001UL);
339 obj->base.uao->pgops->pgo_flush(obj->base.uao, 0, obj->base.size,
340 PGO_ALLPAGES0x010 | PGO_FREE0x008);
341 rw_exit(obj->base.uao->vmobjlock);
342#endif
343 obj->mm.madv = __I915_MADV_PURGED2;
344 obj->mm.pages = ERR_PTR(-EFAULT14);
345
346 return 0;
347}
348
349void __shmem_writeback(size_t size, struct address_space *mapping)
350{
351 STUB()do { printf("%s: stub\n", __func__); } while(0);
352#ifdef notyet
353 struct writeback_control wbc = {
354 .sync_mode = WB_SYNC_NONE,
355 .nr_to_write = SWAP_CLUSTER_MAX,
356 .range_start = 0,
357 .range_end = LLONG_MAX0x7fffffffffffffffLL,
358 .for_reclaim = 1,
359 };
360 unsigned long i;
361
362 /*
363 * Leave mmapings intact (GTT will have been revoked on unbinding,
364 * leaving only CPU mmapings around) and add those pages to the LRU
365 * instead of invoking writeback so they are aged and paged out
366 * as normal.
367 */
368
369 /* Begin writeback on each dirty page */
370 for (i = 0; i < size >> PAGE_SHIFT12; i++) {
371 struct vm_page *page;
372
373 page = find_lock_page(mapping, i);
374 if (!page)
375 continue;
376
377 if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
378 int ret;
379
380 SetPageReclaim(page);
381 ret = mapping->a_ops->writepage(page, &wbc);
382 if (!PageWriteback(page))
383 ClearPageReclaim(page);
384 if (!ret)
385 goto put;
386 }
387 unlock_page(page);
388put:
389 put_page(page);
390 }
391#endif
392}
393
394static void
395shmem_writeback(struct drm_i915_gem_object *obj)
396{
397 STUB()do { printf("%s: stub\n", __func__); } while(0);
398#ifdef notyet
399 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping);
400#endif
401}
402
403static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
404{
405 switch (obj->mm.madv) {
406 case I915_MADV_DONTNEED1:
407 return i915_gem_object_truncate(obj);
408 case __I915_MADV_PURGED2:
409 return 0;
410 }
411
412 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK(1UL << (0)))
413 shmem_writeback(obj);
414
415 return 0;
416}
417
418void
419__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
420 struct sg_table *pages,
421 bool_Bool needs_clflush)
422{
423 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
424
425 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED)((void)0);
426
427 if (obj->mm.madv == I915_MADV_DONTNEED1)
428 obj->mm.dirty = false0;
429
430 if (needs_clflush &&
431 (obj->read_domains & I915_GEM_DOMAIN_CPU0x00000001) == 0 &&
432 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ(1UL << (0))))
433 drm_clflush_sg(pages);
434
435 __start_cpu_write(obj);
436 /*
437 * On non-LLC igfx platforms, force the flush-on-acquire if this is ever
438 * swapped-in. Our async flush path is not trust worthy enough yet(and
439 * happens in the wrong order), and with some tricks it's conceivable
440 * for userspace to change the cache-level to I915_CACHE_NONE after the
441 * pages are swapped-in, and since execbuf binds the object before doing
442 * the async flush, we have a race window.
443 */
444 if (!HAS_LLC(i915)((&(i915)->__info)->has_llc) && !IS_DGFX(i915)((&(i915)->__info)->is_dgfx))
445 obj->cache_dirty = true1;
446}
447
448void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
449{
450 __i915_gem_object_release_shmem(obj, pages, true1);
451
452 i915_gem_gtt_finish_pages(obj, pages);
453
454 if (i915_gem_object_needs_bit17_swizzle(obj))
455 i915_gem_object_save_bit_17_swizzle(obj, pages);
456
457#ifdef __linux__
458 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping,
459 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED0);
460#else
461 shmem_sg_free_table(pages, NULL((void *)0),
462 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED0, obj);
463#endif
464 kfree(pages);
465 obj->mm.dirty = false0;
466}
467
468static void
469shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
470{
471 if (likely(i915_gem_object_has_struct_page(obj))__builtin_expect(!!(i915_gem_object_has_struct_page(obj)), 1))
472 i915_gem_object_put_pages_shmem(obj, pages);
473 else
474 i915_gem_object_put_pages_phys(obj, pages);
475}
476
477static int
478shmem_pwrite(struct drm_i915_gem_object *obj,
479 const struct drm_i915_gem_pwrite *arg)
480{
481#ifdef __linux__
482 struct address_space *mapping = obj->base.filp->f_mapping;
483 const struct address_space_operations *aops = mapping->a_ops;
484#endif
485 char __user *user_data = u64_to_user_ptr(arg->data_ptr)((void *)(uintptr_t)(arg->data_ptr));
486 u64 remain, offset;
487 unsigned int pg;
488
489 /* Caller already validated user args */
490 GEM_BUG_ON(!access_ok(user_data, arg->size))((void)0);
491
492 if (!i915_gem_object_has_struct_page(obj))
493 return i915_gem_object_pwrite_phys(obj, arg);
494
495 /*
496 * Before we instantiate/pin the backing store for our use, we
497 * can prepopulate the shmemfs filp efficiently using a write into
498 * the pagecache. We avoid the penalty of instantiating all the
499 * pages, important if the user is just writing to a few and never
500 * uses the object on the GPU, and using a direct write into shmemfs
501 * allows it to avoid the cost of retrieving a page (either swapin
502 * or clearing-before-use) before it is overwritten.
503 */
504 if (i915_gem_object_has_pages(obj))
505 return -ENODEV19;
506
507 if (obj->mm.madv != I915_MADV_WILLNEED0)
508 return -EFAULT14;
509
510 /*
511 * Before the pages are instantiated the object is treated as being
512 * in the CPU domain. The pages will be clflushed as required before
513 * use, and we can freely write into the pages directly. If userspace
514 * races pwrite with any other operation; corruption will ensue -
515 * that is userspace's prerogative!
516 */
517
518 remain = arg->size;
519 offset = arg->offset;
520 pg = offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1));
521
522 do {
523 unsigned int len, unwritten;
524 struct vm_page *page;
525 void *data, *vaddr;
526 int err;
527 char c;
528
529 len = PAGE_SIZE(1 << 12) - pg;
530 if (len > remain)
531 len = remain;
532
533 /* Prefault the user page to reduce potential recursion */
534 err = __get_user(c, user_data)-copyin((user_data), &((c)), sizeof((c)));
535 if (err)
536 return err;
537
538 err = __get_user(c, user_data + len - 1)-copyin((user_data + len - 1), &((c)), sizeof((c)));
539 if (err)
540 return err;
541
542#ifdef __linux__
543 err = aops->write_begin(obj->base.filp, mapping, offset, len,
544 &page, &data);
545 if (err < 0)
546 return err;
547#else
548 struct pglist plist;
549 TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0);
550 if (uvm_obj_wire(obj->base.uao, trunc_page(offset)((offset) & ~((1 << 12) - 1)),
551 trunc_page(offset)((offset) & ~((1 << 12) - 1)) + PAGE_SIZE(1 << 12), &plist)) {
552 return -ENOMEM12;
553 }
554 page = TAILQ_FIRST(&plist)((&plist)->tqh_first);
555#endif
556
557 vaddr = kmap_atomic(page);
558 unwritten = __copy_from_user_inatomic(vaddr + pg,
559 user_data,
560 len);
561 kunmap_atomic(vaddr);
562
563#ifdef __linux__
564 err = aops->write_end(obj->base.filp, mapping, offset, len,
565 len - unwritten, page, data);
566 if (err < 0)
567 return err;
568#else
569 uvm_obj_unwire(obj->base.uao, trunc_page(offset)((offset) & ~((1 << 12) - 1)),
570 trunc_page(offset)((offset) & ~((1 << 12) - 1)) + PAGE_SIZE(1 << 12));
571#endif
572
573 /* We don't handle -EFAULT, leave it to the caller to check */
574 if (unwritten)
575 return -ENODEV19;
576
577 remain -= len;
578 user_data += len;
579 offset += len;
580 pg = 0;
581 } while (remain);
582
583 return 0;
584}
585
586static int
587shmem_pread(struct drm_i915_gem_object *obj,
588 const struct drm_i915_gem_pread *arg)
589{
590 if (!i915_gem_object_has_struct_page(obj))
591 return i915_gem_object_pread_phys(obj, arg);
592
593 return -ENODEV19;
594}
595
596static void shmem_release(struct drm_i915_gem_object *obj)
597{
598 if (i915_gem_object_has_struct_page(obj))
599 i915_gem_object_release_memory_region(obj);
600
601#ifdef __linux__
602 fput(obj->base.filp);
603#endif
604}
605
606const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
607 .name = "i915_gem_object_shmem",
608 .flags = I915_GEM_OBJECT_IS_SHRINKABLE(1UL << (1)),
609
610 .get_pages = shmem_get_pages,
611 .put_pages = shmem_put_pages,
612 .truncate = shmem_truncate,
613 .shrink = shmem_shrink,
614
615 .pwrite = shmem_pwrite,
616 .pread = shmem_pread,
617
618 .release = shmem_release,
619};
620
621#ifdef __linux__
622static int __create_shmem(struct drm_i915_privateinteldrm_softc *i915,
623 struct drm_gem_object *obj,
624 resource_size_t size)
625{
626 unsigned long flags = VM_NORESERVE;
627 struct file *filp;
628
629 drm_gem_private_object_init(&i915->drm, obj, size);
630
631 if (i915->mm.gemfs)
632 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
633 flags);
634 else
635 filp = shmem_file_setup("i915", size, flags);
636 if (IS_ERR(filp))
637 return PTR_ERR(filp);
638
639 obj->filp = filp;
640 return 0;
641}
642#endif
643
644static int shmem_object_init(struct intel_memory_region *mem,
645 struct drm_i915_gem_object *obj,
646 resource_size_t offset,
647 resource_size_t size,
648 resource_size_t page_size,
649 unsigned int flags)
650{
651 static struct lock_class_key lock_class;
652 struct drm_i915_privateinteldrm_softc *i915 = mem->i915;
653 struct address_space *mapping;
654 unsigned int cache_level;
655 gfp_t mask;
656 int ret;
657
658#ifdef __linux__
659 ret = __create_shmem(i915, &obj->base, size);
660#else
661 ret = drm_gem_object_init(&i915->drm, &obj->base, size);
662#endif
663 if (ret)
664 return ret;
665
666 mask = GFP_HIGHUSER0 | __GFP_RECLAIMABLE0;
667 if (IS_I965GM(i915)IS_PLATFORM(i915, INTEL_I965GM) || IS_I965G(i915)IS_PLATFORM(i915, INTEL_I965G)) {
668 /* 965gm cannot relocate objects above 4GiB. */
669 mask &= ~__GFP_HIGHMEM0;
670 mask |= __GFP_DMA320x00010000;
Value stored to 'mask' is never read
671 }
672
673#ifdef __linux__
674 mapping = obj->base.filp->f_mapping;
675 mapping_set_gfp_mask(mapping, mask);
676 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM))((void)0);
677#endif
678
679 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, flags);
680 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE(1UL << (0));
681 obj->write_domain = I915_GEM_DOMAIN_CPU0x00000001;
682 obj->read_domains = I915_GEM_DOMAIN_CPU0x00000001;
683
684 if (HAS_LLC(i915)((&(i915)->__info)->has_llc))
685 /* On some devices, we can have the GPU use the LLC (the CPU
686 * cache) for about a 10% performance improvement
687 * compared to uncached. Graphics requests other than
688 * display scanout are coherent with the CPU in
689 * accessing this cache. This means in this mode we
690 * don't need to clflush on the CPU side, and on the
691 * GPU side we only need to flush internal caches to
692 * get data visible to the CPU.
693 *
694 * However, we maintain the display planes as UC, and so
695 * need to rebind when first used as such.
696 */
697 cache_level = I915_CACHE_LLC;
698 else
699 cache_level = I915_CACHE_NONE;
700
701 i915_gem_object_set_cache_coherency(obj, cache_level);
702
703 i915_gem_object_init_memory_region(obj, mem);
704
705 return 0;
706}
707
708struct drm_i915_gem_object *
709i915_gem_object_create_shmem(struct drm_i915_privateinteldrm_softc *i915,
710 resource_size_t size)
711{
712 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
713 size, 0, 0);
714}
715
716/* Allocate a new GEM object and fill it with the supplied data */
717#ifdef __linux__
718struct drm_i915_gem_object *
719i915_gem_object_create_shmem_from_data(struct drm_i915_privateinteldrm_softc *dev_priv,
720 const void *data, resource_size_t size)
721{
722 struct drm_i915_gem_object *obj;
723 struct file *file;
724 const struct address_space_operations *aops;
725 resource_size_t offset;
726 int err;
727
728 GEM_WARN_ON(IS_DGFX(dev_priv))({ __builtin_expect(!!(!!(((&(dev_priv)->__info)->is_dgfx
))), 0); });
729 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)((((size) + (((1 << 12)) - 1)) / ((1 << 12))) * (
(1 << 12))));
730 if (IS_ERR(obj))
731 return obj;
732
733 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU)((void)0);
734
735 file = obj->base.filp;
736 aops = file->f_mapping->a_ops;
737 offset = 0;
738 do {
739 unsigned int len = min_t(typeof(size), size, PAGE_SIZE)({ typeof(size) __min_a = (size); typeof(size) __min_b = ((1 <<
12)); __min_a < __min_b ? __min_a : __min_b; });
740 struct vm_page *page;
741 void *pgdata, *vaddr;
742
743 err = aops->write_begin(file, file->f_mapping, offset, len,
744 &page, &pgdata);
745 if (err < 0)
746 goto fail;
747
748 vaddr = kmap(page);
749 memcpy(vaddr, data, len)__builtin_memcpy((vaddr), (data), (len));
750 kunmap(page);
751
752 err = aops->write_end(file, file->f_mapping, offset, len, len,
753 page, pgdata);
754 if (err < 0)
755 goto fail;
756
757 size -= len;
758 data += len;
759 offset += len;
760 } while (size);
761
762 return obj;
763
764fail:
765 i915_gem_object_put(obj);
766 return ERR_PTR(err);
767}
768#else /* !__linux__ */
769struct drm_i915_gem_object *
770i915_gem_object_create_shmem_from_data(struct drm_i915_privateinteldrm_softc *dev_priv,
771 const void *data, resource_size_t size)
772{
773 struct drm_i915_gem_object *obj;
774 struct uvm_object *uao;
775 resource_size_t offset;
776 int err;
777
778 GEM_WARN_ON(IS_DGFX(dev_priv))({ __builtin_expect(!!(!!(((&(dev_priv)->__info)->is_dgfx
))), 0); });
779 obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE)((((size) + (((1 << 12)) - 1)) / ((1 << 12))) * (
(1 << 12))));
780 if (IS_ERR(obj))
781 return obj;
782
783 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU)((void)0);
784
785 uao = obj->base.uao;
786 offset = 0;
787 do {
788 unsigned int len = min_t(typeof(size), size, PAGE_SIZE)({ typeof(size) __min_a = (size); typeof(size) __min_b = ((1 <<
12)); __min_a < __min_b ? __min_a : __min_b; });
789 struct vm_page *page;
790 void *pgdata, *vaddr;
791 struct pglist plist;
792
793 TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0);
794 if (uvm_obj_wire(uao, trunc_page(offset)((offset) & ~((1 << 12) - 1)),
795 trunc_page(offset)((offset) & ~((1 << 12) - 1)) + PAGE_SIZE(1 << 12), &plist)) {
796 err = -ENOMEM12;
797 goto fail;
798 }
799 page = TAILQ_FIRST(&plist)((&plist)->tqh_first);
800
801 vaddr = kmap(page);
802 memcpy(vaddr, data, len)__builtin_memcpy((vaddr), (data), (len));
803 kunmap_va(vaddr);
804
805 uvm_obj_unwire(uao, trunc_page(offset)((offset) & ~((1 << 12) - 1)),
806 trunc_page(offset)((offset) & ~((1 << 12) - 1)) + PAGE_SIZE(1 << 12));
807
808 size -= len;
809 data += len;
810 offset += len;
811 } while (size);
812
813 return obj;
814
815fail:
816 i915_gem_object_put(obj);
817 return ERR_PTR(err);
818}
819#endif
820
821static int init_shmem(struct intel_memory_region *mem)
822{
823 i915_gemfs_init(mem->i915);
824 intel_memory_region_set_name(mem, "system");
825
826 return 0; /* We have fallback to the kernel mnt if gemfs init failed. */
827}
828
829static int release_shmem(struct intel_memory_region *mem)
830{
831 i915_gemfs_fini(mem->i915);
832 return 0;
833}
834
835static const struct intel_memory_region_ops shmem_region_ops = {
836 .init = init_shmem,
837 .release = release_shmem,
838 .init_object = shmem_object_init,
839};
840
841struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_privateinteldrm_softc *i915,
842 u16 type, u16 instance)
843{
844 return intel_memory_region_create(i915, 0,
845 totalram_pages() << PAGE_SHIFT12,
846 PAGE_SIZE(1 << 12), 0, 0,
847 type, instance,
848 &shmem_region_ops);
849}
850
851bool_Bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
852{
853 return obj->ops == &i915_gem_shmem_ops;
854}