Bug Summary

File:dev/pci/drm/i915/gem/i915_gem_pages.c
Warning:line 61, column 37
The result of the left shift is undefined due to shifting by '32', which is greater or equal to the width of type 'unsigned int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name i915_gem_pages.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/gem/i915_gem_pages.c

/usr/src/sys/dev/pci/drm/i915/gem/i915_gem_pages.c

→
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7#include <drm/drm_cache.h>
8
9#include "gt/intel_gt.h"
10#include "gt/intel_gt_pm.h"
11
12#include "i915_drv.h"
13#include "i915_gem_object.h"
14#include "i915_scatterlist.h"
15#include "i915_gem_lmem.h"
16#include "i915_gem_mman.h"
17
18void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
19 struct sg_table *pages,
20 unsigned int sg_page_sizes)
21{
22 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
23 unsigned long supported = RUNTIME_INFO(i915)(&(i915)->__runtime)->page_sizes;
24 bool_Bool shrinkable;
25 int i;
26
27 assert_object_held_shared(obj);
28
29 if (i915_gem_object_is_volatile(obj))
1
Assuming the condition is false
→
2
←
Taking false branch
→
30 obj->mm.madv = I915_MADV_DONTNEED1;
31
32 /* Make the pages coherent with the GPU (flushing any swapin). */
33 if (obj->cache_dirty) {
3
←
Assuming field 'cache_dirty' is 0
→
4
←
Taking false branch
→
34 WARN_ON_ONCE(IS_DGFX(i915))({ static int __warned; int __ret = !!(((&(i915)->__info
)->is_dgfx)); if (__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "((&(i915)->__info)->is_dgfx)", "/usr/src/sys/dev/pci/drm/i915/gem/i915_gem_pages.c"
, 34); __warned = 1; } __builtin_expect(!!(__ret), 0); });
35 obj->write_domain = 0;
36 if (i915_gem_object_has_struct_page(obj))
37 drm_clflush_sg(pages);
38 obj->cache_dirty = false0;
39 }
40
41 obj->mm.get_page.sg_pos = pages->sgl;
42 obj->mm.get_page.sg_idx = 0;
43 obj->mm.get_dma_page.sg_pos = pages->sgl;
44 obj->mm.get_dma_page.sg_idx = 0;
45
46 obj->mm.pages = pages;
47
48 GEM_BUG_ON(!sg_page_sizes)((void)0);
49 obj->mm.page_sizes.phys = sg_page_sizes;
50
51 /*
52 * Calculate the supported page-sizes which fit into the given
53 * sg_page_sizes. This will give us the page-sizes which we may be able
54 * to use opportunistically when later inserting into the GTT. For
55 * example if phys=2G, then in theory we should be able to use 1G, 2M,
56 * 64K or 4K pages, although in practice this will depend on a number of
57 * other factors.
58 */
59 obj->mm.page_sizes.sg = 0;
60 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)for ((i) = find_first_bit((&supported), (((sizeof((1ULL <<
(21))) <= 4) ? (fls((1ULL << (21))) - 1) : (flsl((1ULL
<< (21))) - 1)) + 1)); (i) < (((sizeof((1ULL <<
(21))) <= 4) ? (fls((1ULL << (21))) - 1) : (flsl((1ULL
<< (21))) - 1)) + 1); (i) = find_next_bit((&supported
), (((sizeof((1ULL << (21))) <= 4) ? (fls((1ULL <<
(21))) - 1) : (flsl((1ULL << (21))) - 1)) + 1), (i) + 1
)) {
5
←
'?' condition is false
→
6
←
Calling 'find_first_bit'
→
20
←
Returning from 'find_first_bit'
→
21
←
The value 32 is assigned to 'i'
→
22
←
'?' condition is false
→
23
←
Assuming the condition is true
→
24
←
Loop condition is true. Entering loop body
→
61 if (obj->mm.page_sizes.phys & ~0u << i)
25
←
The result of the left shift is undefined due to shifting by '32', which is greater or equal to the width of type 'unsigned int'
62 obj->mm.page_sizes.sg |= BIT(i)(1UL << (i));
63 }
64 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg))((void)0);
65
66 shrinkable = i915_gem_object_is_shrinkable(obj);
67
68 if (i915_gem_object_is_tiled(obj) &&
69 i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES(1UL << (0))) {
70 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj))((void)0);
71 i915_gem_object_set_tiling_quirk(obj);
72 GEM_BUG_ON(!list_empty(&obj->mm.link))((void)0);
73 atomic_inc(&obj->mm.shrink_pin)__sync_fetch_and_add(&obj->mm.shrink_pin, 1);
74 shrinkable = false0;
75 }
76
77 if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) {
78 struct list_head *list;
79 unsigned long flags;
80
81 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
82 spin_lock_irqsave(&i915->mm.obj_lock, flags)do { flags = 0; mtx_enter(&i915->mm.obj_lock); } while
(0);
83
84 i915->mm.shrink_count++;
85 i915->mm.shrink_memory += obj->base.size;
86
87 if (obj->mm.madv != I915_MADV_WILLNEED0)
88 list = &i915->mm.purge_list;
89 else
90 list = &i915->mm.shrink_list;
91 list_add_tail(&obj->mm.link, list);
92
93 atomic_set(&obj->mm.shrink_pin, 0)({ typeof(*(&obj->mm.shrink_pin)) __tmp = ((0)); *(volatile
typeof(*(&obj->mm.shrink_pin)) *)&(*(&obj->
mm.shrink_pin)) = __tmp; __tmp; });
94 spin_unlock_irqrestore(&i915->mm.obj_lock, flags)do { (void)(flags); mtx_leave(&i915->mm.obj_lock); } while
(0);
95 }
96}
97
98int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
99{
100 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
101 int err;
102
103 assert_object_held_shared(obj);
104
105 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)__builtin_expect(!!(obj->mm.madv != 0), 0)) {
106 drm_dbg(&i915->drm,__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Attempting to obtain a purgeable object\n"
)
107 "Attempting to obtain a purgeable object\n")__drm_dev_dbg(((void *)0), (&i915->drm) ? (&i915->
drm)->dev : ((void *)0), DRM_UT_DRIVER, "Attempting to obtain a purgeable object\n"
);
108 return -EFAULT14;
109 }
110
111 err = obj->ops->get_pages(obj);
112 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj))((void)0);
113
114 return err;
115}
116
117/* Ensure that the associated pages are gathered from the backing storage
118 * and pinned into our object. i915_gem_object_pin_pages() may be called
119 * multiple times before they are released by a single call to
120 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
121 * either as a result of memory pressure (reaping pages under the shrinker)
122 * or as the object is itself released.
123 */
124int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
125{
126 int err;
127
128 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
129
130 assert_object_held_shared(obj);
131
132 if (unlikely(!i915_gem_object_has_pages(obj))__builtin_expect(!!(!i915_gem_object_has_pages(obj)), 0)) {
133 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj))((void)0);
134
135 err = ____i915_gem_object_get_pages(obj);
136 if (err)
137 return err;
138
139 smp_mb__before_atomic()do { } while (0);
140 }
141 atomic_inc(&obj->mm.pages_pin_count)__sync_fetch_and_add(&obj->mm.pages_pin_count, 1);
142
143 return 0;
144}
145
146int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
147{
148 struct i915_gem_ww_ctx ww;
149 int err;
150
151 i915_gem_ww_ctx_init(&ww, true1);
152retry:
153 err = i915_gem_object_lock(obj, &ww);
154 if (!err)
155 err = i915_gem_object_pin_pages(obj);
156
157 if (err == -EDEADLK11) {
158 err = i915_gem_ww_ctx_backoff(&ww);
159 if (!err)
160 goto retry;
161 }
162 i915_gem_ww_ctx_fini(&ww);
163 return err;
164}
165
166/* Immediately discard the backing storage */
167int i915_gem_object_truncate(struct drm_i915_gem_object *obj)
168{
169 if (obj->ops->truncate)
170 return obj->ops->truncate(obj);
171
172 return 0;
173}
174
175static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
176{
177 struct radix_tree_iter iter;
178 void __rcu **slot;
179
180 rcu_read_lock();
181 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)for ((&iter)->index = (0); radix_tree_iter_find(&obj
->mm.get_page.radix, &iter, &(slot)); (&iter)->
index++)
182 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
183 radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)for ((&iter)->index = (0); radix_tree_iter_find(&obj
->mm.get_dma_page.radix, &iter, &(slot)); (&iter
)->index++)
184 radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
185 rcu_read_unlock();
186}
187
188static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
189{
190 if (is_vmalloc_addr(ptr))
191 vunmap(ptr, obj->base.size);
192}
193
194static void flush_tlb_invalidate(struct drm_i915_gem_object *obj)
195{
196 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
197 struct intel_gt *gt = to_gt(i915);
198
199 if (!obj->mm.tlb)
200 return;
201
202 intel_gt_invalidate_tlb(gt, obj->mm.tlb);
203 obj->mm.tlb = 0;
204}
205
206struct sg_table *
207__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
208{
209 struct sg_table *pages;
210
211 assert_object_held_shared(obj);
212
213 pages = fetch_and_zero(&obj->mm.pages)({ typeof(*&obj->mm.pages) __T = *(&obj->mm.pages
); *(&obj->mm.pages) = (typeof(*&obj->mm.pages)
)0; __T; });
214 if (IS_ERR_OR_NULL(pages))
215 return pages;
216
217 if (i915_gem_object_is_volatile(obj))
218 obj->mm.madv = I915_MADV_WILLNEED0;
219
220 if (!i915_gem_object_has_self_managed_shrink_list(obj))
221 i915_gem_object_make_unshrinkable(obj);
222
223 if (obj->mm.mapping) {
224 unmap_object(obj, page_mask_bits(obj->mm.mapping)({ unsigned long __v = (unsigned long)(obj->mm.mapping); (
typeof(obj->mm.mapping))(__v & -(1UL << (12))); }
));
225 obj->mm.mapping = NULL((void *)0);
226 }
227
228 __i915_gem_object_reset_page_iter(obj);
229 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
230
231 flush_tlb_invalidate(obj);
232
233 return pages;
234}
235
236int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
237{
238 struct sg_table *pages;
239
240 if (i915_gem_object_has_pinned_pages(obj))
241 return -EBUSY16;
242
243 /* May be called by shrinker from within get_pages() (on another bo) */
244 assert_object_held_shared(obj);
245
246 i915_gem_object_release_mmap_offset(obj);
247
248 /*
249 * ->put_pages might need to allocate memory for the bit17 swizzle
250 * array, hence protect them from being reaped by removing them from gtt
251 * lists early.
252 */
253 pages = __i915_gem_object_unset_pages(obj);
254
255 /*
256 * XXX Temporary hijinx to avoid updating all backends to handle
257 * NULL pages. In the future, when we have more asynchronous
258 * get_pages backends we should be better able to handle the
259 * cancellation of the async task in a more uniform manner.
260 */
261 if (!IS_ERR_OR_NULL(pages))
262 obj->ops->put_pages(obj, pages);
263
264 return 0;
265}
266
267/* The 'mapping' part of i915_gem_object_pin_map() below */
268static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
269 enum i915_map_type type)
270{
271 unsigned long n_pages = obj->base.size >> PAGE_SHIFT12, i;
272 struct vm_page *stack[32], **pages = stack, *page;
273 struct sgt_iter iter;
274 pgprot_t pgprot;
275 void *vaddr;
276
277 switch (type) {
278 default:
279 MISSING_CASE(type)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "type", (long)(type)); __builtin_expect(!!(__ret), 0); });
280 fallthroughdo {} while (0); /* to use PAGE_KERNEL anyway */
281 case I915_MAP_WB:
282 /*
283 * On 32b, highmem using a finite set of indirect PTE (i.e.
284 * vmap) to provide virtual mappings of the high pages.
285 * As these are finite, map_new_virtual() must wait for some
286 * other kmap() to finish when it runs out. If we map a large
287 * number of objects, there is no method for it to tell us
288 * to release the mappings, and we deadlock.
289 *
290 * However, if we make an explicit vmap of the page, that
291 * uses a larger vmalloc arena, and also has the ability
292 * to tell us to release unwanted mappings. Most importantly,
293 * it will fail and propagate an error instead of waiting
294 * forever.
295 *
296 * So if the page is beyond the 32b boundary, make an explicit
297 * vmap.
298 */
299#ifdef notyet
300 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))0)
301 return page_address(sg_page(obj->mm.pages->sgl));
302#endif
303 pgprot = PAGE_KERNEL0;
304 break;
305 case I915_MAP_WC:
306 pgprot = pgprot_writecombine(PAGE_KERNEL_IO0);
307 break;
308 }
309
310 if (n_pages > ARRAY_SIZE(stack)(sizeof((stack)) / sizeof((stack)[0]))) {
311 /* Too big for stack -- allocate temporary array instead */
312 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL(0x0001 | 0x0004));
313 if (!pages)
314 return ERR_PTR(-ENOMEM12);
315 }
316
317 i = 0;
318 for_each_sgt_page(page, iter, obj->mm.pages)for ((iter) = __sgt_iter((obj->mm.pages)->sgl, 0); ((page
) = (iter).pfn == 0 ? ((void *)0) : (PHYS_TO_VM_PAGE(((paddr_t
)((iter).pfn + ((iter).curr >> 12)) << 12)))); ((
(iter).curr += (1 << 12)) >= (iter).max) ? (iter) = __sgt_iter
(__sg_next((iter).sgp), 0), 0 : 0)
319 pages[i++] = page;
320 vaddr = vmap(pages, n_pages, 0, pgprot);
321 if (pages != stack)
322 kvfree(pages);
323
324 return vaddr ?: ERR_PTR(-ENOMEM12);
325}
326
327static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
328 enum i915_map_type type)
329{
330 STUB()do { printf("%s: stub\n", __func__); } while(0);
331 return NULL((void *)0);
332#ifdef notyet
333 resource_size_t iomap = obj->mm.region->iomap.base -
334 obj->mm.region->region.start;
335 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT12;
336 unsigned long stack[32], *pfns = stack, i;
337 struct sgt_iter iter;
338 dma_addr_t addr;
339 void *vaddr;
340
341 GEM_BUG_ON(type != I915_MAP_WC)((void)0);
342
343 if (n_pfn > ARRAY_SIZE(stack)(sizeof((stack)) / sizeof((stack)[0]))) {
344 /* Too big for stack -- allocate temporary array instead */
345 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL(0x0001 | 0x0004));
346 if (!pfns)
347 return ERR_PTR(-ENOMEM12);
348 }
349
350 i = 0;
351 for_each_sgt_daddr(addr, iter, obj->mm.pages)for ((iter) = __sgt_iter((obj->mm.pages)->sgl, 1); ((addr
) = (iter).dma + (iter).curr), (iter).sgp; (((iter).curr += (
(1ULL << (12)))) >= (iter).max) ? (iter) = __sgt_iter
(__sg_next((iter).sgp), 1), 0 : 0)
352 pfns[i++] = (iomap + addr) >> PAGE_SHIFT12;
353 vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO0));
354 if (pfns != stack)
355 kvfree(pfns);
356
357 return vaddr ?: ERR_PTR(-ENOMEM12);
358#endif
359}
360
361static void *i915_gem_object_map(struct drm_i915_gem_object *obj,
362 enum i915_map_type type)
363{
364 unsigned long n_pages = obj->base.size >> PAGE_SHIFT12;
365 struct sg_table *sgt = obj->mm.pages;
366 struct vm_page *stack_pages[32];
367 struct vm_page **pages = stack_pages;
368 struct vm_struct *area;
369 pgprot_t pgprot;
370 void *addr;
371
372 if (!i915_gem_object_has_struct_page(obj) && type != I915_MAP_WC)
373 return NULL((void *)0);
374
375#if 0
376 /* A single page can always be kmapped */
377 if (n_pages == 1 && type == I915_MAP_WB)
378 return kmap(sg_page(sgt->sgl));
379#endif
380
381 if (n_pages > ARRAY_SIZE(stack_pages)(sizeof((stack_pages)) / sizeof((stack_pages)[0]))) {
382 /* Too big for stack -- allocate temporary array instead */
383 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL(0x0001 | 0x0004));
384 if (!pages)
385 return NULL((void *)0);
386 }
387
388 switch (type) {
389 default:
390 MISSING_CASE(type)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "type", (long)(type)); __builtin_expect(!!(__ret), 0); });
391 /* fallthrough - to use PAGE_KERNEL anyway */
392 case I915_MAP_WB:
393 pgprot = PAGE_KERNEL0;
394 break;
395 case I915_MAP_WC:
396 pgprot = pgprot_writecombine(PAGE_KERNEL_IO0);
397 break;
398 }
399
400 if (i915_gem_object_has_struct_page(obj)) {
401 struct sgt_iter iter;
402 struct vm_page *page;
403 unsigned long i = 0;
404
405 for_each_sgt_page(page, iter, sgt)for ((iter) = __sgt_iter((sgt)->sgl, 0); ((page) = (iter).
pfn == 0 ? ((void *)0) : (PHYS_TO_VM_PAGE(((paddr_t)((iter).pfn
+ ((iter).curr >> 12)) << 12)))); (((iter).curr +=
(1 << 12)) >= (iter).max) ? (iter) = __sgt_iter(__sg_next
((iter).sgp), 0), 0 : 0)
406 pages[i++] = page;
407 } else {
408 STUB()do { printf("%s: stub\n", __func__); } while(0);
409#ifdef notyet
410 resource_size_t iomap;
411 struct sgt_iter iter;
412 pte_t **ptes = mem;
413 dma_addr_t addr;
414
415 iomap = obj->mm.region->iomap.base;
416 iomap -= obj->mm.region->region.start;
417
418 for_each_sgt_daddr(addr, iter, sgt)for ((iter) = __sgt_iter((sgt)->sgl, 1); ((addr) = (iter).
dma + (iter).curr), (iter).sgp; (((iter).curr += ((1ULL <<
(12)))) >= (iter).max) ? (iter) = __sgt_iter(__sg_next((iter
).sgp), 1), 0 : 0)
419 **ptes++ = iomap_pte(iomap, addr, pgprot);
420#endif
421 }
422 addr = vmap(pages, n_pages, 0, pgprot);
423
424 if (pages != stack_pages)
425 kvfree(pages);
426
427 return addr;
428}
429
430/* get, pin, and map the pages of the object into kernel space */
431void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
432 enum i915_map_type type)
433{
434 enum i915_map_type has_type;
435 bool_Bool pinned;
436 void *ptr;
437 int err;
438
439 if (!i915_gem_object_has_struct_page(obj) &&
440 !i915_gem_object_has_iomem(obj))
441 return ERR_PTR(-ENXIO6);
442
443 if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY)({ static int __warned; int __ret = !!(obj->flags & (1UL
<< (6))); if (__ret && !__warned) { printf("WARNING %s failed at %s:%d\n"
, "obj->flags & (1UL << (6))", "/usr/src/sys/dev/pci/drm/i915/gem/i915_gem_pages.c"
, 443); __warned = 1; } __builtin_expect(!!(__ret), 0); }))
444 return ERR_PTR(-EINVAL22);
445
446 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
447
448 pinned = !(type & I915_MAP_OVERRIDE(1UL << (31)));
449 type &= ~I915_MAP_OVERRIDE(1UL << (31));
450
451 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)atomic_add_unless((&obj->mm.pages_pin_count), 1, 0)) {
452 if (unlikely(!i915_gem_object_has_pages(obj))__builtin_expect(!!(!i915_gem_object_has_pages(obj)), 0)) {
453 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj))((void)0);
454
455 err = ____i915_gem_object_get_pages(obj);
456 if (err)
457 return ERR_PTR(err);
458
459 smp_mb__before_atomic()do { } while (0);
460 }
461 atomic_inc(&obj->mm.pages_pin_count)__sync_fetch_and_add(&obj->mm.pages_pin_count, 1);
462 pinned = false0;
463 }
464 GEM_BUG_ON(!i915_gem_object_has_pages(obj))((void)0);
465
466 /*
467 * For discrete our CPU mappings needs to be consistent in order to
468 * function correctly on !x86. When mapping things through TTM, we use
469 * the same rules to determine the caching type.
470 *
471 * The caching rules, starting from DG1:
472 *
473 * - If the object can be placed in device local-memory, then the
474 * pages should be allocated and mapped as write-combined only.
475 *
476 * - Everything else is always allocated and mapped as write-back,
477 * with the guarantee that everything is also coherent with the
478 * GPU.
479 *
480 * Internal users of lmem are already expected to get this right, so no
481 * fudging needed there.
482 */
483 if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) {
484 if (type != I915_MAP_WC && !obj->mm.n_placements) {
485 ptr = ERR_PTR(-ENODEV19);
486 goto err_unpin;
487 }
488
489 type = I915_MAP_WC;
490 } else if (IS_DGFX(to_i915(obj->base.dev))((&(to_i915(obj->base.dev))->__info)->is_dgfx)) {
491 type = I915_MAP_WB;
492 }
493
494 ptr = page_unpack_bits(obj->mm.mapping, &has_type)({ unsigned long __v = (unsigned long)(obj->mm.mapping); *
(&has_type) = __v & ((1UL << (12)) - 1); (typeof
(obj->mm.mapping))(__v & -(1UL << (12))); });
495 if (ptr && has_type != type) {
496 if (pinned) {
497 ptr = ERR_PTR(-EBUSY16);
498 goto err_unpin;
499 }
500
501 unmap_object(obj, ptr);
502
503 ptr = obj->mm.mapping = NULL((void *)0);
504 }
505
506 if (!ptr) {
507 err = i915_gem_object_wait_moving_fence(obj, true1);
508 if (err) {
509 ptr = ERR_PTR(err);
510 goto err_unpin;
511 }
512
513 if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled())({ __builtin_expect(!!(!!(type == I915_MAP_WC && !pat_enabled
())), 0); }))
514 ptr = ERR_PTR(-ENODEV19);
515 else if (i915_gem_object_has_struct_page(obj))
516 ptr = i915_gem_object_map_page(obj, type);
517 else
518 ptr = i915_gem_object_map_pfn(obj, type);
519 if (IS_ERR(ptr))
520 goto err_unpin;
521
522 obj->mm.mapping = page_pack_bits(ptr, type)({ unsigned long __bits = (type); ((void)0); ((typeof(ptr))((
unsigned long)(ptr) | __bits)); });
523 }
524
525 return ptr;
526
527err_unpin:
528 atomic_dec(&obj->mm.pages_pin_count)__sync_fetch_and_sub(&obj->mm.pages_pin_count, 1);
529 return ptr;
530}
531
532void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
533 enum i915_map_type type)
534{
535 void *ret;
536
537 i915_gem_object_lock(obj, NULL((void *)0));
538 ret = i915_gem_object_pin_map(obj, type);
539 i915_gem_object_unlock(obj);
540
541 return ret;
542}
543
544void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
545 unsigned long offset,
546 unsigned long size)
547{
548 enum i915_map_type has_type;
549 void *ptr;
550
551 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj))((void)0);
552 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),((void)0)
553 offset, size, obj->base.size))((void)0);
554
555 wmb()do { __asm volatile("sfence" ::: "memory"); } while (0); /* let all previous writes be visible to coherent partners */
556 obj->mm.dirty = true1;
557
558 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE(1UL << (1)))
559 return;
560
561 ptr = page_unpack_bits(obj->mm.mapping, &has_type)({ unsigned long __v = (unsigned long)(obj->mm.mapping); *
(&has_type) = __v & ((1UL << (12)) - 1); (typeof
(obj->mm.mapping))(__v & -(1UL << (12))); });
562 if (has_type == I915_MAP_WC)
563 return;
564
565 drm_clflush_virt_range(ptr + offset, size);
566 if (size == obj->base.size) {
567 obj->write_domain &= ~I915_GEM_DOMAIN_CPU0x00000001;
568 obj->cache_dirty = false0;
569 }
570}
571
572void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
573{
574 GEM_BUG_ON(!obj->mm.mapping)((void)0);
575
576 /*
577 * We allow removing the mapping from underneath pinned pages!
578 *
579 * Furthermore, since this is an unsafe operation reserved only
580 * for construction time manipulation, we ignore locking prudence.
581 */
582 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))({ unsigned long __v = (unsigned long)(({ typeof(*&obj->
mm.mapping) __T = *(&obj->mm.mapping); *(&obj->
mm.mapping) = (typeof(*&obj->mm.mapping))0; __T; })); (
typeof(({ typeof(*&obj->mm.mapping) __T = *(&obj->
mm.mapping); *(&obj->mm.mapping) = (typeof(*&obj->
mm.mapping))0; __T; })))(__v & -(1UL << (12))); }));
583
584 i915_gem_object_unpin_map(obj);
585}
586
587struct scatterlist *
588__i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
589 struct i915_gem_object_page_iter *iter,
590 unsigned int n,
591 unsigned int *offset,
592 bool_Bool dma)
593{
594 struct scatterlist *sg;
595 unsigned int idx, count;
596
597 might_sleep()assertwaitok();
598 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT)((void)0);
599 if (!i915_gem_object_has_pinned_pages(obj))
600 assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);
601
602 /* As we iterate forward through the sg, we record each entry in a
603 * radixtree for quick repeated (backwards) lookups. If we have seen
604 * this index previously, we will have an entry for it.
605 *
606 * Initial lookup is O(N), but this is amortized to O(1) for
607 * sequential page access (where each new request is consecutive
608 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
609 * i.e. O(1) with a large constant!
610 */
611 if (n < READ_ONCE(iter->sg_idx)({ typeof(iter->sg_idx) __tmp = *(volatile typeof(iter->
sg_idx) *)&(iter->sg_idx); membar_datadep_consumer(); __tmp
; }))
612 goto lookup;
613
614 mutex_lock(&iter->lock)rw_enter_write(&iter->lock);
615
616 /* We prefer to reuse the last sg so that repeated lookup of this
617 * (or the subsequent) sg are fast - comparing against the last
618 * sg is faster than going through the radixtree.
619 */
620
621 sg = iter->sg_pos;
622 idx = iter->sg_idx;
623 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
624
625 while (idx + count <= n) {
626 void *entry;
627 unsigned long i;
628 int ret;
629
630 /* If we cannot allocate and insert this entry, or the
631 * individual pages from this range, cancel updating the
632 * sg_idx so that on this lookup we are forced to linearly
633 * scan onwards, but on future lookups we will try the
634 * insertion again (in which case we need to be careful of
635 * the error return reporting that we have already inserted
636 * this index).
637 */
638 ret = radix_tree_insert(&iter->radix, idx, sg);
639 if (ret && ret != -EEXIST17)
640 goto scan;
641
642 entry = xa_mk_value(idx);
643 for (i = 1; i < count; i++) {
644 ret = radix_tree_insert(&iter->radix, idx + i, entry);
645 if (ret && ret != -EEXIST17)
646 goto scan;
647 }
648
649 idx += count;
650 sg = ____sg_next(sg);
651 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
652 }
653
654scan:
655 iter->sg_pos = sg;
656 iter->sg_idx = idx;
657
658 mutex_unlock(&iter->lock)rw_exit_write(&iter->lock);
659
660 if (unlikely(n < idx)__builtin_expect(!!(n < idx), 0)) /* insertion completed by another thread */
661 goto lookup;
662
663 /* In case we failed to insert the entry into the radixtree, we need
664 * to look beyond the current sg.
665 */
666 while (idx + count <= n) {
667 idx += count;
668 sg = ____sg_next(sg);
669 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
670 }
671
672 *offset = n - idx;
673 return sg;
674
675lookup:
676 rcu_read_lock();
677
678 sg = radix_tree_lookup(&iter->radix, n);
679 GEM_BUG_ON(!sg)((void)0);
680
681 /* If this index is in the middle of multi-page sg entry,
682 * the radix tree will contain a value entry that points
683 * to the start of that range. We will return the pointer to
684 * the base page and the offset of this page within the
685 * sg entry's range.
686 */
687 *offset = 0;
688 if (unlikely(xa_is_value(sg))__builtin_expect(!!(xa_is_value(sg)), 0)) {
689 unsigned long base = xa_to_value(sg);
690
691 sg = radix_tree_lookup(&iter->radix, base);
692 GEM_BUG_ON(!sg)((void)0);
693
694 *offset = n - base;
695 }
696
697 rcu_read_unlock();
698
699 return sg;
700}
701
702struct vm_page *
703i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
704{
705 struct scatterlist *sg;
706 unsigned int offset;
707
708 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj))((void)0);
709
710 sg = i915_gem_object_get_sg(obj, n, &offset);
711 return nth_page(sg_page(sg), offset)(&(sg_page(sg))[(offset)]);
712}
713
714/* Like i915_gem_object_get_page(), but mark the returned page dirty */
715struct vm_page *
716i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
717 unsigned int n)
718{
719 struct vm_page *page;
720
721 page = i915_gem_object_get_page(obj, n);
722 if (!obj->mm.dirty)
723 set_page_dirty(page)x86_atomic_clearbits_u32(&page->pg_flags, 0x00000008);
724
725 return page;
726}
727
728dma_addr_t
729i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
730 unsigned long n,
731 unsigned int *len)
732{
733 struct scatterlist *sg;
734 unsigned int offset;
735
736 sg = i915_gem_object_get_sg_dma(obj, n, &offset);
737
738 if (len)
739 *len = sg_dma_len(sg)((sg)->length) - (offset << PAGE_SHIFT12);
740
741 return sg_dma_address(sg)((sg)->dma_address) + (offset << PAGE_SHIFT12);
742}
743
744dma_addr_t
745i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
746 unsigned long n)
747{
748 return i915_gem_object_get_dma_address_len(obj, n, NULL((void *)0));
749}
←

/usr/src/sys/dev/pci/drm/include/linux/atomic.h

1/* $OpenBSD: atomic.h,v 1.22 2024/01/06 12:52:20 jsg Exp $ */
2/**
3 * \file drm_atomic.h
4 * Atomic operations used in the DRM which may or may not be provided by the OS.
5 *
6 * \author Eric Anholt <anholt@FreeBSD.org>
7 */
8
9/*-
10 * Copyright 2004 Eric Anholt
11 * All Rights Reserved.
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
19 *
20 * The above copyright notice and this permission notice (including the next
21 * paragraph) shall be included in all copies or substantial portions of the
22 * Software.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
28 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
29 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
30 * OTHER DEALINGS IN THE SOFTWARE.
31 */
32
33#ifndef _DRM_LINUX_ATOMIC_H_
34#define _DRM_LINUX_ATOMIC_H_
35
36#include <sys/types.h>
37#include <sys/mutex.h>
38#include <machine/intr.h>
39#include <linux/types.h>
40#include <linux/compiler.h> /* via x86/include/asm/atomic.h */
41
42#define ATOMIC_INIT(x)(x) (x)
43
44#define atomic_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) WRITE_ONCE(*(p), (v))({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
45#define atomic_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; }) READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
46#define atomic_inc(p)__sync_fetch_and_add(p, 1) __sync_fetch_and_add(p, 1)
47#define atomic_dec(p)__sync_fetch_and_sub(p, 1) __sync_fetch_and_sub(p, 1)
48#define atomic_add(n, p)__sync_fetch_and_add(p, n) __sync_fetch_and_add(p, n)
49#define atomic_sub(n, p)__sync_fetch_and_sub(p, n) __sync_fetch_and_sub(p, n)
50#define atomic_and(n, p)__sync_fetch_and_and(p, n) __sync_fetch_and_and(p, n)
51#define atomic_or(n, p)x86_atomic_setbits_u32(p, n) atomic_setbits_intx86_atomic_setbits_u32(p, n)
52#define atomic_add_return(n, p)__sync_add_and_fetch(p, n) __sync_add_and_fetch(p, n)
53#define atomic_sub_return(n, p)__sync_sub_and_fetch(p, n) __sync_sub_and_fetch(p, n)
54#define atomic_sub_and_test(n, p)(__sync_sub_and_fetch(p, n) == 0) (atomic_sub_return(n, p)__sync_sub_and_fetch(p, n) == 0)
55#define atomic_inc_return(v)__sync_add_and_fetch((v), 1) atomic_add_return(1, (v))__sync_add_and_fetch((v), 1)
56#define atomic_dec_return(v)__sync_sub_and_fetch((v), 1) atomic_sub_return(1, (v))__sync_sub_and_fetch((v), 1)
57#define atomic_dec_and_test(v)(__sync_sub_and_fetch((v), 1) == 0) (atomic_dec_return(v)__sync_sub_and_fetch((v), 1) == 0)
58#define atomic_inc_and_test(v)(__sync_add_and_fetch((v), 1) == 0) (atomic_inc_return(v)__sync_add_and_fetch((v), 1) == 0)
59#define atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) __sync_val_compare_and_swap(p, o, n)
60#define cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) __sync_val_compare_and_swap(p, o, n)
61#define cmpxchg64(p, o, n)__sync_val_compare_and_swap(p, o, n) __sync_val_compare_and_swap(p, o, n)
62#define atomic_set_release(p, v)({ typeof(*((p))) __tmp = (((v))); *(volatile typeof(*((p))) *
)&(*((p))) = __tmp; __tmp; }) atomic_set((p), (v))({ typeof(*((p))) __tmp = (((v))); *(volatile typeof(*((p))) *
)&(*((p))) = __tmp; __tmp; })
63#define atomic_andnot(bits, p)x86_atomic_clearbits_u32(p,bits) atomic_clearbits_intx86_atomic_clearbits_u32(p,bits)
64#define atomic_fetch_inc(p)__sync_fetch_and_add(p, 1) __sync_fetch_and_add(p, 1)
65#define atomic_fetch_xor(n, p)__sync_fetch_and_xor(p, n) __sync_fetch_and_xor(p, n)
66
67#define try_cmpxchg(p, op, n)({ __typeof(p) __op = (__typeof((p)))(op); __typeof(*(p)) __o
= *__op; __typeof(*(p)) __p = __sync_val_compare_and_swap((p
), (__o), (n)); if (__p != __o) *__op = __p; (__p == __o); }) \
68({ \
69 __typeof(p) __op = (__typeof((p)))(op); \
70 __typeof(*(p)) __o = *__op; \
71 __typeof(*(p)) __p = __sync_val_compare_and_swap((p), (__o), (n)); \
72 if (__p != __o) \
73 *__op = __p; \
74 (__p == __o); \
75})
76
77static inline bool_Bool
78atomic_try_cmpxchg(volatile int *p, int *op, int n)
79{
80 return try_cmpxchg(p, op, n)({ __typeof(p) __op = (__typeof((p)))(op); __typeof(*(p)) __o
= *__op; __typeof(*(p)) __p = __sync_val_compare_and_swap((p
), (__o), (n)); if (__p != __o) *__op = __p; (__p == __o); });
81}
82
83static inline int
84atomic_xchg(volatile int *v, int n)
85{
86 __sync_synchronize();
87 return __sync_lock_test_and_set(v, n);
88}
89
90#define xchg(v, n)__sync_lock_test_and_set(v, n) __sync_lock_test_and_set(v, n)
91
92static inline int
93atomic_add_unless(volatile int *v, int n, int u)
94{
95 int o;
96
97 do {
98 o = *v;
99 if (o == u)
100 return 0;
101 } while (__sync_val_compare_and_swap(v, o, o +n) != o);
102
103 return 1;
104}
105
106#define atomic_inc_not_zero(v)atomic_add_unless((v), 1, 0) atomic_add_unless((v), 1, 0)
107
108static inline int
109atomic_dec_if_positive(volatile int *v)
110{
111 int r, o;
112
113 do {
114 o = *v;
115 r = o - 1;
116 if (r < 0)
117 break;
118 } while (__sync_val_compare_and_swap(v, o, r) != o);
119
120 return r;
121}
122
123#define atomic_long_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; }) READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
124
125/* 32 bit powerpc lacks 64 bit atomics */
126#if !defined(__powerpc__) || defined(__powerpc64__)
127
128typedef int64_t atomic64_t;
129
130#define ATOMIC64_INIT(x)(x) (x)
131
132#define atomic64_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) WRITE_ONCE(*(p), (v))({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
133#define atomic64_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; }) READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
134
135static inline int64_t
136atomic64_xchg(volatile int64_t *v, int64_t n)
137{
138 __sync_synchronize();
139 return __sync_lock_test_and_set(v, n);
140}
141
142static inline int64_t
143atomic64_cmpxchg(volatile int64_t *v, int64_t o, int64_t n)
144{
145 return __sync_val_compare_and_swap(v, o, n);
146}
147
148#define atomic64_add(n, p)__sync_fetch_and_add_8(p, n) __sync_fetch_and_add_8(p, n)
149#define atomic64_sub(n, p)__sync_fetch_and_sub_8(p, n) __sync_fetch_and_sub_8(p, n)
150#define atomic64_inc(p)__sync_fetch_and_add_8(p, 1) __sync_fetch_and_add_8(p, 1)
151#define atomic64_add_return(n, p)__sync_add_and_fetch_8(p, n) __sync_add_and_fetch_8(p, n)
152#define atomic64_inc_return(p)__sync_add_and_fetch_8(p, 1) __sync_add_and_fetch_8(p, 1)
153
154#else
155
156extern struct mutex atomic64_mtx;
157
158typedef struct {
159 volatile int64_t val;
160} atomic64_t;
161
162#define ATOMIC64_INIT(x)(x) { (x) }
163
164static inline void
165atomic64_set(atomic64_t *v, int64_t i)({ typeof(*(atomic64_t *v)) __tmp = ((int64_t i)); *(volatile
typeof(*(atomic64_t *v)) *)&(*(atomic64_t *v)) = __tmp; __tmp
; })
166{
167 mtx_enter(&atomic64_mtx);
168 v->val = i;
169 mtx_leave(&atomic64_mtx);
170}
171
172static inline int64_t
173atomic64_read(atomic64_t *v)({ typeof(*(atomic64_t *v)) __tmp = *(volatile typeof(*(atomic64_t
*v)) *)&(*(atomic64_t *v)); membar_datadep_consumer(); __tmp
; })
174{
175 int64_t val;
176
177 mtx_enter(&atomic64_mtx);
178 val = v->val;
179 mtx_leave(&atomic64_mtx);
180
181 return val;
182}
183
184static inline int64_t
185atomic64_xchg(atomic64_t *v, int64_t n)
186{
187 int64_t val;
188
189 mtx_enter(&atomic64_mtx);
190 val = v->val;
191 v->val = n;
192 mtx_leave(&atomic64_mtx);
193
194 return val;
195}
196
197static inline void
198atomic64_add(int i, atomic64_t *v)__sync_fetch_and_add_8(atomic64_t *v, int i)
199{
200 mtx_enter(&atomic64_mtx);
201 v->val += i;
202 mtx_leave(&atomic64_mtx);
203}
204
205#define atomic64_inc(p)__sync_fetch_and_add_8(p, 1) atomic64_add(p, 1)__sync_fetch_and_add_8(1, p)
206
207static inline int64_t
208atomic64_add_return(int i, atomic64_t *v)__sync_add_and_fetch_8(atomic64_t *v, int i)
209{
210 int64_t val;
211
212 mtx_enter(&atomic64_mtx);
213 val = v->val + i;
214 v->val = val;
215 mtx_leave(&atomic64_mtx);
216
217 return val;
218}
219
220#define atomic64_inc_return(p)__sync_add_and_fetch_8(p, 1) atomic64_add_return(p, 1)__sync_add_and_fetch_8(1, p)
221
222static inline void
223atomic64_sub(int i, atomic64_t *v)__sync_fetch_and_sub_8(atomic64_t *v, int i)
224{
225 mtx_enter(&atomic64_mtx);
226 v->val -= i;
227 mtx_leave(&atomic64_mtx);
228}
229#endif
230
231#ifdef __LP64__1
232typedef int64_t atomic_long_t;
233#define atomic_long_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) atomic64_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
234#define atomic_long_xchg(v, n)atomic64_xchg(v, n) atomic64_xchg(v, n)
235#define atomic_long_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)
236#define atomic_long_add(i, v)__sync_fetch_and_add_8(v, i) atomic64_add(i, v)__sync_fetch_and_add_8(v, i)
237#define atomic_long_sub(i, v)__sync_fetch_and_sub_8(v, i) atomic64_sub(i, v)__sync_fetch_and_sub_8(v, i)
238#else
239typedef int32_t atomic_long_t;
240#define atomic_long_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) atomic_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
241#define atomic_long_xchg(v, n)atomic64_xchg(v, n) atomic_xchg(v, n)
242#define atomic_long_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)
243#define atomic_long_add(i, v)__sync_fetch_and_add_8(v, i) atomic_add(i, v)__sync_fetch_and_add(v, i)
244#define atomic_long_sub(i, v)__sync_fetch_and_sub_8(v, i) atomic_sub(i, v)__sync_fetch_and_sub(v, i)
245#endif
246
247static inline atomic_t
248test_and_set_bit(u_int b, volatile void *p)
249{
250 unsigned int m = 1 << (b & 0x1f);
251 unsigned int prev = __sync_fetch_and_or((volatile u_int *)p + (b >> 5), m);
252 return (prev & m) != 0;
253}
254
255static inline void
256clear_bit(u_int b, volatile void *p)
257{
258 atomic_clearbits_intx86_atomic_clearbits_u32(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
259}
260
261static inline void
262clear_bit_unlock(u_int b, volatile void *p)
263{
264 membar_enter()do { __asm volatile("mfence" ::: "memory"); } while (0);
265 clear_bit(b, p);
266}
267
268static inline void
269set_bit(u_int b, volatile void *p)
270{
271 atomic_setbits_intx86_atomic_setbits_u32(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
272}
273
274static inline void
275__clear_bit(u_int b, volatile void *p)
276{
277 volatile u_int *ptr = (volatile u_int *)p;
278 ptr[b >> 5] &= ~(1 << (b & 0x1f));
279}
280
281static inline void
282__set_bit(u_int b, volatile void *p)
283{
284 volatile u_int *ptr = (volatile u_int *)p;
285 ptr[b >> 5] |= (1 << (b & 0x1f));
286}
287
288static inline int
289test_bit(u_int b, const volatile void *p)
290{
291 return !!(((volatile u_int *)p)[b >> 5] & (1 << (b & 0x1f)));
292}
293
294static inline int
295__test_and_set_bit(u_int b, volatile void *p)
296{
297 unsigned int m = 1 << (b & 0x1f);
298 volatile u_int *ptr = (volatile u_int *)p;
299 unsigned int prev = ptr[b >> 5];
300 ptr[b >> 5] |= m;
301
302 return (prev & m) != 0;
303}
304
305static inline int
306test_and_clear_bit(u_int b, volatile void *p)
307{
308 unsigned int m = 1 << (b & 0x1f);
309 unsigned int prev = __sync_fetch_and_and((volatile u_int *)p + (b >> 5), ~m);
310 return (prev & m) != 0;
311}
312
313static inline int
314__test_and_clear_bit(u_int b, volatile void *p)
315{
316 volatile u_int *ptr = (volatile u_int *)p;
317 int rv = !!(ptr[b >> 5] & (1 << (b & 0x1f)));
318 ptr[b >> 5] &= ~(1 << (b & 0x1f));
319 return rv;
320}
321
322static inline int
323find_first_zero_bit(volatile void *p, int max)
324{
325 int b;
326 volatile u_int *ptr = (volatile u_int *)p;
327
328 for (b = 0; b < max; b += 32) {
329 if (ptr[b >> 5] != ~0) {
330 for (;;) {
331 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
332 return b;
333 b++;
334 }
335 }
336 }
337 return max;
338}
339
340static inline int
341find_next_zero_bit(volatile void *p, int max, int b)
342{
343 volatile u_int *ptr = (volatile u_int *)p;
344
345 for (; b < max; b += 32) {
346 if (ptr[b >> 5] != ~0) {
347 for (;;) {
348 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
349 return b;
350 b++;
351 }
352 }
353 }
354 return max;
355}
356
357static inline int
358find_first_bit(volatile void *p, int max)
359{
360 int b;
361 volatile u_int *ptr = (volatile u_int *)p;
362
363 for (b = 0; b < max; b += 32) {
7
←
Assuming 'b' is < 'max'
→
8
←
Loop condition is true. Entering loop body
→
11
←
The value 32 is assigned to 'b'
→
12
←
Assuming 'b' is < 'max'
→
13
←
Loop condition is true. Entering loop body
→
364 if (ptr[b >> 5] != 0) {
9
←
Assuming the condition is false
→
10
←
Taking false branch
→
14
←
Assuming the condition is true
→
15
←
Taking true branch
→
365 for (;;) {
16
←
Loop condition is true. Entering loop body
→
366 if (ptr[b >> 5] & (1 << (b & 0x1f)))
17
←
Assuming the condition is true
→
18
←
Taking true branch
→
367 return b;
19
←
Returning the value 32 (loaded from 'b')
→
368 b++;
369 }
370 }
371 }
372 return max;
373}
374
375static inline int
376find_next_bit(const volatile void *p, int max, int b)
377{
378 volatile u_int *ptr = (volatile u_int *)p;
379
380 for (; b < max; b+= 32) {
381 if (ptr[b >> 5] != 0) {
382 for (;;) {
383 if (ptr[b >> 5] & (1 << (b & 0x1f)))
384 return b;
385 b++;
386 }
387 }
388 }
389 return max;
390}
391
392#define for_each_set_bit(b, p, max)for ((b) = find_first_bit((p), (max)); (b) < (max); (b) = find_next_bit
((p), (max), (b) + 1)) \
393 for ((b) = find_first_bit((p), (max)); \
394 (b) < (max); \
395 (b) = find_next_bit((p), (max), (b) + 1))
396
397#define for_each_clear_bit(b, p, max)for ((b) = find_first_zero_bit((p), (max)); (b) < (max); (
b) = find_next_zero_bit((p), (max), (b) + 1)) \
398 for ((b) = find_first_zero_bit((p), (max)); \
399 (b) < (max); \
400 (b) = find_next_zero_bit((p), (max), (b) + 1))
401
402#if defined(__i386__)
403#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
404#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
405#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
406#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
) __asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
407#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
408#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
409#define __smp_store_mb(var, value)do { (void)__sync_lock_test_and_set(&var, value); } while
(0) do { (void)xchg(&var, value)__sync_lock_test_and_set(&var, value); } while (0)
410#define smp_mb__after_atomic()do { } while (0) do { } while (0)
411#define smp_mb__before_atomic()do { } while (0) do { } while (0)
412#elif defined(__amd64__1)
413#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("lfence")do { __asm volatile("lfence" ::: "memory"); } while (0)
414#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("sfence")do { __asm volatile("sfence" ::: "memory"); } while (0)
415#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("mfence")do { __asm volatile("mfence" ::: "memory"); } while (0)
416#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
) __asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc")
417#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
418#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
419#define __smp_store_mb(var, value)do { (void)__sync_lock_test_and_set(&var, value); } while
(0) do { (void)xchg(&var, value)__sync_lock_test_and_set(&var, value); } while (0)
420#define smp_mb__after_atomic()do { } while (0) do { } while (0)
421#define smp_mb__before_atomic()do { } while (0) do { } while (0)
422#elif defined(__aarch64__)
423#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("dsb ld")do { __asm volatile("dsb ld" ::: "memory"); } while (0)
424#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("dsb st")do { __asm volatile("dsb st" ::: "memory"); } while (0)
425#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
426#define dma_rmb() __membar("dmb oshld")do { __asm volatile("dmb oshld" ::: "memory"); } while (0)
427#define dma_wmb() __membar("dmb oshst")do { __asm volatile("dmb oshst" ::: "memory"); } while (0)
428#define dma_mb() __membar("dmb osh")do { __asm volatile("dmb osh" ::: "memory"); } while (0)
429#elif defined(__arm__)
430#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
431#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
432#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
433#elif defined(__mips64__)
434#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) mips_sync()
435#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) mips_sync()
436#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) mips_sync()
437#elif defined(__powerpc64__)
438#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
439#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
440#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
441#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("lwsync")do { __asm volatile("lwsync" ::: "memory"); } while (0)
442#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("lwsync")do { __asm volatile("lwsync" ::: "memory"); } while (0)
443#elif defined(__powerpc__)
444#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
445#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
446#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
447#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("eieio")do { __asm volatile("eieio" ::: "memory"); } while (0)
448#elif defined(__riscv)
449#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("fence ir,ir")do { __asm volatile("fence ir,ir" ::: "memory"); } while (0)
450#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("fence ow,ow")do { __asm volatile("fence ow,ow" ::: "memory"); } while (0)
451#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("fence iorw,iorw")do { __asm volatile("fence iorw,iorw" ::: "memory"); } while (
0)
452#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("fence r,r")do { __asm volatile("fence r,r" ::: "memory"); } while (0)
453#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("fence w,w")do { __asm volatile("fence w,w" ::: "memory"); } while (0)
454#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
) __membar("fence rw,rw")do { __asm volatile("fence rw,rw" ::: "memory"); } while (0)
455#elif defined(__sparc64__)
456#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
457#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
458#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
459#endif
460
461#ifndef smp_rmb
462#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)
463#endif
464
465#ifndef smp_wmb
466#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)
467#endif
468
469#ifndef mmiowb
470#define mmiowb()do { __asm volatile("sfence" ::: "memory"); } while (0) wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)
471#endif
472
473#ifndef smp_mb__before_atomic
474#define smp_mb__before_atomic()do { } while (0) mb()do { __asm volatile("mfence" ::: "memory"); } while (0)
475#endif
476
477#ifndef smp_mb__after_atomic
478#define smp_mb__after_atomic()do { } while (0) mb()do { __asm volatile("mfence" ::: "memory"); } while (0)
479#endif
480
481#ifndef smp_store_mb
482#define smp_store_mb(x, v)do { x = v; do { __asm volatile("mfence" ::: "memory"); } while
(0); } while (0) do { x = v; mb()do { __asm volatile("mfence" ::: "memory"); } while (0); } while (0)
483#endif
484
485#endif