Bug Summary

File:dev/pci/drm/i915/gem/i915_gem_pages.c
Warning:line 59, column 37
The result of the left shift is undefined due to shifting by '65', which is greater or equal to the width of type 'unsigned int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name i915_gem_pages.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/dev/pci/drm/i915/gem/i915_gem_pages.c

/usr/src/sys/dev/pci/drm/i915/gem/i915_gem_pages.c

→
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7#include "i915_drv.h"
8#include "i915_gem_object.h"
9#include "i915_scatterlist.h"
10#include "i915_gem_lmem.h"
11#include "i915_gem_mman.h"
12
13void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
14 struct sg_table *pages,
15 unsigned int sg_page_sizes)
16{
17 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
18 unsigned long supported = INTEL_INFO(i915)(&(i915)->__info)->page_sizes;
19 int i;
20
21 lockdep_assert_held(&obj->mm.lock)do { (void)(&obj->mm.lock); } while(0);
1
Loop condition is false. Exiting loop
→
22
23 if (i915_gem_object_is_volatile(obj))
2
←
Assuming the condition is false
→
3
←
Taking false branch
→
24 obj->mm.madv = I915_MADV_DONTNEED1;
25
26 /* Make the pages coherent with the GPU (flushing any swapin). */
27 if (obj->cache_dirty) {
4
←
Assuming field 'cache_dirty' is 0
→
5
←
Taking false branch
→
28 obj->write_domain = 0;
29 if (i915_gem_object_has_struct_page(obj))
30 drm_clflush_sg(pages);
31 obj->cache_dirty = false0;
32 }
33
34 obj->mm.get_page.sg_pos = pages->sgl;
35 obj->mm.get_page.sg_idx = 0;
36
37 obj->mm.pages = pages;
38
39 if (i915_gem_object_is_tiled(obj) &&
40 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES(1<<5)) {
41 GEM_BUG_ON(obj->mm.quirked)((void)0);
42 __i915_gem_object_pin_pages(obj);
43 obj->mm.quirked = true1;
44 }
45
46 GEM_BUG_ON(!sg_page_sizes)((void)0);
47 obj->mm.page_sizes.phys = sg_page_sizes;
48
49 /*
50 * Calculate the supported page-sizes which fit into the given
51 * sg_page_sizes. This will give us the page-sizes which we may be able
52 * to use opportunistically when later inserting into the GTT. For
53 * example if phys=2G, then in theory we should be able to use 1G, 2M,
54 * 64K or 4K pages, although in practice this will depend on a number of
55 * other factors.
56 */
57 obj->mm.page_sizes.sg = 0;
58 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)for ((i) = find_first_bit((&supported), (((sizeof((1ULL <<
(21))) <= 4) ? (fls((1ULL << (21))) - 1) : (flsl((1ULL
<< (21))) - 1)) + 1)); (i) < (((sizeof((1ULL <<
(21))) <= 4) ? (fls((1ULL << (21))) - 1) : (flsl((1ULL
<< (21))) - 1)) + 1); (i) = find_next_bit((&supported
), (((sizeof((1ULL << (21))) <= 4) ? (fls((1ULL <<
(21))) - 1) : (flsl((1ULL << (21))) - 1)) + 1), (i) + 1
)) {
6
←
'?' condition is false
→
7
←
Calling 'find_first_bit'
→
28
←
Returning from 'find_first_bit'
→
29
←
The value 65 is assigned to 'i'
→
30
←
'?' condition is false
→
31
←
Assuming the condition is true
→
32
←
Loop condition is true. Entering loop body
→
59 if (obj->mm.page_sizes.phys & ~0u << i)
33
←
The result of the left shift is undefined due to shifting by '65', which is greater or equal to the width of type 'unsigned int'
60 obj->mm.page_sizes.sg |= BIT(i)(1UL << (i));
61 }
62 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg))((void)0);
63
64 if (i915_gem_object_is_shrinkable(obj)) {
65 struct list_head *list;
66 unsigned long flags;
67
68 spin_lock_irqsave(&i915->mm.obj_lock, flags)do { flags = 0; mtx_enter(&i915->mm.obj_lock); } while
(0);
69
70 i915->mm.shrink_count++;
71 i915->mm.shrink_memory += obj->base.size;
72
73 if (obj->mm.madv != I915_MADV_WILLNEED0)
74 list = &i915->mm.purge_list;
75 else
76 list = &i915->mm.shrink_list;
77 list_add_tail(&obj->mm.link, list);
78
79 atomic_set(&obj->mm.shrink_pin, 0)({ typeof(*(&obj->mm.shrink_pin)) __tmp = ((0)); *(volatile
typeof(*(&obj->mm.shrink_pin)) *)&(*(&obj->
mm.shrink_pin)) = __tmp; __tmp; });
80 spin_unlock_irqrestore(&i915->mm.obj_lock, flags)do { (void)(flags); mtx_leave(&i915->mm.obj_lock); } while
(0);
81 }
82}
83
84int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
85{
86 struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
87 int err;
88
89 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)__builtin_expect(!!(obj->mm.madv != 0), 0)) {
90 drm_dbg(&i915->drm,drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Attempting to obtain a purgeable object\n"
)
91 "Attempting to obtain a purgeable object\n")drm_dev_dbg((&i915->drm)->dev, DRM_UT_DRIVER, "Attempting to obtain a purgeable object\n"
);
92 return -EFAULT14;
93 }
94
95 err = obj->ops->get_pages(obj);
96 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj))((void)0);
97
98 return err;
99}
100
101/* Ensure that the associated pages are gathered from the backing storage
102 * and pinned into our object. i915_gem_object_pin_pages() may be called
103 * multiple times before they are released by a single call to
104 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
105 * either as a result of memory pressure (reaping pages under the shrinker)
106 * or as the object is itself released.
107 */
108int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
109{
110 int err;
111
112 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES)mutex_lock_interruptible(&obj->mm.lock);
113 if (err)
114 return err;
115
116 if (unlikely(!i915_gem_object_has_pages(obj))__builtin_expect(!!(!i915_gem_object_has_pages(obj)), 0)) {
117 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj))((void)0);
118
119 err = ____i915_gem_object_get_pages(obj);
120 if (err)
121 goto unlock;
122
123 smp_mb__before_atomic()do { } while (0);
124 }
125 atomic_inc(&obj->mm.pages_pin_count)__sync_fetch_and_add(&obj->mm.pages_pin_count, 1);
126
127unlock:
128 mutex_unlock(&obj->mm.lock)rw_exit_write(&obj->mm.lock);
129 return err;
130}
131
132/* Immediately discard the backing storage */
133void i915_gem_object_truncate(struct drm_i915_gem_object *obj)
134{
135 drm_gem_free_mmap_offset(&obj->base);
136 if (obj->ops->truncate)
137 obj->ops->truncate(obj);
138}
139
140/* Try to discard unwanted pages */
141void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
142{
143 lockdep_assert_held(&obj->mm.lock)do { (void)(&obj->mm.lock); } while(0);
144 GEM_BUG_ON(i915_gem_object_has_pages(obj))((void)0);
145
146 if (obj->ops->writeback)
147 obj->ops->writeback(obj);
148}
149
150static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
151{
152 struct radix_tree_iter iter;
153 void __rcu **slot;
154
155 rcu_read_lock();
156 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)for ((&iter)->index = (0); radix_tree_iter_find(&obj
->mm.get_page.radix, &iter, &(slot)); (&iter)->
index++)
157 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
158 rcu_read_unlock();
159}
160
161static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
162{
163 if (is_vmalloc_addr(ptr))
164 vunmap(ptr, obj->base.size);
165}
166
167struct sg_table *
168__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
169{
170 struct sg_table *pages;
171
172 pages = fetch_and_zero(&obj->mm.pages)({ typeof(*&obj->mm.pages) __T = *(&obj->mm.pages
); *(&obj->mm.pages) = (typeof(*&obj->mm.pages)
)0; __T; });
173 if (IS_ERR_OR_NULL(pages))
174 return pages;
175
176 if (i915_gem_object_is_volatile(obj))
177 obj->mm.madv = I915_MADV_WILLNEED0;
178
179 i915_gem_object_make_unshrinkable(obj);
180
181 if (obj->mm.mapping) {
182 unmap_object(obj, page_mask_bits(obj->mm.mapping)({ unsigned long __v = (unsigned long)(obj->mm.mapping); (
typeof(obj->mm.mapping))(__v & -(1UL << (12))); }
));
183 obj->mm.mapping = NULL((void *)0);
184 }
185
186 __i915_gem_object_reset_page_iter(obj);
187 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
188
189 return pages;
190}
191
192int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
193{
194 struct sg_table *pages;
195 int err;
196
197 if (i915_gem_object_has_pinned_pages(obj))
198 return -EBUSY16;
199
200 /* May be called by shrinker from within get_pages() (on another bo) */
201 mutex_lock(&obj->mm.lock)rw_enter_write(&obj->mm.lock);
202 if (unlikely(atomic_read(&obj->mm.pages_pin_count))__builtin_expect(!!(({ typeof(*(&obj->mm.pages_pin_count
)) __tmp = *(volatile typeof(*(&obj->mm.pages_pin_count
)) *)&(*(&obj->mm.pages_pin_count)); membar_datadep_consumer
(); __tmp; })), 0)) {
203 err = -EBUSY16;
204 goto unlock;
205 }
206
207 i915_gem_object_release_mmap_offset(obj);
208
209 /*
210 * ->put_pages might need to allocate memory for the bit17 swizzle
211 * array, hence protect them from being reaped by removing them from gtt
212 * lists early.
213 */
214 pages = __i915_gem_object_unset_pages(obj);
215
216 /*
217 * XXX Temporary hijinx to avoid updating all backends to handle
218 * NULL pages. In the future, when we have more asynchronous
219 * get_pages backends we should be better able to handle the
220 * cancellation of the async task in a more uniform manner.
221 */
222 if (!pages && !i915_gem_object_needs_async_cancel(obj))
223 pages = ERR_PTR(-EINVAL22);
224
225 if (!IS_ERR(pages))
226 obj->ops->put_pages(obj, pages);
227
228 err = 0;
229unlock:
230 mutex_unlock(&obj->mm.lock)rw_exit_write(&obj->mm.lock);
231
232 return err;
233}
234
235/* The 'mapping' part of i915_gem_object_pin_map() below */
236static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
237 enum i915_map_type type)
238{
239 unsigned long n_pages = obj->base.size >> PAGE_SHIFT12, i;
240 struct vm_page *stack[32], **pages = stack, *page;
241 struct sgt_iter iter;
242 pgprot_t pgprot;
243 void *vaddr;
244
245 switch (type) {
246 default:
247 MISSING_CASE(type)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "type", (long)(type)); __builtin_expect(!!(__ret), 0); });
248 fallthroughdo {} while (0); /* to use PAGE_KERNEL anyway */
249 case I915_MAP_WB:
250 /*
251 * On 32b, highmem using a finite set of indirect PTE (i.e.
252 * vmap) to provide virtual mappings of the high pages.
253 * As these are finite, map_new_virtual() must wait for some
254 * other kmap() to finish when it runs out. If we map a large
255 * number of objects, there is no method for it to tell us
256 * to release the mappings, and we deadlock.
257 *
258 * However, if we make an explicit vmap of the page, that
259 * uses a larger vmalloc arena, and also has the ability
260 * to tell us to release unwanted mappings. Most importantly,
261 * it will fail and propagate an error instead of waiting
262 * forever.
263 *
264 * So if the page is beyond the 32b boundary, make an explicit
265 * vmap.
266 */
267#ifdef notyet
268 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))0)
269 return page_address(sg_page(obj->mm.pages->sgl));
270#endif
271 pgprot = PAGE_KERNEL0;
272 break;
273 case I915_MAP_WC:
274 pgprot = pgprot_writecombine(PAGE_KERNEL_IO0);
275 break;
276 }
277
278 if (n_pages > ARRAY_SIZE(stack)(sizeof((stack)) / sizeof((stack)[0]))) {
279 /* Too big for stack -- allocate temporary array instead */
280 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL(0x0001 | 0x0004));
281 if (!pages)
282 return NULL((void *)0);
283 }
284
285 i = 0;
286 for_each_sgt_page(page, iter, obj->mm.pages)for ((iter) = __sgt_iter((obj->mm.pages)->sgl, 0); ((page
) = (iter).pfn == 0 ? ((void *)0) : (PHYS_TO_VM_PAGE(((paddr_t
)((iter).pfn + ((iter).curr >> 12)) << 12)))); ((
(iter).curr += (1 << 12)) >= (iter).max) ? (iter) = __sgt_iter
(__sg_next((iter).sgp), 0), 0 : 0)
287 pages[i++] = page;
288 vaddr = vmap(pages, n_pages, 0, pgprot);
289 if (pages != stack)
290 kvfree(pages);
291 return vaddr;
292}
293
294static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
295 enum i915_map_type type)
296{
297 STUB()do { printf("%s: stub\n", __func__); } while(0);
298 return NULL((void *)0);
299#ifdef notyet
300 resource_size_t iomap = obj->mm.region->iomap.base -
301 obj->mm.region->region.start;
302 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT12;
303 unsigned long stack[32], *pfns = stack, i;
304 struct sgt_iter iter;
305 dma_addr_t addr;
306 void *vaddr;
307
308 if (type != I915_MAP_WC)
309 return NULL((void *)0);
310
311 if (n_pfn > ARRAY_SIZE(stack)(sizeof((stack)) / sizeof((stack)[0]))) {
312 /* Too big for stack -- allocate temporary array instead */
313 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL(0x0001 | 0x0004));
314 if (!pfns)
315 return NULL((void *)0);
316 }
317
318 i = 0;
319 for_each_sgt_daddr(addr, iter, obj->mm.pages)for ((iter) = __sgt_iter((obj->mm.pages)->sgl, 1); ((addr
) = (iter).dma + (iter).curr), (iter).sgp; (((iter).curr += (
(1ULL << (12)))) >= (iter).max) ? (iter) = __sgt_iter
(__sg_next((iter).sgp), 1), 0 : 0)
320 pfns[i++] = (iomap + addr) >> PAGE_SHIFT12;
321 vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO0));
322 if (pfns != stack)
323 kvfree(pfns);
324 return vaddr;
325#endif
326}
327
328static void *i915_gem_object_map(struct drm_i915_gem_object *obj,
329 enum i915_map_type type)
330{
331 unsigned long n_pages = obj->base.size >> PAGE_SHIFT12;
332 struct sg_table *sgt = obj->mm.pages;
333 struct vm_page *stack_pages[32];
334 struct vm_page **pages = stack_pages;
335 struct vm_struct *area;
336 pgprot_t pgprot;
337 void *addr;
338
339 if (!i915_gem_object_has_struct_page(obj) && type != I915_MAP_WC)
340 return NULL((void *)0);
341
342#if 0
343 /* A single page can always be kmapped */
344 if (n_pages == 1 && type == I915_MAP_WB)
345 return kmap(sg_page(sgt->sgl));
346#endif
347
348 if (n_pages > ARRAY_SIZE(stack_pages)(sizeof((stack_pages)) / sizeof((stack_pages)[0]))) {
349 /* Too big for stack -- allocate temporary array instead */
350 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL(0x0001 | 0x0004));
351 if (!pages)
352 return NULL((void *)0);
353 }
354
355 switch (type) {
356 default:
357 MISSING_CASE(type)({ int __ret = !!(1); if (__ret) printf("Missing case (%s == %ld)\n"
, "type", (long)(type)); __builtin_expect(!!(__ret), 0); });
358 /* fallthrough - to use PAGE_KERNEL anyway */
359 case I915_MAP_WB:
360 pgprot = PAGE_KERNEL0;
361 break;
362 case I915_MAP_WC:
363 pgprot = pgprot_writecombine(PAGE_KERNEL_IO0);
364 break;
365 }
366
367 if (i915_gem_object_has_struct_page(obj)) {
368 struct sgt_iter iter;
369 struct vm_page *page;
370 unsigned long i = 0;
371
372 for_each_sgt_page(page, iter, sgt)for ((iter) = __sgt_iter((sgt)->sgl, 0); ((page) = (iter).
pfn == 0 ? ((void *)0) : (PHYS_TO_VM_PAGE(((paddr_t)((iter).pfn
+ ((iter).curr >> 12)) << 12)))); (((iter).curr +=
(1 << 12)) >= (iter).max) ? (iter) = __sgt_iter(__sg_next
((iter).sgp), 0), 0 : 0)
373 pages[i++] = page;
374 } else {
375 STUB()do { printf("%s: stub\n", __func__); } while(0);
376#ifdef notyet
377 resource_size_t iomap;
378 struct sgt_iter iter;
379 pte_t **ptes = mem;
380 dma_addr_t addr;
381
382 iomap = obj->mm.region->iomap.base;
383 iomap -= obj->mm.region->region.start;
384
385 for_each_sgt_daddr(addr, iter, sgt)for ((iter) = __sgt_iter((sgt)->sgl, 1); ((addr) = (iter).
dma + (iter).curr), (iter).sgp; (((iter).curr += ((1ULL <<
(12)))) >= (iter).max) ? (iter) = __sgt_iter(__sg_next((iter
).sgp), 1), 0 : 0)
386 **ptes++ = iomap_pte(iomap, addr, pgprot);
387#endif
388 }
389 addr = vmap(pages, n_pages, 0, pgprot);
390
391 if (pages != stack_pages)
392 kvfree(pages);
393
394 return addr;
395}
396
397/* get, pin, and map the pages of the object into kernel space */
398void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
399 enum i915_map_type type)
400{
401 enum i915_map_type has_type;
402 unsigned int flags;
403 bool_Bool pinned;
404 void *ptr;
405 int err;
406
407 flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE(1UL << (0)) | I915_GEM_OBJECT_HAS_IOMEM(1UL << (1));
408 if (!i915_gem_object_type_has(obj, flags))
409 return ERR_PTR(-ENXIO6);
410
411 err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES)mutex_lock_interruptible(&obj->mm.lock);
412 if (err)
413 return ERR_PTR(err);
414
415 pinned = !(type & I915_MAP_OVERRIDE(1UL << (31)));
416 type &= ~I915_MAP_OVERRIDE(1UL << (31));
417
418 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)atomic_add_unless((&obj->mm.pages_pin_count), 1, 0)) {
419 if (unlikely(!i915_gem_object_has_pages(obj))__builtin_expect(!!(!i915_gem_object_has_pages(obj)), 0)) {
420 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj))((void)0);
421
422 err = ____i915_gem_object_get_pages(obj);
423 if (err)
424 goto err_unlock;
425
426 smp_mb__before_atomic()do { } while (0);
427 }
428 atomic_inc(&obj->mm.pages_pin_count)__sync_fetch_and_add(&obj->mm.pages_pin_count, 1);
429 pinned = false0;
430 }
431 GEM_BUG_ON(!i915_gem_object_has_pages(obj))((void)0);
432
433 ptr = page_unpack_bits(obj->mm.mapping, &has_type)({ unsigned long __v = (unsigned long)(obj->mm.mapping); *
(&has_type) = __v & ((1UL << (12)) - 1); (typeof
(obj->mm.mapping))(__v & -(1UL << (12))); });
434 if (ptr && has_type != type) {
435 if (pinned) {
436 err = -EBUSY16;
437 goto err_unpin;
438 }
439
440 unmap_object(obj, ptr);
441
442 ptr = obj->mm.mapping = NULL((void *)0);
443 }
444
445 if (!ptr) {
446 if (GEM_WARN_ON(type == I915_MAP_WC &&({ __builtin_expect(!!(!!(type == I915_MAP_WC && !static_cpu_has
(3))), 0); })
447 !static_cpu_has(X86_FEATURE_PAT))({ __builtin_expect(!!(!!(type == I915_MAP_WC && !static_cpu_has
(3))), 0); }))
448 ptr = NULL((void *)0);
449 else if (i915_gem_object_has_struct_page(obj))
450 ptr = i915_gem_object_map_page(obj, type);
451 else
452 ptr = i915_gem_object_map_pfn(obj, type);
453 if (!ptr) {
454 err = -ENOMEM12;
455 goto err_unpin;
456 }
457
458 obj->mm.mapping = page_pack_bits(ptr, type)({ unsigned long __bits = (type); ((void)0); ((typeof(ptr))((
unsigned long)(ptr) | __bits)); });
459 }
460
461out_unlock:
462 mutex_unlock(&obj->mm.lock)rw_exit_write(&obj->mm.lock);
463 return ptr;
464
465err_unpin:
466 atomic_dec(&obj->mm.pages_pin_count)__sync_fetch_and_sub(&obj->mm.pages_pin_count, 1);
467err_unlock:
468 ptr = ERR_PTR(err);
469 goto out_unlock;
470}
471
472void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
473 unsigned long offset,
474 unsigned long size)
475{
476 enum i915_map_type has_type;
477 void *ptr;
478
479 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj))((void)0);
480 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),((void)0)
481 offset, size, obj->base.size))((void)0);
482
483 wmb()do { __asm volatile("sfence" ::: "memory"); } while (0); /* let all previous writes be visible to coherent partners */
484 obj->mm.dirty = true1;
485
486 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE(1UL << (1)))
487 return;
488
489 ptr = page_unpack_bits(obj->mm.mapping, &has_type)({ unsigned long __v = (unsigned long)(obj->mm.mapping); *
(&has_type) = __v & ((1UL << (12)) - 1); (typeof
(obj->mm.mapping))(__v & -(1UL << (12))); });
490 if (has_type == I915_MAP_WC)
491 return;
492
493 drm_clflush_virt_range(ptr + offset, size);
494 if (size == obj->base.size) {
495 obj->write_domain &= ~I915_GEM_DOMAIN_CPU0x00000001;
496 obj->cache_dirty = false0;
497 }
498}
499
500void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
501{
502 GEM_BUG_ON(!obj->mm.mapping)((void)0);
503
504 /*
505 * We allow removing the mapping from underneath pinned pages!
506 *
507 * Furthermore, since this is an unsafe operation reserved only
508 * for construction time manipulation, we ignore locking prudence.
509 */
510 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))({ unsigned long __v = (unsigned long)(({ typeof(*&obj->
mm.mapping) __T = *(&obj->mm.mapping); *(&obj->
mm.mapping) = (typeof(*&obj->mm.mapping))0; __T; })); (
typeof(({ typeof(*&obj->mm.mapping) __T = *(&obj->
mm.mapping); *(&obj->mm.mapping) = (typeof(*&obj->
mm.mapping))0; __T; })))(__v & -(1UL << (12))); }));
511
512 i915_gem_object_unpin_map(obj);
513}
514
515struct scatterlist *
516i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
517 unsigned int n,
518 unsigned int *offset)
519{
520 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
521 struct scatterlist *sg;
522 unsigned int idx, count;
523
524 might_sleep()assertwaitok();
525 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT)((void)0);
526 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj))((void)0);
527
528 /* As we iterate forward through the sg, we record each entry in a
529 * radixtree for quick repeated (backwards) lookups. If we have seen
530 * this index previously, we will have an entry for it.
531 *
532 * Initial lookup is O(N), but this is amortized to O(1) for
533 * sequential page access (where each new request is consecutive
534 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
535 * i.e. O(1) with a large constant!
536 */
537 if (n < READ_ONCE(iter->sg_idx)({ typeof(iter->sg_idx) __tmp = *(volatile typeof(iter->
sg_idx) *)&(iter->sg_idx); membar_datadep_consumer(); __tmp
; }))
538 goto lookup;
539
540 mutex_lock(&iter->lock)rw_enter_write(&iter->lock);
541
542 /* We prefer to reuse the last sg so that repeated lookup of this
543 * (or the subsequent) sg are fast - comparing against the last
544 * sg is faster than going through the radixtree.
545 */
546
547 sg = iter->sg_pos;
548 idx = iter->sg_idx;
549 count = __sg_page_count(sg);
550
551 while (idx + count <= n) {
552 void *entry;
553 unsigned long i;
554 int ret;
555
556 /* If we cannot allocate and insert this entry, or the
557 * individual pages from this range, cancel updating the
558 * sg_idx so that on this lookup we are forced to linearly
559 * scan onwards, but on future lookups we will try the
560 * insertion again (in which case we need to be careful of
561 * the error return reporting that we have already inserted
562 * this index).
563 */
564 ret = radix_tree_insert(&iter->radix, idx, sg);
565 if (ret && ret != -EEXIST17)
566 goto scan;
567
568 entry = xa_mk_value(idx);
569 for (i = 1; i < count; i++) {
570 ret = radix_tree_insert(&iter->radix, idx + i, entry);
571 if (ret && ret != -EEXIST17)
572 goto scan;
573 }
574
575 idx += count;
576 sg = ____sg_next(sg);
577 count = __sg_page_count(sg);
578 }
579
580scan:
581 iter->sg_pos = sg;
582 iter->sg_idx = idx;
583
584 mutex_unlock(&iter->lock)rw_exit_write(&iter->lock);
585
586 if (unlikely(n < idx)__builtin_expect(!!(n < idx), 0)) /* insertion completed by another thread */
587 goto lookup;
588
589 /* In case we failed to insert the entry into the radixtree, we need
590 * to look beyond the current sg.
591 */
592 while (idx + count <= n) {
593 idx += count;
594 sg = ____sg_next(sg);
595 count = __sg_page_count(sg);
596 }
597
598 *offset = n - idx;
599 return sg;
600
601lookup:
602 rcu_read_lock();
603
604 sg = radix_tree_lookup(&iter->radix, n);
605 GEM_BUG_ON(!sg)((void)0);
606
607 /* If this index is in the middle of multi-page sg entry,
608 * the radix tree will contain a value entry that points
609 * to the start of that range. We will return the pointer to
610 * the base page and the offset of this page within the
611 * sg entry's range.
612 */
613 *offset = 0;
614 if (unlikely(xa_is_value(sg))__builtin_expect(!!(xa_is_value(sg)), 0)) {
615 unsigned long base = xa_to_value(sg);
616
617 sg = radix_tree_lookup(&iter->radix, base);
618 GEM_BUG_ON(!sg)((void)0);
619
620 *offset = n - base;
621 }
622
623 rcu_read_unlock();
624
625 return sg;
626}
627
628struct vm_page *
629i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
630{
631 struct scatterlist *sg;
632 unsigned int offset;
633
634 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj))((void)0);
635
636 sg = i915_gem_object_get_sg(obj, n, &offset);
637 return nth_page(sg_page(sg), offset)(&(sg_page(sg))[(offset)]);
638}
639
640/* Like i915_gem_object_get_page(), but mark the returned page dirty */
641struct vm_page *
642i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
643 unsigned int n)
644{
645 struct vm_page *page;
646
647 page = i915_gem_object_get_page(obj, n);
648 if (!obj->mm.dirty)
649 set_page_dirty(page)x86_atomic_clearbits_u32(&page->pg_flags, 0x00000008);
650
651 return page;
652}
653
654dma_addr_t
655i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
656 unsigned long n,
657 unsigned int *len)
658{
659 struct scatterlist *sg;
660 unsigned int offset;
661
662 sg = i915_gem_object_get_sg(obj, n, &offset);
663
664 if (len)
665 *len = sg_dma_len(sg)((sg)->length) - (offset << PAGE_SHIFT12);
666
667 return sg_dma_address(sg)((sg)->dma_address) + (offset << PAGE_SHIFT12);
668}
669
670dma_addr_t
671i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
672 unsigned long n)
673{
674 return i915_gem_object_get_dma_address_len(obj, n, NULL((void *)0));
675}
←

/usr/src/sys/dev/pci/drm/include/linux/atomic.h

1/* $OpenBSD: atomic.h,v 1.16 2021/06/25 13:41:09 jsg Exp $ */
2/**
3 * \file drm_atomic.h
4 * Atomic operations used in the DRM which may or may not be provided by the OS.
5 *
6 * \author Eric Anholt <anholt@FreeBSD.org>
7 */
8
9/*-
10 * Copyright 2004 Eric Anholt
11 * All Rights Reserved.
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
19 *
20 * The above copyright notice and this permission notice (including the next
21 * paragraph) shall be included in all copies or substantial portions of the
22 * Software.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
27 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
28 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
29 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
30 * OTHER DEALINGS IN THE SOFTWARE.
31 */
32
33#ifndef _DRM_LINUX_ATOMIC_H_
34#define _DRM_LINUX_ATOMIC_H_
35
36#include <sys/types.h>
37#include <sys/mutex.h>
38#include <machine/intr.h>
39#include <linux/types.h>
40#include <linux/compiler.h> /* via x86/include/asm/atomic.h */
41
42#define ATOMIC_INIT(x)(x) (x)
43
44#define atomic_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) WRITE_ONCE(*(p), (v))({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
45#define atomic_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; }) READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
46#define atomic_inc(p)__sync_fetch_and_add(p, 1) __sync_fetch_and_add(p, 1)
47#define atomic_dec(p)__sync_fetch_and_sub(p, 1) __sync_fetch_and_sub(p, 1)
48#define atomic_add(n, p)__sync_fetch_and_add(p, n) __sync_fetch_and_add(p, n)
49#define atomic_sub(n, p)__sync_fetch_and_sub(p, n) __sync_fetch_and_sub(p, n)
50#define atomic_and(n, p)__sync_fetch_and_and(p, n) __sync_fetch_and_and(p, n)
51#define atomic_or(n, p)x86_atomic_setbits_u32(p, n) atomic_setbits_intx86_atomic_setbits_u32(p, n)
52#define atomic_add_return(n, p)__sync_add_and_fetch(p, n) __sync_add_and_fetch(p, n)
53#define atomic_sub_return(n, p)__sync_sub_and_fetch(p, n) __sync_sub_and_fetch(p, n)
54#define atomic_inc_return(v)__sync_add_and_fetch((v), 1) atomic_add_return(1, (v))__sync_add_and_fetch((v), 1)
55#define atomic_dec_return(v)__sync_sub_and_fetch((v), 1) atomic_sub_return(1, (v))__sync_sub_and_fetch((v), 1)
56#define atomic_dec_and_test(v)(__sync_sub_and_fetch((v), 1) == 0) (atomic_dec_return(v)__sync_sub_and_fetch((v), 1) == 0)
57#define atomic_inc_and_test(v)(__sync_add_and_fetch((v), 1) == 0) (atomic_inc_return(v)__sync_add_and_fetch((v), 1) == 0)
58#define atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) __sync_val_compare_and_swap(p, o, n)
59#define cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) __sync_val_compare_and_swap(p, o, n)
60#define atomic_set_release(p, v)({ typeof(*((p))) __tmp = (((v))); *(volatile typeof(*((p))) *
)&(*((p))) = __tmp; __tmp; }) atomic_set((p), (v))({ typeof(*((p))) __tmp = (((v))); *(volatile typeof(*((p))) *
)&(*((p))) = __tmp; __tmp; })
61#define atomic_andnot(bits, p)x86_atomic_clearbits_u32(p,bits) atomic_clearbits_intx86_atomic_clearbits_u32(p,bits)
62#define atomic_fetch_inc(p)__sync_fetch_and_add(p, 1) __sync_fetch_and_add(p, 1)
63#define atomic_fetch_xor(n, p)__sync_fetch_and_xor(p, n) __sync_fetch_and_xor(p, n)
64
65#define try_cmpxchg(p, op, n)({ __typeof(p) __op = (__typeof((p)))(op); __typeof(*(p)) __o
= *__op; __typeof(*(p)) __p = __sync_val_compare_and_swap((p
), (__o), (n)); if (__p != __o) *__op = __p; (__p == __o); }) \
66({ \
67 __typeof(p) __op = (__typeof((p)))(op); \
68 __typeof(*(p)) __o = *__op; \
69 __typeof(*(p)) __p = __sync_val_compare_and_swap((p), (__o), (n)); \
70 if (__p != __o) \
71 *__op = __p; \
72 (__p == __o); \
73})
74
75static inline bool_Bool
76atomic_try_cmpxchg(volatile int *p, int *op, int n)
77{
78 return try_cmpxchg(p, op, n)({ __typeof(p) __op = (__typeof((p)))(op); __typeof(*(p)) __o
= *__op; __typeof(*(p)) __p = __sync_val_compare_and_swap((p
), (__o), (n)); if (__p != __o) *__op = __p; (__p == __o); });
79}
80
81static inline int
82atomic_xchg(volatile int *v, int n)
83{
84 __sync_synchronize();
85 return __sync_lock_test_and_set(v, n);
86}
87
88#define xchg(v, n)__sync_lock_test_and_set(v, n) __sync_lock_test_and_set(v, n)
89
90static inline int
91atomic_add_unless(volatile int *v, int n, int u)
92{
93 int o;
94
95 do {
96 o = *v;
97 if (o == u)
98 return 0;
99 } while (__sync_val_compare_and_swap(v, o, o +n) != o);
100
101 return 1;
102}
103
104#define atomic_inc_not_zero(v)atomic_add_unless((v), 1, 0) atomic_add_unless((v), 1, 0)
105
106static inline int
107atomic_dec_if_positive(volatile int *v)
108{
109 int r, o;
110
111 do {
112 o = *v;
113 r = o - 1;
114 if (r < 0)
115 break;
116 } while (__sync_val_compare_and_swap(v, o, r) != o);
117
118 return r;
119}
120
121#define atomic_long_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; }) READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
122
123/* 32 bit powerpc lacks 64 bit atomics */
124#if !defined(__powerpc__) || defined(__powerpc64__)
125
126typedef int64_t atomic64_t;
127
128#define ATOMIC64_INIT(x)(x) (x)
129
130#define atomic64_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) WRITE_ONCE(*(p), (v))({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
131#define atomic64_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; }) READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
132
133static inline int64_t
134atomic64_xchg(volatile int64_t *v, int64_t n)
135{
136 __sync_synchronize();
137 return __sync_lock_test_and_set(v, n);
138}
139
140#define atomic64_add(n, p)__sync_fetch_and_add_8(p, n) __sync_fetch_and_add_8(p, n)
141#define atomic64_sub(n, p)__sync_fetch_and_sub_8(p, n) __sync_fetch_and_sub_8(p, n)
142#define atomic64_inc(p)__sync_fetch_and_add_8(p, 1) __sync_fetch_and_add_8(p, 1)
143#define atomic64_add_return(n, p)__sync_add_and_fetch_8(p, n) __sync_add_and_fetch_8(p, n)
144#define atomic64_inc_return(p)__sync_add_and_fetch_8(p, 1) __sync_add_and_fetch_8(p, 1)
145
146#else
147
148extern struct mutex atomic64_mtx;
149
150typedef struct {
151 volatile int64_t val;
152} atomic64_t;
153
154#define ATOMIC64_INIT(x)(x) { (x) }
155
156static inline void
157atomic64_set(atomic64_t *v, int64_t i)({ typeof(*(atomic64_t *v)) __tmp = ((int64_t i)); *(volatile
typeof(*(atomic64_t *v)) *)&(*(atomic64_t *v)) = __tmp; __tmp
; })
158{
159 mtx_enter(&atomic64_mtx);
160 v->val = i;
161 mtx_leave(&atomic64_mtx);
162}
163
164static inline int64_t
165atomic64_read(atomic64_t *v)({ typeof(*(atomic64_t *v)) __tmp = *(volatile typeof(*(atomic64_t
*v)) *)&(*(atomic64_t *v)); membar_datadep_consumer(); __tmp
; })
166{
167 int64_t val;
168
169 mtx_enter(&atomic64_mtx);
170 val = v->val;
171 mtx_leave(&atomic64_mtx);
172
173 return val;
174}
175
176static inline int64_t
177atomic64_xchg(atomic64_t *v, int64_t n)
178{
179 int64_t val;
180
181 mtx_enter(&atomic64_mtx);
182 val = v->val;
183 v->val = n;
184 mtx_leave(&atomic64_mtx);
185
186 return val;
187}
188
189static inline void
190atomic64_add(int i, atomic64_t *v)__sync_fetch_and_add_8(atomic64_t *v, int i)
191{
192 mtx_enter(&atomic64_mtx);
193 v->val += i;
194 mtx_leave(&atomic64_mtx);
195}
196
197#define atomic64_inc(p)__sync_fetch_and_add_8(p, 1) atomic64_add(p, 1)__sync_fetch_and_add_8(1, p)
198
199static inline int64_t
200atomic64_add_return(int i, atomic64_t *v)__sync_add_and_fetch_8(atomic64_t *v, int i)
201{
202 int64_t val;
203
204 mtx_enter(&atomic64_mtx);
205 val = v->val + i;
206 v->val = val;
207 mtx_leave(&atomic64_mtx);
208
209 return val;
210}
211
212#define atomic64_inc_return(p)__sync_add_and_fetch_8(p, 1) atomic64_add_return(p, 1)__sync_add_and_fetch_8(1, p)
213
214static inline void
215atomic64_sub(int i, atomic64_t *v)__sync_fetch_and_sub_8(atomic64_t *v, int i)
216{
217 mtx_enter(&atomic64_mtx);
218 v->val -= i;
219 mtx_leave(&atomic64_mtx);
220}
221#endif
222
223#ifdef __LP64__1
224typedef int64_t atomic_long_t;
225#define atomic_long_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) atomic64_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
226#define atomic_long_xchg(v, n)atomic64_xchg(v, n) atomic64_xchg(v, n)
227#define atomic_long_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)
228#else
229typedef int32_t atomic_long_t;
230#define atomic_long_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; }) atomic_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
231#define atomic_long_xchg(v, n)atomic64_xchg(v, n) atomic_xchg(v, n)
232#define atomic_long_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n) atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)
233#endif
234
235static inline atomic_t
236test_and_set_bit(u_int b, volatile void *p)
237{
238 unsigned int m = 1 << (b & 0x1f);
239 unsigned int prev = __sync_fetch_and_or((volatile u_int *)p + (b >> 5), m);
240 return (prev & m) != 0;
241}
242
243static inline void
244clear_bit(u_int b, volatile void *p)
245{
246 atomic_clearbits_intx86_atomic_clearbits_u32(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
247}
248
249static inline void
250clear_bit_unlock(u_int b, volatile void *p)
251{
252 membar_enter()do { __asm volatile("mfence" ::: "memory"); } while (0);
253 clear_bit(b, p);
254}
255
256static inline void
257set_bit(u_int b, volatile void *p)
258{
259 atomic_setbits_intx86_atomic_setbits_u32(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
260}
261
262static inline void
263__clear_bit(u_int b, volatile void *p)
264{
265 volatile u_int *ptr = (volatile u_int *)p;
266 ptr[b >> 5] &= ~(1 << (b & 0x1f));
267}
268
269static inline void
270__set_bit(u_int b, volatile void *p)
271{
272 volatile u_int *ptr = (volatile u_int *)p;
273 ptr[b >> 5] |= (1 << (b & 0x1f));
274}
275
276static inline int
277test_bit(u_int b, const volatile void *p)
278{
279 return !!(((volatile u_int *)p)[b >> 5] & (1 << (b & 0x1f)));
280}
281
282static inline int
283__test_and_set_bit(u_int b, volatile void *p)
284{
285 unsigned int m = 1 << (b & 0x1f);
286 volatile u_int *ptr = (volatile u_int *)p;
287 unsigned int prev = ptr[b >> 5];
288 ptr[b >> 5] |= m;
289
290 return (prev & m) != 0;
291}
292
293static inline int
294test_and_clear_bit(u_int b, volatile void *p)
295{
296 unsigned int m = 1 << (b & 0x1f);
297 unsigned int prev = __sync_fetch_and_and((volatile u_int *)p + (b >> 5), ~m);
298 return (prev & m) != 0;
299}
300
301static inline int
302__test_and_clear_bit(u_int b, volatile void *p)
303{
304 volatile u_int *ptr = (volatile u_int *)p;
305 int rv = !!(ptr[b >> 5] & (1 << (b & 0x1f)));
306 ptr[b >> 5] &= ~(1 << (b & 0x1f));
307 return rv;
308}
309
310static inline int
311find_first_zero_bit(volatile void *p, int max)
312{
313 int b;
314 volatile u_int *ptr = (volatile u_int *)p;
315
316 for (b = 0; b < max; b += 32) {
317 if (ptr[b >> 5] != ~0) {
318 for (;;) {
319 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
320 return b;
321 b++;
322 }
323 }
324 }
325 return max;
326}
327
328static inline int
329find_next_zero_bit(volatile void *p, int max, int b)
330{
331 volatile u_int *ptr = (volatile u_int *)p;
332
333 for (; b < max; b += 32) {
334 if (ptr[b >> 5] != ~0) {
335 for (;;) {
336 if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
337 return b;
338 b++;
339 }
340 }
341 }
342 return max;
343}
344
345static inline int
346find_first_bit(volatile void *p, int max)
347{
348 int b;
349 volatile u_int *ptr = (volatile u_int *)p;
350
351 for (b = 0; b < max; b += 32) {
8
←
Assuming 'b' is < 'max'
→
9
←
Loop condition is true. Entering loop body
→
12
←
Assuming 'b' is < 'max'
→
13
←
Loop condition is true. Entering loop body
→
16
←
Assuming 'b' is < 'max'
→
17
←
Loop condition is true. Entering loop body
→
352 if (ptr[b >> 5] != 0) {
10
←
Assuming the condition is false
→
11
←
Taking false branch
→
14
←
Assuming the condition is false
→
15
←
Taking false branch
→
18
←
Assuming the condition is true
→
19
←
Taking true branch
→
353 for (;;) {
20
←
Loop condition is true. Entering loop body
→
24
←
Loop condition is true. Entering loop body
→
354 if (ptr[b >> 5] & (1 << (b & 0x1f)))
21
←
Assuming the condition is false
→
22
←
Taking false branch
→
25
←
Assuming the condition is true
→
26
←
Taking true branch
→
355 return b;
27
←
Returning the value 65 (loaded from 'b')
→
356 b++;
23
←
The value 65 is assigned to 'b'
→
357 }
358 }
359 }
360 return max;
361}
362
363static inline int
364find_next_bit(volatile void *p, int max, int b)
365{
366 volatile u_int *ptr = (volatile u_int *)p;
367
368 for (; b < max; b+= 32) {
369 if (ptr[b >> 5] != 0) {
370 for (;;) {
371 if (ptr[b >> 5] & (1 << (b & 0x1f)))
372 return b;
373 b++;
374 }
375 }
376 }
377 return max;
378}
379
380#define for_each_set_bit(b, p, max)for ((b) = find_first_bit((p), (max)); (b) < (max); (b) = find_next_bit
((p), (max), (b) + 1)) \
381 for ((b) = find_first_bit((p), (max)); \
382 (b) < (max); \
383 (b) = find_next_bit((p), (max), (b) + 1))
384
385#define for_each_clear_bit(b, p, max)for ((b) = find_first_zero_bit((p), (max)); (b) < (max); (
b) = find_next_zero_bit((p), (max), (b) + 1)) \
386 for ((b) = find_first_zero_bit((p), (max)); \
387 (b) < (max); \
388 (b) = find_next_zero_bit((p), (max), (b) + 1))
389
390#if defined(__i386__)
391#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __asm __volatilevolatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
392#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __asm __volatilevolatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
393#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __asm __volatilevolatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
394#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
) __asm __volatilevolatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
395#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
396#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
397#define __smp_store_mb(var, value)do { (void)__sync_lock_test_and_set(&var, value); } while
(0) do { (void)xchg(&var, value)__sync_lock_test_and_set(&var, value); } while (0)
398#define smp_mb__after_atomic()do { } while (0) do { } while (0)
399#define smp_mb__before_atomic()do { } while (0) do { } while (0)
400#elif defined(__amd64__1)
401#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("lfence")do { __asm volatile("lfence" ::: "memory"); } while (0)
402#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("sfence")do { __asm volatile("sfence" ::: "memory"); } while (0)
403#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("mfence")do { __asm volatile("mfence" ::: "memory"); } while (0)
404#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
) __asm __volatilevolatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc")
405#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
406#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("")do { __asm volatile("" ::: "memory"); } while (0)
407#define __smp_store_mb(var, value)do { (void)__sync_lock_test_and_set(&var, value); } while
(0) do { (void)xchg(&var, value)__sync_lock_test_and_set(&var, value); } while (0)
408#define smp_mb__after_atomic()do { } while (0) do { } while (0)
409#define smp_mb__before_atomic()do { } while (0) do { } while (0)
410#elif defined(__aarch64__)
411#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("dsb ld")do { __asm volatile("dsb ld" ::: "memory"); } while (0)
412#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("dsb st")do { __asm volatile("dsb st" ::: "memory"); } while (0)
413#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
414#elif defined(__mips64__)
415#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) mips_sync()
416#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) mips_sync()
417#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) mips_sync()
418#elif defined(__powerpc64__)
419#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
420#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
421#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
422#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("lwsync")do { __asm volatile("lwsync" ::: "memory"); } while (0)
423#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("lwsync")do { __asm volatile("lwsync" ::: "memory"); } while (0)
424#elif defined(__powerpc__)
425#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
426#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
427#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
428#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("eieio")do { __asm volatile("eieio" ::: "memory"); } while (0)
429#elif defined(__riscv)
430#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) __membar("fence ir,ir")do { __asm volatile("fence ir,ir" ::: "memory"); } while (0)
431#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) __membar("fence ow,ow")do { __asm volatile("fence ow,ow" ::: "memory"); } while (0)
432#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) __membar("fence iorw,iorw")do { __asm volatile("fence iorw,iorw" ::: "memory"); } while (
0)
433#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("fence r,r")do { __asm volatile("fence r,r" ::: "memory"); } while (0)
434#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) __membar("fence w,w")do { __asm volatile("fence w,w" ::: "memory"); } while (0)
435#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
) __membar("fence rw,rw")do { __asm volatile("fence rw,rw" ::: "memory"); } while (0)
436#elif defined(__sparc64__)
437#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0) membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
438#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0) membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
439#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0) membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
440#endif
441
442#ifndef smp_rmb
443#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0) rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)
444#endif
445
446#ifndef smp_wmb
447#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0) wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)
448#endif
449
450#ifndef mmiowb
451#define mmiowb()do { __asm volatile("sfence" ::: "memory"); } while (0) wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)
452#endif
453
454#ifndef smp_mb__before_atomic
455#define smp_mb__before_atomic()do { } while (0) mb()do { __asm volatile("mfence" ::: "memory"); } while (0)
456#endif
457
458#ifndef smp_mb__after_atomic
459#define smp_mb__after_atomic()do { } while (0) mb()do { __asm volatile("mfence" ::: "memory"); } while (0)
460#endif
461
462#ifndef smp_store_mb
463#define smp_store_mb(x, v)do { x = v; do { __asm volatile("mfence" ::: "memory"); } while
(0); } while (0) do { x = v; mb()do { __asm volatile("mfence" ::: "memory"); } while (0); } while (0)
464#endif
465
466#endif