/usr/src/sys/dev/pci/drm/include/linux/atomic.h

Bug Summary

File:	dev/pci/drm/include/linux/atomic.h
Warning:	line 291, column 42 The left operand of '&' is a garbage value

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name i915_gem.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/dev/pci/drm/i915/i915_gem.c

/usr/src/sys/dev/pci/drm/i915/i915_gem.c

→

1/*
* Copyright © 2008-2015 Intel Corporation
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*
* Authors:
*    Eric Anholt <eric@anholt.net>
*
*/

28#include <linux/dma-fence-array.h>
29#include <linux/kthread.h>
30#include <linux/dma-resv.h>
31#include <linux/shmem_fs.h>
32#include <linux/slab.h>
33#include <linux/stop_machine.h>
34#include <linux/swap.h>
35#include <linux/pci.h>
36#include <linux/dma-buf.h>
37#include <linux/mman.h>

39#include <drm/drm_cache.h>
40#include <drm/drm_vma_manager.h>

42#include <dev/pci/agpvar.h>

44#include "display/intel_display.h"
45#include "display/intel_frontbuffer.h"

47#include "gem/i915_gem_clflush.h"
48#include "gem/i915_gem_context.h"
49#include "gem/i915_gem_ioctls.h"
50#include "gem/i915_gem_mman.h"
51#include "gem/i915_gem_pm.h"
52#include "gem/i915_gem_region.h"
53#include "gem/i915_gem_userptr.h"
54#include "gt/intel_engine_user.h"
55#include "gt/intel_gt.h"
56#include "gt/intel_gt_pm.h"
57#include "gt/intel_workarounds.h"

59#include "i915_drv.h"
60#include "i915_file_private.h"
61#include "i915_trace.h"
62#include "i915_vgpu.h"
63#include "intel_pm.h"

65static int
66insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
67{
int err;

err = mutex_lock_interruptible(&ggtt->vm.mutex);
if (err)
return err;

memset(node, 0, sizeof(*node))__builtin_memset((node), (0), (sizeof(*node)));
err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
			  size, 0, I915_COLOR_UNEVICTABLE(-1),
			  0, ggtt->mappable_end,
			  DRM_MM_INSERT_LOW);

mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);

return err;
83}

85static void
86remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
87{
mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex);
drm_mm_remove_node(node);
mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);
91}

93int
94i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
	    struct drm_file *file)
96{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
struct drm_i915_gem_get_aperture *args = data;
struct i915_vma *vma;
u64 pinned;

if (mutex_lock_interruptible(&ggtt->vm.mutex))
return -EINTR4;

pinned = ggtt->vm.reserved;
list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link
 ) *__mptr = ((&ggtt->vm.bound_list)->next); (__typeof
(*vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma)
, vm_link) );}); &vma->vm_link != (&ggtt->vm.bound_list
); vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link
 ) *__mptr = (vma->vm_link.next); (__typeof(*vma) *)( (char
 *)__mptr - __builtin_offsetof(__typeof(*vma), vm_link) );}))
if (i915_vma_is_pinned(vma))
	pinned += vma->node.size;

mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);

args->aper_size = ggtt->vm.total;
args->aper_available_size = args->aper_size - pinned;

return 0;
117}

119int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
	   unsigned long flags)
121{
struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
bool_Bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK(1UL << (3)));
DRM_LIST_HEAD(still_in_list)struct list_head still_in_list = { &(still_in_list), &
(still_in_list) };
intel_wakeref_t wakeref;
struct i915_vma *vma;
int ret;

assert_object_held(obj)do { (void)(&((obj)->base.resv)->lock.base); } while
(0);

if (list_empty(&obj->vma.list))
return 0;

/*
* As some machines use ACPI to handle runtime-resume callbacks, and
* ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
* as they are required by the shrinker. Ergo, we wake the device up
* first just in case.
*/
wakeref = intel_runtime_pm_get(rpm);

142try_again:
ret = 0;
spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,(list_empty(&obj->vma.list) ? ((void *)0) : ({ const __typeof
( ((struct i915_vma *)0)->obj_link ) *__mptr = ((&obj->
vma.list)->next); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_link) );}))
				       struct i915_vma,(list_empty(&obj->vma.list) ? ((void *)0) : ({ const __typeof
( ((struct i915_vma *)0)->obj_link ) *__mptr = ((&obj->
vma.list)->next); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_link) );}))
				       obj_link)(list_empty(&obj->vma.list) ? ((void *)0) : ({ const __typeof
( ((struct i915_vma *)0)->obj_link ) *__mptr = ((&obj->
vma.list)->next); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof
(struct i915_vma, obj_link) );})))) {
list_move_tail(&vma->obj_link, &still_in_list);
if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11))))))
	continue;

if (flags & I915_GEM_OBJECT_UNBIND_TEST(1UL << (2))) {
	ret = -EBUSY16;
	break;
}

/*
 * Requiring the vm destructor to take the object lock
 * before destroying a vma would help us eliminate the
 * i915_vm_tryget() here, AND thus also the barrier stuff
 * at the end. That's an easy fix, but sleeping locks in
 * a kthread should generally be avoided.
 */
ret = -EAGAIN35;
if (!i915_vm_tryget(vma->vm))
	break;

spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);

/*
 * Since i915_vma_parked() takes the object lock
 * before vma destruction, it won't race us here,
 * and destroy the vma from under us.
 */

ret = -EBUSY16;
if (flags & I915_GEM_OBJECT_UNBIND_ASYNC(1UL << (4))) {
	assert_object_held(vma->obj)do { (void)(&((vma->obj)->base.resv)->lock.base)
; } while(0);
	ret = i915_vma_unbind_async(vma, vm_trylock);
}

if (ret == -EBUSY16 && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE(1UL << (0)) ||
		      !i915_vma_is_active(vma))) {
	if (vm_trylock) {
		if (mutex_trylock(&vma->vm->mutex)(rw_enter(&vma->vm->mutex, 0x0001UL | 0x0040UL) == 0
)) {
			ret = __i915_vma_unbind(vma);
			mutex_unlock(&vma->vm->mutex)rw_exit_write(&vma->vm->mutex);
		}
	} else {
		ret = i915_vma_unbind(vma);
	}
}

i915_vm_put(vma->vm);
spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
}
list_splice_init(&still_in_list, &obj->vma.list);
spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);

if (ret == -EAGAIN35 && flags & I915_GEM_OBJECT_UNBIND_BARRIER(1UL << (1))) {
rcu_barrier()__asm volatile("" : : : "memory"); /* flush the i915_vm_release() */
goto try_again;
}

intel_runtime_pm_put(rpm, wakeref);

return ret;
208}

210static int
211shmem_pread(struct vm_page *page, int offset, int len, char __user *user_data,
   bool_Bool needs_clflush)
213{
char *vaddr;
int ret;

vaddr = kmap(page);

if (needs_clflush)
drm_clflush_virt_range(vaddr + offset, len);

ret = __copy_to_user(user_data, vaddr + offset, len);

kunmap_va(vaddr);

return ret ? -EFAULT14 : 0;
227}

229static int
230i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
     struct drm_i915_gem_pread *args)
232{
unsigned int needs_clflush;
unsigned int idx, offset;
char __user *user_data;
u64 remain;
int ret;

ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0));
if (ret)
return ret;

ret = i915_gem_object_pin_pages(obj);
if (ret)
goto err_unlock;

ret = i915_gem_object_prepare_read(obj, &needs_clflush);
if (ret)
goto err_unpin;

i915_gem_object_finish_access(obj);
i915_gem_object_unlock(obj);

remain = args->size;
user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr));
offset = offset_in_page(args->offset)((vaddr_t)(args->offset) & ((1 << 12) - 1));
for (idx = args->offset >> PAGE_SHIFT12; remain; idx++) {
struct vm_page *page = i915_gem_object_get_page(obj, idx);
unsigned int length = min_t(u64, remain, PAGE_SIZE - offset)({ u64 __min_a = (remain); u64 __min_b = ((1 << 12) - offset
); __min_a < __min_b ? __min_a : __min_b; });

ret = shmem_pread(page, offset, length, user_data,
		  needs_clflush);
if (ret)
	break;

remain -= length;
user_data += length;
offset = 0;
}

i915_gem_object_unpin_pages(obj);
return ret;

274err_unpin:
i915_gem_object_unpin_pages(obj);
276err_unlock:
i915_gem_object_unlock(obj);
return ret;
279}

281#ifdef __linux__
282static inline bool_Bool
283gtt_user_read(struct io_mapping *mapping,
     loff_t base, int offset,
     char __user *user_data, int length)
286{
void __iomem *vaddr;
unsigned long unwritten;

/* We can use the cpu mem copy function because this is X86. */
vaddr = io_mapping_map_atomic_wc(mapping, base);
unwritten = __copy_to_user_inatomic(user_data,
			    (void __force *)vaddr + offset,
			    length);
io_mapping_unmap_atomic(vaddr);
if (unwritten) {
vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE(1 << 12));
unwritten = copy_to_user(user_data,
			 (void __force *)vaddr + offset,
			 length);
io_mapping_unmap(vaddr);
}
return unwritten;
304}
305#else
306static inline bool_Bool
307gtt_user_read(struct drm_i915_privateinteldrm_softc *dev_priv,
     loff_t base, int offset,
     char __user *user_data, int length)
310{
bus_space_handle_t bsh;
void __iomem *vaddr;
unsigned long unwritten;

/* We can use the cpu mem copy function because this is X86. */
agp_map_atomic(dev_priv->agph, base, &bsh);
vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh)));
unwritten = __copy_to_user_inatomic(user_data,
			    (void __force *)vaddr + offset,
			    length);
agp_unmap_atomic(dev_priv->agph, bsh);
if (unwritten) {
agp_map_subregion(dev_priv->agph, base, PAGE_SIZE(1 << 12), &bsh);
vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh)));
unwritten = copy_to_user(user_data,
			 (void __force *)vaddr + offset,
			 length);
agp_unmap_subregion(dev_priv->agph, bsh, PAGE_SIZE(1 << 12));
}
return unwritten;
331}
332#endif

334static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
			     struct drm_mm_node *node,
			     bool_Bool write)
337{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
struct i915_vma *vma;
struct i915_gem_ww_ctx ww;
int ret;

i915_gem_ww_ctx_init(&ww, true1);
345retry:
vma = ERR_PTR(-ENODEV19);
ret = i915_gem_object_lock(obj, &ww);
if (ret)
2
←
Assuming 'ret' is not equal to 0→
3
←
Taking true branch→
goto err_ww;
4
←
Control jumps to line 385→

ret = i915_gem_object_set_to_gtt_domain(obj, write);
if (ret)
goto err_ww;

if (!i915_gem_object_is_tiled(obj))
vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL((void *)0), 0, 0,
				  PIN_MAPPABLE(1ULL << (3)) |
				  PIN_NONBLOCK(1ULL << (2)) /* NOWARN */ |
				  PIN_NOEVICT(1ULL << (0)));
if (vma == ERR_PTR(-EDEADLK11)) {
ret = -EDEADLK11;
goto err_ww;
} else if (!IS_ERR(vma)) {
node->start = i915_ggtt_offset(vma);
node->flags = 0;
} else {
ret = insert_mappable_node(ggtt, node, PAGE_SIZE(1 << 12));
if (ret)
	goto err_ww;
GEM_BUG_ON(!drm_mm_node_allocated(node))((void)0);
vma = NULL((void *)0);
}

ret = i915_gem_object_pin_pages(obj);
if (ret) {
if (drm_mm_node_allocated(node)) {
	ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
	remove_mappable_node(ggtt, node);
} else {
	i915_vma_unpin(vma);
}
}

384err_ww:
if (ret == -EDEADLK11) {
5
←
Assuming the condition is false→
6
←
Taking false branch→
ret = i915_gem_ww_ctx_backoff(&ww);
if (!ret)
	goto retry;
}
i915_gem_ww_ctx_fini(&ww);

return ret6.1
'ret' is not equal to 0
1
'ret' is not equal to 0
1
'ret' is not equal to 0
 ? ERR_PTR(ret) : vma;
7
←
'?' condition is true→
393}

395static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
		 struct drm_mm_node *node,
		 struct i915_vma *vma)
398{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
struct i915_ggtt *ggtt = to_gt(i915)->ggtt;

i915_gem_object_unpin_pages(obj);
if (drm_mm_node_allocated(node)) {
13
←
Calling 'drm_mm_node_allocated'→
ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
remove_mappable_node(ggtt, node);
} else {
i915_vma_unpin(vma);
}
409}

411static int
412i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
   const struct drm_i915_gem_pread *args)
414{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
intel_wakeref_t wakeref;
struct drm_mm_node node;
void __user *user_data;
struct i915_vma *vma;
u64 remain, offset;
int ret = 0;

wakeref = intel_runtime_pm_get(&i915->runtime_pm);

vma = i915_gem_gtt_prepare(obj, &node, false0);
1
Calling 'i915_gem_gtt_prepare'→
8
←
Returning from 'i915_gem_gtt_prepare'→
if (IS_ERR(vma)) {
9
←
Taking false branch→
ret = PTR_ERR(vma);
goto out_rpm;
}

user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr));
remain = args->size;
offset = args->offset;

while (remain > 0) {
10
←
Assuming 'remain' is <= 0→
11
←
Loop condition is false. Execution continues on line 466→
/* Operation in this page
 *
 * page_base = page offset within aperture
 * page_offset = offset within page
 * page_length = bytes to copy for this page
 */
u32 page_base = node.start;
unsigned page_offset = offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1));
unsigned page_length = PAGE_SIZE(1 << 12) - page_offset;
page_length = remain < page_length ? remain : page_length;
if (drm_mm_node_allocated(&node)) {
	ggtt->vm.insert_page(&ggtt->vm,
			     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT12),
			     node.start, I915_CACHE_NONE, 0);
} else {
	page_base += offset & LINUX_PAGE_MASK(~((1 << 12) - 1));
}

if (gtt_user_read(i915, page_base, page_offset,
		  user_data, page_length)) {
	ret = -EFAULT14;
	break;
}

remain -= page_length;
user_data += page_length;
offset += page_length;
}

i915_gem_gtt_cleanup(obj, &node, vma);
12
←
Calling 'i915_gem_gtt_cleanup'→
467out_rpm:
intel_runtime_pm_put(&i915->runtime_pm, wakeref);
return ret;
470}

472/**
* Reads data from the object referenced by handle.
* @dev: drm device pointer
* @data: ioctl data blob
* @file: drm file pointer
*
* On error, the contents of *data are undefined.
*/
480int
481i915_gem_pread_ioctl(struct drm_device *dev, void *data,
     struct drm_file *file)
483{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct drm_i915_gem_pread *args = data;
struct drm_i915_gem_object *obj;
int ret;

/* PREAD is disallowed for all platforms after TGL-LP.  This also
* covers all platforms with local memory.
*/
if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 12 && !IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE))
return -EOPNOTSUPP45;

if (args->size == 0)
return 0;

if (!access_ok(u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)),
       args->size))
return -EFAULT14;

obj = i915_gem_object_lookup(file, args->handle);
if (!obj)
return -ENOENT2;

/* Bounds check source.  */
if (range_overflows_t(u64, args->offset, args->size, obj->base.size)({ typeof((u64)(args->offset)) start__ = ((u64)(args->offset
)); typeof((u64)(args->size)) size__ = ((u64)(args->size
)); typeof((u64)(obj->base.size)) max__ = ((u64)(obj->base
.size)); (void)(&start__ == &size__); (void)(&start__
 == &max__); start__ >= max__ || size__ > max__ - start__
; })) {
ret = -EINVAL22;
goto out;
}

trace_i915_gem_object_pread(obj, args->offset, args->size);
ret = -ENODEV19;
if (obj->ops->pread)
ret = obj->ops->pread(obj, args);
if (ret != -ENODEV19)
goto out;

ret = i915_gem_object_wait(obj,
		   I915_WAIT_INTERRUPTIBLE(1UL << (0)),
		   MAX_SCHEDULE_TIMEOUT(0x7fffffff));
if (ret)
goto out;

ret = i915_gem_shmem_pread(obj, args);
if (ret == -EFAULT14 || ret == -ENODEV19)
ret = i915_gem_gtt_pread(obj, args);

529out:
i915_gem_object_put(obj);
return ret;
532}

534/* This is the fast write path which cannot handle
* page faults in the source data
*/
537#ifdef __linux__
538static inline bool_Bool
539ggtt_write(struct io_mapping *mapping,
  loff_t base, int offset,
  char __user *user_data, int length)
542{
void __iomem *vaddr;
unsigned long unwritten;

/* We can use the cpu mem copy function because this is X86. */
vaddr = io_mapping_map_atomic_wc(mapping, base);
unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
				      user_data, length);
io_mapping_unmap_atomic(vaddr);
if (unwritten) {
vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE(1 << 12));
unwritten = copy_from_user((void __force *)vaddr + offset,
			   user_data, length);
io_mapping_unmap(vaddr);
}

return unwritten;
559}
560#else
561static inline bool_Bool
562ggtt_write(struct drm_i915_privateinteldrm_softc *dev_priv,
  loff_t base, int offset,
  char __user *user_data, int length)
565{
bus_space_handle_t bsh;
void __iomem *vaddr;
unsigned long unwritten;

/* We can use the cpu mem copy function because this is X86. */
agp_map_atomic(dev_priv->agph, base, &bsh);
vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh)));
unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
				      user_data, length);
agp_unmap_atomic(dev_priv->agph, bsh);
if (unwritten) {
agp_map_subregion(dev_priv->agph, base, PAGE_SIZE(1 << 12), &bsh);
vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh)));
unwritten = copy_from_user((void __force *)vaddr + offset,
			   user_data, length);
agp_unmap_subregion(dev_priv->agph, bsh, PAGE_SIZE(1 << 12));
}

return unwritten;
585}
586#endif

588/**
* This is the fast pwrite path, where we copy the data directly from the
* user into the GTT, uncached.
* @obj: i915 GEM object
* @args: pwrite arguments structure
*/
594static int
595i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
	 const struct drm_i915_gem_pwrite *args)
597{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
struct intel_runtime_pm *rpm = &i915->runtime_pm;
intel_wakeref_t wakeref;
struct drm_mm_node node;
struct i915_vma *vma;
u64 remain, offset;
void __user *user_data;
int ret = 0;

if (i915_gem_object_has_struct_page(obj)) {
/*
 * Avoid waking the device up if we can fallback, as
 * waking/resuming is very slow (worst-case 10-100 ms
 * depending on PCI sleeps and our own resume time).
 * This easily dwarfs any performance advantage from
 * using the cache bypass of indirect GGTT access.
 */
wakeref = intel_runtime_pm_get_if_in_use(rpm);
if (!wakeref)
	return -EFAULT14;
} else {
/* No backing pages, no fallback, we must force GGTT access */
wakeref = intel_runtime_pm_get(rpm);
}

vma = i915_gem_gtt_prepare(obj, &node, true1);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto out_rpm;
}

i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);

user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr));
offset = args->offset;
remain = args->size;
while (remain) {
/* Operation in this page
 *
 * page_base = page offset within aperture
 * page_offset = offset within page
 * page_length = bytes to copy for this page
 */
u32 page_base = node.start;
unsigned int page_offset = offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1));
unsigned int page_length = PAGE_SIZE(1 << 12) - page_offset;
page_length = remain < page_length ? remain : page_length;
if (drm_mm_node_allocated(&node)) {
	/* flush the write before we modify the GGTT */
	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
	ggtt->vm.insert_page(&ggtt->vm,
			     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT12),
			     node.start, I915_CACHE_NONE, 0);
	wmb()do { __asm volatile("sfence" ::: "memory"); } while (0); /* flush modifications to the GGTT (insert_page) */
} else {
	page_base += offset & LINUX_PAGE_MASK(~((1 << 12) - 1));
}
/* If we get a fault while copying data, then (presumably) our
 * source page isn't available.  Return the error and we'll
 * retry in the slow path.
 * If the object is non-shmem backed, we retry again with the
 * path that handles page fault.
 */
if (ggtt_write(i915, page_base, page_offset,
	       user_data, page_length)) {
	ret = -EFAULT14;
	break;
}

remain -= page_length;
user_data += page_length;
offset += page_length;
}

intel_gt_flush_ggtt_writes(ggtt->vm.gt);
i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);

i915_gem_gtt_cleanup(obj, &node, vma);
677out_rpm:
intel_runtime_pm_put(rpm, wakeref);
return ret;
680}

682/* Per-page copy function for the shmem pwrite fastpath.
* Flushes invalid cachelines before writing to the target if
* needs_clflush_before is set and flushes out any written cachelines after
* writing if needs_clflush is set.
*/
687static int
688shmem_pwrite(struct vm_page *page, int offset, int len, char __user *user_data,
    bool_Bool needs_clflush_before,
    bool_Bool needs_clflush_after)
691{
char *vaddr;
int ret;

vaddr = kmap(page);

if (needs_clflush_before)
drm_clflush_virt_range(vaddr + offset, len);

ret = __copy_from_user(vaddr + offset, user_data, len);
if (!ret && needs_clflush_after)
drm_clflush_virt_range(vaddr + offset, len);

kunmap_va(vaddr);

return ret ? -EFAULT14 : 0;
707}

709static int
710i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
      const struct drm_i915_gem_pwrite *args)
712{
unsigned int partial_cacheline_write;
unsigned int needs_clflush;
unsigned int offset, idx;
void __user *user_data;
u64 remain;
int ret;

ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0));
if (ret)
return ret;

ret = i915_gem_object_pin_pages(obj);
if (ret)
goto err_unlock;

ret = i915_gem_object_prepare_write(obj, &needs_clflush);
if (ret)
goto err_unpin;

i915_gem_object_finish_access(obj);
i915_gem_object_unlock(obj);

/* If we don't overwrite a cacheline completely we need to be
* careful to have up-to-date data by first clflushing. Don't
* overcomplicate things and flush the entire patch.
*/
partial_cacheline_write = 0;
if (needs_clflush & CLFLUSH_BEFORE(1UL << (0)))
partial_cacheline_write = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_cflushsz - 1;

user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr));
remain = args->size;
offset = offset_in_page(args->offset)((vaddr_t)(args->offset) & ((1 << 12) - 1));
for (idx = args->offset >> PAGE_SHIFT12; remain; idx++) {
struct vm_page *page = i915_gem_object_get_page(obj, idx);
unsigned int length = min_t(u64, remain, PAGE_SIZE - offset)({ u64 __min_a = (remain); u64 __min_b = ((1 << 12) - offset
); __min_a < __min_b ? __min_a : __min_b; });

ret = shmem_pwrite(page, offset, length, user_data,
		   (offset | length) & partial_cacheline_write,
		   needs_clflush & CLFLUSH_AFTER(1UL << (1)));
if (ret)
	break;

remain -= length;
user_data += length;
offset = 0;
}

i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);

i915_gem_object_unpin_pages(obj);
return ret;

766err_unpin:
i915_gem_object_unpin_pages(obj);
768err_unlock:
i915_gem_object_unlock(obj);
return ret;
771}

773/**
* Writes data to the object referenced by handle.
* @dev: drm device
* @data: ioctl data blob
* @file: drm file
*
* On error, the contents of the buffer that were to be modified are undefined.
*/
781int
782i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
      struct drm_file *file)
784{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct drm_i915_gem_pwrite *args = data;
struct drm_i915_gem_object *obj;
int ret;

/* PWRITE is disallowed for all platforms after TGL-LP.  This also
* covers all platforms with local memory.
*/
if (GRAPHICS_VER(i915)((&(i915)->__runtime)->graphics.ip.ver) >= 12 && !IS_TIGERLAKE(i915)IS_PLATFORM(i915, INTEL_TIGERLAKE))
return -EOPNOTSUPP45;

if (args->size == 0)
return 0;

if (!access_ok(u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)), args->size))
return -EFAULT14;

obj = i915_gem_object_lookup(file, args->handle);
if (!obj)
return -ENOENT2;

/* Bounds check destination. */
if (range_overflows_t(u64, args->offset, args->size, obj->base.size)({ typeof((u64)(args->offset)) start__ = ((u64)(args->offset
)); typeof((u64)(args->size)) size__ = ((u64)(args->size
)); typeof((u64)(obj->base.size)) max__ = ((u64)(obj->base
.size)); (void)(&start__ == &size__); (void)(&start__
 == &max__); start__ >= max__ || size__ > max__ - start__
; })) {
ret = -EINVAL22;
goto err;
}

/* Writes not allowed into this read-only object */
if (i915_gem_object_is_readonly(obj)) {
ret = -EINVAL22;
goto err;
}

trace_i915_gem_object_pwrite(obj, args->offset, args->size);

ret = -ENODEV19;
if (obj->ops->pwrite)
ret = obj->ops->pwrite(obj, args);
if (ret != -ENODEV19)
goto err;

ret = i915_gem_object_wait(obj,
		   I915_WAIT_INTERRUPTIBLE(1UL << (0)) |
		   I915_WAIT_ALL(1UL << (2)),
		   MAX_SCHEDULE_TIMEOUT(0x7fffffff));
if (ret)
goto err;

ret = -EFAULT14;
/* We can only do the GTT pwrite on untiled buffers, as otherwise
* it would end up going through the fenced access, and we'll get
* different detiling behavior between reading and writing.
* pread/pwrite currently are reading and writing from the CPU
* perspective, requiring manual detiling by the client.
*/
if (!i915_gem_object_has_struct_page(obj) ||
   i915_gem_cpu_write_needs_clflush(obj))
/* Note that the gtt paths might fail with non-page-backed user
 * pointers (e.g. gtt mappings when moving data between
 * textures). Fallback to the shmem path in that case.
 */
ret = i915_gem_gtt_pwrite_fast(obj, args);

if (ret == -EFAULT14 || ret == -ENOSPC28) {
if (i915_gem_object_has_struct_page(obj))
	ret = i915_gem_shmem_pwrite(obj, args);
}

853err:
i915_gem_object_put(obj);
return ret;
856}

858/**
* Called when user space has done writes to this buffer
* @dev: drm device
* @data: ioctl data blob
* @file: drm file
*/
864int
865i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
	 struct drm_file *file)
867{
struct drm_i915_gem_sw_finish *args = data;
struct drm_i915_gem_object *obj;

obj = i915_gem_object_lookup(file, args->handle);
if (!obj)
return -ENOENT2;

/*
* Proxy objects are barred from CPU access, so there is no
* need to ban sw_finish as it is a nop.
*/

/* Pinned buffers may be scanout, so flush the cache */
i915_gem_object_flush_if_display(obj);
i915_gem_object_put(obj);

return 0;
885}

887void i915_gem_runtime_suspend(struct drm_i915_privateinteldrm_softc *i915)
888{
struct drm_i915_gem_object *obj, *on;
int i;

/*
* Only called during RPM suspend. All users of the userfault_list
* must be holding an RPM wakeref to ensure that this can not
* run concurrently with themselves (and use the struct_mutex for
* protection between themselves).
*/

list_for_each_entry_safe(obj, on,for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->userfault_link
 ) *__mptr = ((&to_gt(i915)->ggtt->userfault_list)->
next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}), on = ({ const __typeof
( ((__typeof(*obj) *)0)->userfault_link ) *__mptr = (obj->
userfault_link.next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}); &obj->userfault_link
 != (&to_gt(i915)->ggtt->userfault_list); obj = on,
 on = ({ const __typeof( ((__typeof(*on) *)0)->userfault_link
 ) *__mptr = (on->userfault_link.next); (__typeof(*on) *)(
 (char *)__mptr - __builtin_offsetof(__typeof(*on), userfault_link
) );}))
		 &to_gt(i915)->ggtt->userfault_list, userfault_link)for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->userfault_link
 ) *__mptr = ((&to_gt(i915)->ggtt->userfault_list)->
next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}), on = ({ const __typeof
( ((__typeof(*obj) *)0)->userfault_link ) *__mptr = (obj->
userfault_link.next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}); &obj->userfault_link
 != (&to_gt(i915)->ggtt->userfault_list); obj = on,
 on = ({ const __typeof( ((__typeof(*on) *)0)->userfault_link
 ) *__mptr = (on->userfault_link.next); (__typeof(*on) *)(
 (char *)__mptr - __builtin_offsetof(__typeof(*on), userfault_link
) );}))
__i915_gem_object_release_mmap_gtt(obj);

list_for_each_entry_safe(obj, on,for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->userfault_link
 ) *__mptr = ((&i915->runtime_pm.lmem_userfault_list)->
next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}), on = ({ const __typeof
( ((__typeof(*obj) *)0)->userfault_link ) *__mptr = (obj->
userfault_link.next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}); &obj->userfault_link
 != (&i915->runtime_pm.lmem_userfault_list); obj = on,
 on = ({ const __typeof( ((__typeof(*on) *)0)->userfault_link
 ) *__mptr = (on->userfault_link.next); (__typeof(*on) *)(
 (char *)__mptr - __builtin_offsetof(__typeof(*on), userfault_link
) );}))
		 &i915->runtime_pm.lmem_userfault_list, userfault_link)for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->userfault_link
 ) *__mptr = ((&i915->runtime_pm.lmem_userfault_list)->
next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}), on = ({ const __typeof
( ((__typeof(*obj) *)0)->userfault_link ) *__mptr = (obj->
userfault_link.next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*obj), userfault_link) );}); &obj->userfault_link
 != (&i915->runtime_pm.lmem_userfault_list); obj = on,
 on = ({ const __typeof( ((__typeof(*on) *)0)->userfault_link
 ) *__mptr = (on->userfault_link.next); (__typeof(*on) *)(
 (char *)__mptr - __builtin_offsetof(__typeof(*on), userfault_link
) );}))
i915_gem_object_runtime_pm_release_mmap_offset(obj);

/*
* The fence will be lost when the device powers down. If any were
* in use by hardware (i.e. they are pinned), we should not be powering
* down! All other fences will be reacquired by the user upon waking.
*/
for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {
struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];

/*
 * Ideally we want to assert that the fence register is not
 * live at this point (i.e. that no piece of code will be
 * trying to write through fence + GTT, as that both violates
 * our tracking of activity and associated locking/barriers,
 * but also is illegal given that the hw is powered down).
 *
 * Previously we used reg->pin_count as a "liveness" indicator.
 * That is not sufficient, and we need a more fine-grained
 * tool if we want to have a sanity check here.
 */

if (!reg->vma)
	continue;

GEM_BUG_ON(i915_vma_has_userfault(reg->vma))((void)0);
reg->dirty = true1;
}
933}

935static void discard_ggtt_vma(struct i915_vma *vma)
936{
struct drm_i915_gem_object *obj = vma->obj;

spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock);
if (!RB_EMPTY_NODE(&vma->obj_node)((&vma->obj_node)->__entry.rbe_parent == &vma->
obj_node)) {
rb_erase(&vma->obj_node, &obj->vma.tree)linux_root_RB_REMOVE((struct linux_root *)(&obj->vma.tree
), (&vma->obj_node));
RB_CLEAR_NODE(&vma->obj_node)(((&vma->obj_node))->__entry.rbe_parent = (&vma
->obj_node));
}
spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock);
945}

947struct i915_vma *
948i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
	    struct i915_gem_ww_ctx *ww,
	    const struct i915_gtt_view *view,
	    u64 size, u64 alignment, u64 flags)
952{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev);
struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
struct i915_vma *vma;
int ret;

GEM_WARN_ON(!ww)({ __builtin_expect(!!(!!(!ww)), 0); });

if (flags & PIN_MAPPABLE(1ULL << (3)) &&
   (!view || view->type == I915_GTT_VIEW_NORMAL)) {
/*
 * If the required space is larger than the available
 * aperture, we will not able to find a slot for the
 * object and unbinding the object now will be in
 * vain. Worse, doing so may cause us to ping-pong
 * the object in and out of the Global GTT and
 * waste a lot of cycles under the mutex.
 */
if (obj->base.size > ggtt->mappable_end)
	return ERR_PTR(-E2BIG7);

/*
 * If NONBLOCK is set the caller is optimistically
 * trying to cache the full object within the mappable
 * aperture, and *must* have a fallback in place for
 * situations where we cannot bind the object. We
 * can be a little more lax here and use the fallback
 * more often to avoid costly migrations of ourselves
 * and other objects within the aperture.
 *
 * Half-the-aperture is used as a simple heuristic.
 * More interesting would to do search for a free
 * block prior to making the commitment to unbind.
 * That caters for the self-harm case, and with a
 * little more heuristics (e.g. NOFAULT, NOEVICT)
 * we could try to minimise harm to others.
 */
if (flags & PIN_NONBLOCK(1ULL << (2)) &&
    obj->base.size > ggtt->mappable_end / 2)
	return ERR_PTR(-ENOSPC28);
}

994new_vma:
vma = i915_vma_instance(obj, &ggtt->vm, view);
if (IS_ERR(vma))
return vma;

if (i915_vma_misplaced(vma, size, alignment, flags)) {
if (flags & PIN_NONBLOCK(1ULL << (2))) {
	if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
		return ERR_PTR(-ENOSPC28);

	/*
	 * If this misplaced vma is too big (i.e, at-least
	 * half the size of aperture) or hasn't been pinned
	 * mappable before, we ignore the misplacement when
	 * PIN_NONBLOCK is set in order to avoid the ping-pong
	 * issue described above. In other words, we try to
	 * avoid the costly operation of unbinding this vma
	 * from the GGTT and rebinding it back because there
	 * may not be enough space for this vma in the aperture.
	 */
	if (flags & PIN_MAPPABLE(1ULL << (3)) &&
	    (vma->fence_size > ggtt->mappable_end / 2 ||
	    !i915_vma_is_map_and_fenceable(vma)))
		return ERR_PTR(-ENOSPC28);
}

if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
	discard_ggtt_vma(vma);
	goto new_vma;
}

ret = i915_vma_unbind(vma);
if (ret)
	return ERR_PTR(ret);
}

ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL(1ULL << (10)));

if (ret)
return ERR_PTR(ret);

if (vma->fence && !i915_gem_object_is_tiled(obj)) {
mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex);
i915_vma_revoke_fence(vma);
mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex);
}

ret = i915_vma_wait_for_bind(vma);
if (ret) {
i915_vma_unpin(vma);
return ERR_PTR(ret);
}

return vma;
1048}

1050struct i915_vma * __must_check
1051i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
	 const struct i915_gtt_view *view,
	 u64 size, u64 alignment, u64 flags)
1054{
struct i915_gem_ww_ctx ww;
struct i915_vma *ret;
int err;

for_i915_gem_ww(&ww, err, true)for (i915_gem_ww_ctx_init(&ww, 1), (err) = -11; (err) == -
11; (err) = __i915_gem_ww_fini(&ww, err)) {
err = i915_gem_object_lock(obj, &ww);
if (err)
	continue;

ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,
				  alignment, flags);
if (IS_ERR(ret))
	err = PTR_ERR(ret);
}

return err ? ERR_PTR(err) : ret;
1071}

1073int
1074i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
       struct drm_file *file_priv)
1076{
struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev);
struct drm_i915_gem_madvise *args = data;
struct drm_i915_gem_object *obj;
int err;

switch (args->madv) {
case I915_MADV_DONTNEED1:
case I915_MADV_WILLNEED0:
   break;
default:
   return -EINVAL22;
}

obj = i915_gem_object_lookup(file_priv, args->handle);
if (!obj)
return -ENOENT2;

err = i915_gem_object_lock_interruptible(obj, NULL((void *)0));
if (err)
goto out;

if (i915_gem_object_has_pages(obj) &&
   i915_gem_object_is_tiled(obj) &&
   i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES(1UL << (0))) {
if (obj->mm.madv == I915_MADV_WILLNEED0) {
	GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj))((void)0);
	i915_gem_object_clear_tiling_quirk(obj);
	i915_gem_object_make_shrinkable(obj);
}
if (args->madv == I915_MADV_WILLNEED0) {
	GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj))((void)0);
	i915_gem_object_make_unshrinkable(obj);
	i915_gem_object_set_tiling_quirk(obj);
}
}

if (obj->mm.madv != __I915_MADV_PURGED2) {
obj->mm.madv = args->madv;
if (obj->ops->adjust_lru)
	obj->ops->adjust_lru(obj);
}

if (i915_gem_object_has_pages(obj) ||
   i915_gem_object_has_self_managed_shrink_list(obj)) {
unsigned long flags;

spin_lock_irqsave(&i915->mm.obj_lock, flags)do { flags = 0; mtx_enter(&i915->mm.obj_lock); } while
 (0);
if (!list_empty(&obj->mm.link)) {
	struct list_head *list;

	if (obj->mm.madv != I915_MADV_WILLNEED0)
		list = &i915->mm.purge_list;
	else
		list = &i915->mm.shrink_list;
	list_move_tail(&obj->mm.link, list);

}
spin_unlock_irqrestore(&i915->mm.obj_lock, flags)do { (void)(flags); mtx_leave(&i915->mm.obj_lock); } while
 (0);
}

/* if the object is no longer attached, discard its backing storage */
if (obj->mm.madv == I915_MADV_DONTNEED1 &&
   !i915_gem_object_has_pages(obj))
i915_gem_object_truncate(obj);

args->retained = obj->mm.madv != __I915_MADV_PURGED2;

i915_gem_object_unlock(obj);
1145out:
i915_gem_object_put(obj);
return err;
1148}

1150/*
* A single pass should suffice to release all the freed objects (along most
* call paths), but be a little more paranoid in that freeing the objects does
* take a little amount of time, during which the rcu callbacks could have added
* new objects into the freed list, and armed the work again.
*/
1156void i915_gem_drain_freed_objects(struct drm_i915_privateinteldrm_softc *i915)
1157{
while (atomic_read(&i915->mm.free_count)({ typeof(*(&i915->mm.free_count)) __tmp = *(volatile typeof
(*(&i915->mm.free_count)) *)&(*(&i915->mm.free_count
)); membar_datadep_consumer(); __tmp; })) {
flush_work(&i915->mm.free_work);
flush_delayed_work(&i915->bdev.wq);
rcu_barrier()__asm volatile("" : : : "memory");
}
1163}

1165/*
* Similar to objects above (see i915_gem_drain_freed-objects), in general we
* have workers that are armed by RCU and then rearm themselves in their
* callbacks. To be paranoid, we need to drain the workqueue a second time after
* waiting for the RCU grace period so that we catch work queued via RCU from
* the first pass. As neither drain_workqueue() nor flush_workqueue() report a
* result, we make an assumption that we only don't require more than 3 passes
* to catch all _recursive_ RCU delayed work.
*/
1174void i915_gem_drain_workqueue(struct drm_i915_privateinteldrm_softc *i915)
1175{
int i;

for (i = 0; i < 3; i++) {
flush_workqueue(i915->wq);
rcu_barrier()__asm volatile("" : : : "memory");
i915_gem_drain_freed_objects(i915);
}

drain_workqueue(i915->wq);
1185}

1187int i915_gem_init(struct drm_i915_privateinteldrm_softc *dev_priv)
1188{
struct intel_gt *gt;
unsigned int i;
int ret;

/* We need to fallback to 4K pages if host doesn't support huge gtt. */
if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
RUNTIME_INFO(dev_priv)(&(dev_priv)->__runtime)->page_sizes = I915_GTT_PAGE_SIZE_4K(1ULL << (12));

ret = i915_gem_init_userptr(dev_priv);
if (ret)
return ret;

intel_uc_fetch_firmwares(&to_gt(dev_priv)->uc);
intel_wopcm_init(&dev_priv->wopcm);

ret = i915_init_ggtt(dev_priv);
if (ret) {
GEM_BUG_ON(ret == -EIO)((void)0);
goto err_unlock;
}

/*
* Despite its name intel_init_clock_gating applies both display
* clock gating workarounds; GT mmio workarounds and the occasional
* GT power context workaround. Worse, sometimes it includes a context
* register workaround which we need to apply before we record the
* default HW state for all contexts.
*
* FIXME: break up the workarounds and apply them at the right time!
*/
intel_init_clock_gating(dev_priv);

for_each_gt(gt, dev_priv, i)for ((i) = 0; (i) < 4; (i)++) if (!(((gt) = (dev_priv)->
gt[(i)]))) {} else {
ret = intel_gt_init(gt);
if (ret)
	goto err_unlock;
}

return 0;

/*
* Unwinding is complicated by that we want to handle -EIO to mean
* disable GPU submission but keep KMS alive. We want to mark the
* HW as irrevisibly wedged, but keep enough state around that the
* driver doesn't explode during runtime.
*/
1235err_unlock:
i915_gem_drain_workqueue(dev_priv);

if (ret != -EIO5) {
for_each_gt(gt, dev_priv, i)for ((i) = 0; (i) < 4; (i)++) if (!(((gt) = (dev_priv)->
gt[(i)]))) {} else {
	intel_gt_driver_remove(gt);
	intel_gt_driver_release(gt);
	intel_uc_cleanup_firmwares(&gt->uc);
}
}

if (ret == -EIO5) {
/*
 * Allow engines or uC initialisation to fail by marking the GPU
 * as wedged. But we only want to do this when the GPU is angry,
 * for all other failure, such as an allocation failure, bail.
 */
for_each_gt(gt, dev_priv, i)for ((i) = 0; (i) < 4; (i)++) if (!(((gt) = (dev_priv)->
gt[(i)]))) {} else {
	if (!intel_gt_is_wedged(gt)) {
		i915_probe_error(dev_priv,__i915_printk(dev_priv, 0 ? "\0017" : "\0013", "Failed to initialize GPU, declaring it wedged!\n"
)
				 "Failed to initialize GPU, declaring it wedged!\n")__i915_printk(dev_priv, 0 ? "\0017" : "\0013", "Failed to initialize GPU, declaring it wedged!\n"
);
		intel_gt_set_wedged(gt);
	}
}

/* Minimal basic recovery for KMS */
ret = i915_ggtt_enable_hw(dev_priv);
i915_ggtt_resume(to_gt(dev_priv)->ggtt);
intel_init_clock_gating(dev_priv);
}

i915_gem_drain_freed_objects(dev_priv);

return ret;
1269}

1271void i915_gem_driver_register(struct drm_i915_privateinteldrm_softc *i915)
1272{
i915_gem_driver_register__shrinker(i915);

intel_engines_driver_register(i915);
1276}

1278void i915_gem_driver_unregister(struct drm_i915_privateinteldrm_softc *i915)
1279{
i915_gem_driver_unregister__shrinker(i915);
1281}

1283void i915_gem_driver_remove(struct drm_i915_privateinteldrm_softc *dev_priv)
1284{
struct intel_gt *gt;
unsigned int i;

i915_gem_suspend_late(dev_priv);
for_each_gt(gt, dev_priv, i)for ((i) = 0; (i) < 4; (i)++) if (!(((gt) = (dev_priv)->
gt[(i)]))) {} else
intel_gt_driver_remove(gt);
dev_priv->uabi_engines = RB_ROOT(struct rb_root) { ((void *)0) };

/* Flush any outstanding unpin_work. */
i915_gem_drain_workqueue(dev_priv);

i915_gem_drain_freed_objects(dev_priv);
1297}

1299void i915_gem_driver_release(struct drm_i915_privateinteldrm_softc *dev_priv)
1300{
struct intel_gt *gt;
unsigned int i;

for_each_gt(gt, dev_priv, i)for ((i) = 0; (i) < 4; (i)++) if (!(((gt) = (dev_priv)->
gt[(i)]))) {} else {
intel_gt_driver_release(gt);
intel_uc_cleanup_firmwares(&gt->uc);
}

/* Flush any outstanding work, including i915_gem_context.release_work. */
i915_gem_drain_workqueue(dev_priv);

drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list))({ int __ret = !!((!list_empty(&dev_priv->gem.contexts
.list))); if (__ret) printf("%s %s: " "%s", dev_driver_string
(((&dev_priv->drm))->dev), "", "drm_WARN_ON(" "!list_empty(&dev_priv->gem.contexts.list)"
 ")"); __builtin_expect(!!(__ret), 0); });
1313}

1315static void i915_gem_init__mm(struct drm_i915_privateinteldrm_softc *i915)
1316{
mtx_init(&i915->mm.obj_lock, IPL_TTY)do { (void)(((void *)0)); (void)(0); __mtx_init((&i915->
mm.obj_lock), ((((0x9)) > 0x0 && ((0x9)) < 0x9)
 ? 0x9 : ((0x9)))); } while (0);

init_llist_head(&i915->mm.free_list);

INIT_LIST_HEAD(&i915->mm.purge_list);
INIT_LIST_HEAD(&i915->mm.shrink_list);

i915_gem_init__objects(i915);
1325}

1327void i915_gem_init_early(struct drm_i915_privateinteldrm_softc *dev_priv)
1328{
i915_gem_init__mm(dev_priv);
i915_gem_init__contexts(dev_priv);

mtx_init(&dev_priv->display.fb_tracking.lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&dev_priv
->display.fb_tracking.lock), ((((0x0)) > 0x0 &&
 ((0x0)) < 0x9) ? 0x9 : ((0x0)))); } while (0);
1333}

1335void i915_gem_cleanup_early(struct drm_i915_privateinteldrm_softc *dev_priv)
1336{
i915_gem_drain_freed_objects(dev_priv);
GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list))((void)0);
GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count))((void)0);
drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count)({ int __ret = !!((dev_priv->mm.shrink_count)); if (__ret)
 printf("%s %s: " "%s", dev_driver_string(((&dev_priv->
drm))->dev), "", "drm_WARN_ON(" "dev_priv->mm.shrink_count"
 ")"); __builtin_expect(!!(__ret), 0); });
1341}

1343int i915_gem_open(struct drm_i915_privateinteldrm_softc *i915, struct drm_file *file)
1344{
struct drm_i915_file_private *file_priv;
struct i915_drm_client *client;
int ret = -ENOMEM12;

DRM_DEBUG("\n")___drm_dbg(((void *)0), DRM_UT_CORE, "\n");

file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL(0x0001 | 0x0004));
if (!file_priv)
goto err_alloc;

client = i915_drm_client_add(&i915->clients);
if (IS_ERR(client)) {
ret = PTR_ERR(client);
goto err_client;
}

file->driver_priv = file_priv;
file_priv->dev_priv = i915;
file_priv->file = file;
file_priv->client = client;

file_priv->bsd_engine = -1;
file_priv->hang_timestamp = jiffies;

ret = i915_gem_context_open(i915, file);
if (ret)
goto err_context;

return 0;

1375err_context:
i915_drm_client_put(client);
1377err_client:
kfree(file_priv);
1379err_alloc:
return ret;
1381}

1383#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0
1384#include "selftests/mock_gem_device.c"
1385#include "selftests/i915_gem.c"
1386#endif

←

/usr/src/sys/dev/pci/drm/include/drm/drm_mm.h

→

1/**************************************************************************
*
* Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX. USA.
* Copyright 2016 Intel Corporation
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*
*
**************************************************************************/
29/*
* Authors:
* Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
*/

34#ifndef _DRM_MM_H_
35#define _DRM_MM_H_

37/*
* Generic range manager structs
*/
40#include <linux/bug.h>
41#include <linux/rbtree.h>
42#include <linux/limits.h>
43#include <linux/mm_types.h>
44#include <linux/list.h>
45#include <linux/spinlock.h>
46#ifdef CONFIG_DRM_DEBUG_MM
47#include <linux/stackdepot.h>
48#endif
49#include <linux/types.h>

51#include <drm/drm_print.h>

53#ifdef CONFIG_DRM_DEBUG_MM
54#define DRM_MM_BUG_ON(expr)((void)0) BUG_ON(expr)((!(expr)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/dev/pci/drm/include/drm/drm_mm.h"
, 54, "!(expr)"))
55#else
56#define DRM_MM_BUG_ON(expr)((void)0) BUILD_BUG_ON_INVALID(expr)((void)0)
57#endif

59/**
* enum drm_mm_insert_mode - control search and allocation behaviour
*
* The &struct drm_mm range manager supports finding a suitable modes using
* a number of search trees. These trees are oranised by size, by address and
* in most recent eviction order. This allows the user to find either the
* smallest hole to reuse, the lowest or highest address to reuse, or simply
* reuse the most recent eviction that fits. When allocating the &drm_mm_node
* from within the hole, the &drm_mm_insert_mode also dictate whether to
* allocate the lowest matching address or the highest.
*/
70enum drm_mm_insert_mode {
/**
* @DRM_MM_INSERT_BEST:
*
* Search for the smallest hole (within the search range) that fits
* the desired node.
*
* Allocates the node from the bottom of the found hole.
*/
DRM_MM_INSERT_BEST = 0,

/**
* @DRM_MM_INSERT_LOW:
*
* Search for the lowest hole (address closest to 0, within the search
* range) that fits the desired node.
*
* Allocates the node from the bottom of the found hole.
*/
DRM_MM_INSERT_LOW,

/**
* @DRM_MM_INSERT_HIGH:
*
* Search for the highest hole (address closest to U64_MAX, within the
* search range) that fits the desired node.
*
* Allocates the node from the *top* of the found hole. The specified
* alignment for the node is applied to the base of the node
* (&drm_mm_node.start).
*/
DRM_MM_INSERT_HIGH,

/**
* @DRM_MM_INSERT_EVICT:
*
* Search for the most recently evicted hole (within the search range)
* that fits the desired node. This is appropriate for use immediately
* after performing an eviction scan (see drm_mm_scan_init()) and
* removing the selected nodes to form a hole.
*
* Allocates the node from the bottom of the found hole.
*/
DRM_MM_INSERT_EVICT,

/**
* @DRM_MM_INSERT_ONCE:
*
* Only check the first hole for suitablity and report -ENOSPC
* immediately otherwise, rather than check every hole until a
* suitable one is found. Can only be used in conjunction with another
* search method such as DRM_MM_INSERT_HIGH or DRM_MM_INSERT_LOW.
*/
DRM_MM_INSERT_ONCE = BIT(31)(1UL << (31)),

/**
* @DRM_MM_INSERT_HIGHEST:
*
* Only check the highest hole (the hole with the largest address) and
* insert the node at the top of the hole or report -ENOSPC if
* unsuitable.
*
* Does not search all holes.
*/
DRM_MM_INSERT_HIGHEST = DRM_MM_INSERT_HIGH | DRM_MM_INSERT_ONCE,

/**
* @DRM_MM_INSERT_LOWEST:
*
* Only check the lowest hole (the hole with the smallest address) and
* insert the node at the bottom of the hole or report -ENOSPC if
* unsuitable.
*
* Does not search all holes.
*/
DRM_MM_INSERT_LOWEST  = DRM_MM_INSERT_LOW | DRM_MM_INSERT_ONCE,
146};

148/**
* struct drm_mm_node - allocated block in the DRM allocator
*
* This represents an allocated block in a &drm_mm allocator. Except for
* pre-reserved nodes inserted using drm_mm_reserve_node() the structure is
* entirely opaque and should only be accessed through the provided funcions.
* Since allocation of these nodes is entirely handled by the driver they can be
* embedded.
*/
157struct drm_mm_node {
/** @color: Opaque driver-private tag. */
unsigned long color;
/** @start: Start address of the allocated block. */
u64 start;
/** @size: Size of the allocated block. */
u64 size;
/* private: */
struct drm_mm *mm;
struct list_head node_list;
struct list_head hole_stack;
struct rb_node rb;
struct rb_node rb_hole_size;
struct rb_node rb_hole_addr;
u64 __subtree_last;
u64 hole_size;
unsigned long flags;
174#define DRM_MM_NODE_ALLOCATED_BIT0	0
175#define DRM_MM_NODE_SCANNED_BIT1		1
176#ifdef CONFIG_DRM_DEBUG_MM
depot_stack_handle_t stack;
178#endif
179};

181/**
* struct drm_mm - DRM allocator
*
* DRM range allocator with a few special functions and features geared towards
* managing GPU memory. Except for the @color_adjust callback the structure is
* entirely opaque and should only be accessed through the provided functions
* and macros. This structure can be embedded into larger driver structures.
*/
189struct drm_mm {
/**
* @color_adjust:
*
* Optional driver callback to further apply restrictions on a hole. The
* node argument points at the node containing the hole from which the
* block would be allocated (see drm_mm_hole_follows() and friends). The
* other arguments are the size of the block to be allocated. The driver
* can adjust the start and end as needed to e.g. insert guard pages.
*/
void (*color_adjust)(const struct drm_mm_node *node,
	     unsigned long color,
	     u64 *start, u64 *end);

/* private: */
/* List of all memory nodes that immediately precede a free hole. */
struct list_head hole_stack;
/* head_node.node_list is the list of all memory nodes, ordered
* according to the (increasing) start address of the memory node. */
struct drm_mm_node head_node;
/* Keep an interval_tree for fast lookup of drm_mm_nodes by address. */
struct rb_root_cached interval_tree;
struct rb_root_cached holes_size;
struct rb_root holes_addr;

unsigned long scan_active;
215};

217/**
* struct drm_mm_scan - DRM allocator eviction roaster data
*
* This structure tracks data needed for the eviction roaster set up using
* drm_mm_scan_init(), and used with drm_mm_scan_add_block() and
* drm_mm_scan_remove_block(). The structure is entirely opaque and should only
* be accessed through the provided functions and macros. It is meant to be
* allocated temporarily by the driver on the stack.
*/
226struct drm_mm_scan {
/* private: */
struct drm_mm *mm;

u64 size;
u64 alignment;
u64 remainder_mask;

u64 range_start;
u64 range_end;

u64 hit_start;
u64 hit_end;

unsigned long color;
enum drm_mm_insert_mode mode;
242};

244/**
* drm_mm_node_allocated - checks whether a node is allocated
* @node: drm_mm_node to check
*
* Drivers are required to clear a node prior to using it with the
* drm_mm range manager.
*
* Drivers should use this helper for proper encapsulation of drm_mm
* internals.
*
* Returns:
* True if the @node is allocated.
*/
257static inline bool_Bool drm_mm_node_allocated(const struct drm_mm_node *node)
258{
return test_bit(DRM_MM_NODE_ALLOCATED_BIT0, &node->flags);
14
←
Calling 'test_bit'→
260}

262/**
* drm_mm_initialized - checks whether an allocator is initialized
* @mm: drm_mm to check
*
* Drivers should clear the struct drm_mm prior to initialisation if they
* want to use this function.
*
* Drivers should use this helper for proper encapsulation of drm_mm
* internals.
*
* Returns:
* True if the @mm is initialized.
*/
275static inline bool_Bool drm_mm_initialized(const struct drm_mm *mm)
276{
return READ_ONCE(mm->hole_stack.next)({ typeof(mm->hole_stack.next) __tmp = *(volatile typeof(mm
->hole_stack.next) *)&(mm->hole_stack.next); membar_datadep_consumer
(); __tmp; });
278}

280/**
* drm_mm_hole_follows - checks whether a hole follows this node
* @node: drm_mm_node to check
*
* Holes are embedded into the drm_mm using the tail of a drm_mm_node.
* If you wish to know whether a hole follows this particular node,
* query this function. See also drm_mm_hole_node_start() and
* drm_mm_hole_node_end().
*
* Returns:
* True if a hole follows the @node.
*/
292static inline bool_Bool drm_mm_hole_follows(const struct drm_mm_node *node)
293{
return node->hole_size;
295}

297static inline u64 __drm_mm_hole_node_start(const struct drm_mm_node *hole_node)
298{
return hole_node->start + hole_node->size;
300}

302/**
* drm_mm_hole_node_start - computes the start of the hole following @node
* @hole_node: drm_mm_node which implicitly tracks the following hole
*
* This is useful for driver-specific debug dumpers. Otherwise drivers should
* not inspect holes themselves. Drivers must check first whether a hole indeed
* follows by looking at drm_mm_hole_follows()
*
* Returns:
* Start of the subsequent hole.
*/
313static inline u64 drm_mm_hole_node_start(const struct drm_mm_node *hole_node)
314{
DRM_MM_BUG_ON(!drm_mm_hole_follows(hole_node))((void)0);
return __drm_mm_hole_node_start(hole_node);
317}

319static inline u64 __drm_mm_hole_node_end(const struct drm_mm_node *hole_node)
320{
return list_next_entry(hole_node, node_list)({ const __typeof( ((typeof(*(hole_node)) *)0)->node_list )
 *__mptr = (((hole_node)->node_list.next)); (typeof(*(hole_node
)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(hole_node
)), node_list) );})->start;
322}

324/**
* drm_mm_hole_node_end - computes the end of the hole following @node
* @hole_node: drm_mm_node which implicitly tracks the following hole
*
* This is useful for driver-specific debug dumpers. Otherwise drivers should
* not inspect holes themselves. Drivers must check first whether a hole indeed
* follows by looking at drm_mm_hole_follows().
*
* Returns:
* End of the subsequent hole.
*/
335static inline u64 drm_mm_hole_node_end(const struct drm_mm_node *hole_node)
336{
return __drm_mm_hole_node_end(hole_node);
338}

340/**
* drm_mm_nodes - list of nodes under the drm_mm range manager
* @mm: the struct drm_mm range manger
*
* As the drm_mm range manager hides its node_list deep with its
* structure, extracting it looks painful and repetitive. This is
* not expected to be used outside of the drm_mm_for_each_node()
* macros and similar internal functions.
*
* Returns:
* The node list, may be empty.
*/
352#define drm_mm_nodes(mm)(&(mm)->head_node.node_list) (&(mm)->head_node.node_list)

354/**
* drm_mm_for_each_node - iterator to walk over all allocated nodes
* @entry: &struct drm_mm_node to assign to in each iteration step
* @mm: &drm_mm allocator to walk
*
* This iterator walks over all nodes in the range allocator. It is implemented
* with list_for_each(), so not save against removal of elements.
*/
362#define drm_mm_for_each_node(entry, mm)for (entry = ({ const __typeof( ((__typeof(*entry) *)0)->node_list
 ) *__mptr = (((&(mm)->head_node.node_list))->next)
; (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*entry), node_list) );}); &entry->node_list != ((&
(mm)->head_node.node_list)); entry = ({ const __typeof( ((
__typeof(*entry) *)0)->node_list ) *__mptr = (entry->node_list
.next); (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*entry), node_list) );})) \
list_for_each_entry(entry, drm_mm_nodes(mm), node_list)for (entry = ({ const __typeof( ((__typeof(*entry) *)0)->node_list
 ) *__mptr = (((&(mm)->head_node.node_list))->next)
; (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*entry), node_list) );}); &entry->node_list != ((&
(mm)->head_node.node_list)); entry = ({ const __typeof( ((
__typeof(*entry) *)0)->node_list ) *__mptr = (entry->node_list
.next); (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*entry), node_list) );}))

365/**
* drm_mm_for_each_node_safe - iterator to walk over all allocated nodes
* @entry: &struct drm_mm_node to assign to in each iteration step
* @next: &struct drm_mm_node to store the next step
* @mm: &drm_mm allocator to walk
*
* This iterator walks over all nodes in the range allocator. It is implemented
* with list_for_each_safe(), so save against removal of elements.
*/
374#define drm_mm_for_each_node_safe(entry, next, mm)for (entry = ({ const __typeof( ((__typeof(*entry) *)0)->node_list
 ) *__mptr = (((&(mm)->head_node.node_list))->next)
; (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*entry), node_list) );}), next = ({ const __typeof( ((__typeof
(*entry) *)0)->node_list ) *__mptr = (entry->node_list.
next); (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*entry), node_list) );}); &entry->node_list !=
 ((&(mm)->head_node.node_list)); entry = next, next = (
{ const __typeof( ((__typeof(*next) *)0)->node_list ) *__mptr
 = (next->node_list.next); (__typeof(*next) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*next), node_list) );})) \
list_for_each_entry_safe(entry, next, drm_mm_nodes(mm), node_list)for (entry = ({ const __typeof( ((__typeof(*entry) *)0)->node_list
 ) *__mptr = (((&(mm)->head_node.node_list))->next)
; (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof(__typeof
(*entry), node_list) );}), next = ({ const __typeof( ((__typeof
(*entry) *)0)->node_list ) *__mptr = (entry->node_list.
next); (__typeof(*entry) *)( (char *)__mptr - __builtin_offsetof
(__typeof(*entry), node_list) );}); &entry->node_list !=
 ((&(mm)->head_node.node_list)); entry = next, next = (
{ const __typeof( ((__typeof(*next) *)0)->node_list ) *__mptr
 = (next->node_list.next); (__typeof(*next) *)( (char *)__mptr
 - __builtin_offsetof(__typeof(*next), node_list) );}))

377/**
* drm_mm_for_each_hole - iterator to walk over all holes
* @pos: &drm_mm_node used internally to track progress
* @mm: &drm_mm allocator to walk
* @hole_start: ulong variable to assign the hole start to on each iteration
* @hole_end: ulong variable to assign the hole end to on each iteration
*
* This iterator walks over all holes in the range allocator. It is implemented
* with list_for_each(), so not save against removal of elements. @entry is used
* internally and will not reflect a real drm_mm_node for the very first hole.
* Hence users of this iterator may not access it.
*
* Implementation Note:
* We need to inline list_for_each_entry in order to be able to set hole_start
* and hole_end on each iteration while keeping the macro sane.
*/
393#define drm_mm_for_each_hole(pos, mm, hole_start, hole_end)for (pos = ({ const __typeof( ((typeof(*pos) *)0)->hole_stack
 ) *__mptr = ((&(mm)->hole_stack)->next); (typeof(*
pos) *)( (char *)__mptr - __builtin_offsetof(typeof(*pos), hole_stack
) );}); &pos->hole_stack != &(mm)->hole_stack ?
 hole_start = drm_mm_hole_node_start(pos), hole_end = hole_start
 + pos->hole_size, 1 : 0; pos = ({ const __typeof( ((typeof
(*(pos)) *)0)->hole_stack ) *__mptr = (((pos)->hole_stack
.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof
(typeof(*(pos)), hole_stack) );})) \
for (pos = list_first_entry(&(mm)->hole_stack, \({ const __typeof( ((typeof(*pos) *)0)->hole_stack ) *__mptr
 = ((&(mm)->hole_stack)->next); (typeof(*pos) *)( (
char *)__mptr - __builtin_offsetof(typeof(*pos), hole_stack) )
;})
		    typeof(*pos), hole_stack)({ const __typeof( ((typeof(*pos) *)0)->hole_stack ) *__mptr
 = ((&(mm)->hole_stack)->next); (typeof(*pos) *)( (
char *)__mptr - __builtin_offsetof(typeof(*pos), hole_stack) )
;}); \
    &pos->hole_stack != &(mm)->hole_stack ? \
    hole_start = drm_mm_hole_node_start(pos), \
    hole_end = hole_start + pos->hole_size, \
    1 : 0; \
    pos = list_next_entry(pos, hole_stack)({ const __typeof( ((typeof(*(pos)) *)0)->hole_stack ) *__mptr
 = (((pos)->hole_stack.next)); (typeof(*(pos)) *)( (char *
)__mptr - __builtin_offsetof(typeof(*(pos)), hole_stack) );}))

402/*
* Basic range manager support (drm_mm.c)
*/
405int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node);
406int drm_mm_insert_node_in_range(struct drm_mm *mm,
		struct drm_mm_node *node,
		u64 size,
		u64 alignment,
		unsigned long color,
		u64 start,
		u64 end,
		enum drm_mm_insert_mode mode);

415/**
* drm_mm_insert_node_generic - search for space and insert @node
* @mm: drm_mm to allocate from
* @node: preallocate node to insert
* @size: size of the allocation
* @alignment: alignment of the allocation
* @color: opaque tag value to use for this node
* @mode: fine-tune the allocation search and placement
*
* This is a simplified version of drm_mm_insert_node_in_range() with no
* range restrictions applied.
*
* The preallocated node must be cleared to 0.
*
* Returns:
* 0 on success, -ENOSPC if there's no suitable hole.
*/
432static inline int
433drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
	   u64 size, u64 alignment,
	   unsigned long color,
	   enum drm_mm_insert_mode mode)
437{
return drm_mm_insert_node_in_range(mm, node,
			   size, alignment, color,
			   0, U64_MAX0xffffffffffffffffULL, mode);
441}

443/**
* drm_mm_insert_node - search for space and insert @node
* @mm: drm_mm to allocate from
* @node: preallocate node to insert
* @size: size of the allocation
*
* This is a simplified version of drm_mm_insert_node_generic() with @color set
* to 0.
*
* The preallocated node must be cleared to 0.
*
* Returns:
* 0 on success, -ENOSPC if there's no suitable hole.
*/
457static inline int drm_mm_insert_node(struct drm_mm *mm,
		     struct drm_mm_node *node,
		     u64 size)
460{
return drm_mm_insert_node_generic(mm, node, size, 0, 0, 0);
462}

464void drm_mm_remove_node(struct drm_mm_node *node);
465void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new);
466void drm_mm_init(struct drm_mm *mm, u64 start, u64 size);
467void drm_mm_takedown(struct drm_mm *mm);

469/**
* drm_mm_clean - checks whether an allocator is clean
* @mm: drm_mm allocator to check
*
* Returns:
* True if the allocator is completely free, false if there's still a node
* allocated in it.
*/
477static inline bool_Bool drm_mm_clean(const struct drm_mm *mm)
478{
return list_empty(drm_mm_nodes(mm)(&(mm)->head_node.node_list));
480}

482struct drm_mm_node *
483__drm_mm_interval_first(const struct drm_mm *mm, u64 start, u64 last);

485/**
* drm_mm_for_each_node_in_range - iterator to walk over a range of
* allocated nodes
* @node__: drm_mm_node structure to assign to in each iteration step
* @mm__: drm_mm allocator to walk
* @start__: starting offset, the first node will overlap this
* @end__: ending offset, the last node will start before this (but may overlap)
*
* This iterator walks over all nodes in the range allocator that lie
* between @start and @end. It is implemented similarly to list_for_each(),
* but using the internal interval tree to accelerate the search for the
* starting node, and so not safe against removal of elements. It assumes
* that @end is within (or is the upper limit of) the drm_mm allocator.
* If [@start, @end] are beyond the range of the drm_mm, the iterator may walk
* over the special _unallocated_ &drm_mm.head_node, and may even continue
* indefinitely.
*/
502#define drm_mm_for_each_node_in_range(node__, mm__, start__, end__)for (node__ = __drm_mm_interval_first((mm__), (start__), (end__
)-1); node__->start < (end__); node__ = ({ const __typeof
( ((typeof(*(node__)) *)0)->node_list ) *__mptr = (((node__
)->node_list.next)); (typeof(*(node__)) *)( (char *)__mptr
 - __builtin_offsetof(typeof(*(node__)), node_list) );}))	\
for (node__ = __drm_mm_interval_first((mm__), (start__), (end__)-1); \
    node__->start < (end__);					\
    node__ = list_next_entry(node__, node_list)({ const __typeof( ((typeof(*(node__)) *)0)->node_list ) *
__mptr = (((node__)->node_list.next)); (typeof(*(node__)) *
)( (char *)__mptr - __builtin_offsetof(typeof(*(node__)), node_list
) );}))

507void drm_mm_scan_init_with_range(struct drm_mm_scan *scan,
		 struct drm_mm *mm,
		 u64 size, u64 alignment, unsigned long color,
		 u64 start, u64 end,
		 enum drm_mm_insert_mode mode);

513/**
* drm_mm_scan_init - initialize lru scanning
* @scan: scan state
* @mm: drm_mm to scan
* @size: size of the allocation
* @alignment: alignment of the allocation
* @color: opaque tag value to use for the allocation
* @mode: fine-tune the allocation search and placement
*
* This is a simplified version of drm_mm_scan_init_with_range() with no range
* restrictions applied.
*
* This simply sets up the scanning routines with the parameters for the desired
* hole.
*
* Warning:
* As long as the scan list is non-empty, no other operations than
* adding/removing nodes to/from the scan list are allowed.
*/
532static inline void drm_mm_scan_init(struct drm_mm_scan *scan,
		    struct drm_mm *mm,
		    u64 size,
		    u64 alignment,
		    unsigned long color,
		    enum drm_mm_insert_mode mode)
538{
drm_mm_scan_init_with_range(scan, mm,
		    size, alignment, color,
		    0, U64_MAX0xffffffffffffffffULL, mode);
542}

544bool_Bool drm_mm_scan_add_block(struct drm_mm_scan *scan,
	   struct drm_mm_node *node);
546bool_Bool drm_mm_scan_remove_block(struct drm_mm_scan *scan,
	      struct drm_mm_node *node);
548struct drm_mm_node *drm_mm_scan_color_evict(struct drm_mm_scan *scan);

550void drm_mm_print(const struct drm_mm *mm, struct drm_printer *p);

552#endif

←

/usr/src/sys/dev/pci/drm/include/linux/atomic.h

1/* $OpenBSD: atomic.h,v 1.22 2024/01/06 12:52:20 jsg Exp $ */
2/**
3 * \file drm_atomic.h
4 * Atomic operations used in the DRM which may or may not be provided by the OS.
5 * 
6 * \author Eric Anholt <anholt@FreeBSD.org>
7 */
8 
9/*-
10 * Copyright 2004 Eric Anholt
11 * All Rights Reserved.
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
19 *
20 * The above copyright notice and this permission notice (including the next
21 * paragraph) shall be included in all copies or substantial portions of the
22 * Software.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
27 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
28 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
29 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
30 * OTHER DEALINGS IN THE SOFTWARE.
31 */
32 
33#ifndef _DRM_LINUX_ATOMIC_H_
34#define _DRM_LINUX_ATOMIC_H_
35 
36#include <sys/types.h>
37#include <sys/mutex.h>
38#include <machine/intr.h>
39#include <linux/types.h>
40#include <linux/compiler.h>	/* via x86/include/asm/atomic.h */
41 
42#define ATOMIC_INIT(x)(x)		(x)
43 
44#define atomic_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })	WRITE_ONCE(*(p), (v))({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
45#define atomic_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })		READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
46#define atomic_inc(p)__sync_fetch_and_add(p, 1)		__sync_fetch_and_add(p, 1)
47#define atomic_dec(p)__sync_fetch_and_sub(p, 1)		__sync_fetch_and_sub(p, 1)
48#define atomic_add(n, p)__sync_fetch_and_add(p, n)	__sync_fetch_and_add(p, n)
49#define atomic_sub(n, p)__sync_fetch_and_sub(p, n)	__sync_fetch_and_sub(p, n)
50#define atomic_and(n, p)__sync_fetch_and_and(p, n)	__sync_fetch_and_and(p, n)
51#define atomic_or(n, p)x86_atomic_setbits_u32(p, n)		atomic_setbits_intx86_atomic_setbits_u32(p, n)
52#define atomic_add_return(n, p)__sync_add_and_fetch(p, n) __sync_add_and_fetch(p, n)
53#define atomic_sub_return(n, p)__sync_sub_and_fetch(p, n) __sync_sub_and_fetch(p, n)
54#define atomic_sub_and_test(n, p)(__sync_sub_and_fetch(p, n) == 0)	(atomic_sub_return(n, p)__sync_sub_and_fetch(p, n) == 0)
55#define atomic_inc_return(v)__sync_add_and_fetch((v), 1)	atomic_add_return(1, (v))__sync_add_and_fetch((v), 1)
56#define atomic_dec_return(v)__sync_sub_and_fetch((v), 1)	atomic_sub_return(1, (v))__sync_sub_and_fetch((v), 1)
57#define atomic_dec_and_test(v)(__sync_sub_and_fetch((v), 1) == 0)	(atomic_dec_return(v)__sync_sub_and_fetch((v), 1) == 0)
58#define atomic_inc_and_test(v)(__sync_add_and_fetch((v), 1) == 0)	(atomic_inc_return(v)__sync_add_and_fetch((v), 1) == 0)
59#define atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)	__sync_val_compare_and_swap(p, o, n)
60#define cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)	__sync_val_compare_and_swap(p, o, n)
61#define cmpxchg64(p, o, n)__sync_val_compare_and_swap(p, o, n)	__sync_val_compare_and_swap(p, o, n)
62#define atomic_set_release(p, v)({ typeof(*((p))) __tmp = (((v))); *(volatile typeof(*((p))) *
)&(*((p))) = __tmp; __tmp; })	atomic_set((p), (v))({ typeof(*((p))) __tmp = (((v))); *(volatile typeof(*((p))) *
)&(*((p))) = __tmp; __tmp; })
63#define atomic_andnot(bits, p)x86_atomic_clearbits_u32(p,bits)		atomic_clearbits_intx86_atomic_clearbits_u32(p,bits)
64#define atomic_fetch_inc(p)__sync_fetch_and_add(p, 1)		__sync_fetch_and_add(p, 1)
65#define atomic_fetch_xor(n, p)__sync_fetch_and_xor(p, n)		__sync_fetch_and_xor(p, n)
66 
67#define try_cmpxchg(p, op, n)({ __typeof(p) __op = (__typeof((p)))(op); __typeof(*(p)) __o
 = *__op; __typeof(*(p)) __p = __sync_val_compare_and_swap((p
), (__o), (n)); if (__p != __o) *__op = __p; (__p == __o); })						\
68({									\
69	__typeof(p) __op = (__typeof((p)))(op);				\
70	__typeof(*(p)) __o = *__op;					\
71	__typeof(*(p)) __p = __sync_val_compare_and_swap((p), (__o), (n)); \
72	if (__p != __o)							\
73		*__op = __p;						\
74	(__p == __o);							\
75})
76 
77static inline bool_Bool
78atomic_try_cmpxchg(volatile int *p, int *op, int n)
79{
80	return try_cmpxchg(p, op, n)({ __typeof(p) __op = (__typeof((p)))(op); __typeof(*(p)) __o
 = *__op; __typeof(*(p)) __p = __sync_val_compare_and_swap((p
), (__o), (n)); if (__p != __o) *__op = __p; (__p == __o); });
81}
82 
83static inline int
84atomic_xchg(volatile int *v, int n)
85{
86	__sync_synchronize();
87	return __sync_lock_test_and_set(v, n);
88}
89 
90#define xchg(v, n)__sync_lock_test_and_set(v, n)	__sync_lock_test_and_set(v, n)
91 
92static inline int
93atomic_add_unless(volatile int *v, int n, int u)
94{
95	int o;
96 
97	do {
98		o = *v;
99		if (o == u)
100			return 0;
101	} while (__sync_val_compare_and_swap(v, o, o +n) != o);
102 
103	return 1;
104}
105 
106#define atomic_inc_not_zero(v)atomic_add_unless((v), 1, 0)	atomic_add_unless((v), 1, 0)
107 
108static inline int
109atomic_dec_if_positive(volatile int *v)
110{
111	int r, o;
112 
113	do {
114		o = *v;
115		r = o - 1;
116		if (r < 0)
117			break;
118	} while (__sync_val_compare_and_swap(v, o, r) != o);
119 
120	return r;
121}
122 
123#define atomic_long_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })	READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
124 
125/* 32 bit powerpc lacks 64 bit atomics */
126#if !defined(__powerpc__) || defined(__powerpc64__)
127 
128typedef int64_t atomic64_t;
129 
130#define ATOMIC64_INIT(x)(x)	(x)
131 
132#define atomic64_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })	WRITE_ONCE(*(p), (v))({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
133#define atomic64_read(p)({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })	READ_ONCE(*(p))({ typeof(*(p)) __tmp = *(volatile typeof(*(p)) *)&(*(p))
; membar_datadep_consumer(); __tmp; })
134 
135static inline int64_t
136atomic64_xchg(volatile int64_t *v, int64_t n)
137{
138	__sync_synchronize();
139	return __sync_lock_test_and_set(v, n);
140}
141 
142static inline int64_t
143atomic64_cmpxchg(volatile int64_t *v, int64_t o, int64_t n)
144{
145	return __sync_val_compare_and_swap(v, o, n);
146}
147 
148#define atomic64_add(n, p)__sync_fetch_and_add_8(p, n)	__sync_fetch_and_add_8(p, n)
149#define atomic64_sub(n, p)__sync_fetch_and_sub_8(p, n)	__sync_fetch_and_sub_8(p, n)
150#define atomic64_inc(p)__sync_fetch_and_add_8(p, 1)		__sync_fetch_and_add_8(p, 1)
151#define atomic64_add_return(n, p)__sync_add_and_fetch_8(p, n) __sync_add_and_fetch_8(p, n)
152#define atomic64_inc_return(p)__sync_add_and_fetch_8(p, 1)	__sync_add_and_fetch_8(p, 1)
153 
154#else
155 
156extern struct mutex atomic64_mtx;
157 
158typedef struct {
159	volatile int64_t val;
160} atomic64_t;
161 
162#define ATOMIC64_INIT(x)(x)	{ (x) }
163 
164static inline void
165atomic64_set(atomic64_t *v, int64_t i)({ typeof(*(atomic64_t *v)) __tmp = ((int64_t i)); *(volatile
 typeof(*(atomic64_t *v)) *)&(*(atomic64_t *v)) = __tmp; __tmp
; })
166{
167	mtx_enter(&atomic64_mtx);
168	v->val = i;
169	mtx_leave(&atomic64_mtx);
170}
171 
172static inline int64_t
173atomic64_read(atomic64_t *v)({ typeof(*(atomic64_t *v)) __tmp = *(volatile typeof(*(atomic64_t
 *v)) *)&(*(atomic64_t *v)); membar_datadep_consumer(); __tmp
; })
174{
175	int64_t val;
176 
177	mtx_enter(&atomic64_mtx);
178	val = v->val;
179	mtx_leave(&atomic64_mtx);
180 
181	return val;
182}
183 
184static inline int64_t
185atomic64_xchg(atomic64_t *v, int64_t n)
186{
187	int64_t val;
188 
189	mtx_enter(&atomic64_mtx);
190	val = v->val;
191	v->val = n;
192	mtx_leave(&atomic64_mtx);
193 
194	return val;
195}
196 
197static inline void
198atomic64_add(int i, atomic64_t *v)__sync_fetch_and_add_8(atomic64_t *v, int i)
199{
200	mtx_enter(&atomic64_mtx);
201	v->val += i;
202	mtx_leave(&atomic64_mtx);
203}
204 
205#define atomic64_inc(p)__sync_fetch_and_add_8(p, 1)		atomic64_add(p, 1)__sync_fetch_and_add_8(1, p)
206 
207static inline int64_t
208atomic64_add_return(int i, atomic64_t *v)__sync_add_and_fetch_8(atomic64_t *v, int i)
209{
210	int64_t val;
211 
212	mtx_enter(&atomic64_mtx);
213	val = v->val + i;
214	v->val = val;
215	mtx_leave(&atomic64_mtx);
216 
217	return val;
218}
219 
220#define atomic64_inc_return(p)__sync_add_and_fetch_8(p, 1)		atomic64_add_return(p, 1)__sync_add_and_fetch_8(1, p)
221 
222static inline void
223atomic64_sub(int i, atomic64_t *v)__sync_fetch_and_sub_8(atomic64_t *v, int i)
224{
225	mtx_enter(&atomic64_mtx);
226	v->val -= i;
227	mtx_leave(&atomic64_mtx);
228}
229#endif
230 
231#ifdef __LP64__1
232typedef int64_t atomic_long_t;
233#define atomic_long_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })		atomic64_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
234#define atomic_long_xchg(v, n)atomic64_xchg(v, n)		atomic64_xchg(v, n)
235#define atomic_long_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)	atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)
236#define atomic_long_add(i, v)__sync_fetch_and_add_8(v, i)		atomic64_add(i, v)__sync_fetch_and_add_8(v, i)
237#define atomic_long_sub(i, v)__sync_fetch_and_sub_8(v, i)		atomic64_sub(i, v)__sync_fetch_and_sub_8(v, i)
238#else
239typedef int32_t atomic_long_t;
240#define atomic_long_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })		atomic_set(p, v)({ typeof(*(p)) __tmp = ((v)); *(volatile typeof(*(p)) *)&
(*(p)) = __tmp; __tmp; })
241#define atomic_long_xchg(v, n)atomic64_xchg(v, n)		atomic_xchg(v, n)
242#define atomic_long_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)	atomic_cmpxchg(p, o, n)__sync_val_compare_and_swap(p, o, n)
243#define atomic_long_add(i, v)__sync_fetch_and_add_8(v, i)		atomic_add(i, v)__sync_fetch_and_add(v, i)
244#define atomic_long_sub(i, v)__sync_fetch_and_sub_8(v, i)		atomic_sub(i, v)__sync_fetch_and_sub(v, i)
245#endif
246 
247static inline atomic_t
248test_and_set_bit(u_int b, volatile void *p)
249{
250	unsigned int m = 1 << (b & 0x1f);
251	unsigned int prev = __sync_fetch_and_or((volatile u_int *)p + (b >> 5), m);
252	return (prev & m) != 0;
253}
254 
255static inline void
256clear_bit(u_int b, volatile void *p)
257{
258	atomic_clearbits_intx86_atomic_clearbits_u32(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
259}
260 
261static inline void
262clear_bit_unlock(u_int b, volatile void *p)
263{
264	membar_enter()do { __asm volatile("mfence" ::: "memory"); } while (0);
265	clear_bit(b, p);
266}
267 
268static inline void
269set_bit(u_int b, volatile void *p)
270{
271	atomic_setbits_intx86_atomic_setbits_u32(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
272}
273 
274static inline void
275__clear_bit(u_int b, volatile void *p)
276{
277	volatile u_int *ptr = (volatile u_int *)p;
278	ptr[b >> 5] &= ~(1 << (b & 0x1f));
279}
280 
281static inline void
282__set_bit(u_int b, volatile void *p)
283{
284	volatile u_int *ptr = (volatile u_int *)p;
285	ptr[b >> 5] |= (1 << (b & 0x1f));
286}
287 
288static inline int
289test_bit(u_int b, const volatile void *p)
290{
291	return !!(((volatile u_int *)p)[b >> 5] & (1 << (b & 0x1f)));
15
←
The left operand of '&' is a garbage value
292}
293 
294static inline int
295__test_and_set_bit(u_int b, volatile void *p)
296{
297	unsigned int m = 1 << (b & 0x1f);
298	volatile u_int *ptr = (volatile u_int *)p;
299	unsigned int prev = ptr[b >> 5];
300	ptr[b >> 5] |= m;
301	
302	return (prev & m) != 0;
303}
304 
305static inline int
306test_and_clear_bit(u_int b, volatile void *p)
307{
308	unsigned int m = 1 << (b & 0x1f);
309	unsigned int prev = __sync_fetch_and_and((volatile u_int *)p + (b >> 5), ~m);
310	return (prev & m) != 0;
311}
312 
313static inline int
314__test_and_clear_bit(u_int b, volatile void *p)
315{
316	volatile u_int *ptr = (volatile u_int *)p;
317	int rv = !!(ptr[b >> 5] & (1 << (b & 0x1f)));
318	ptr[b >> 5] &= ~(1 << (b & 0x1f));
319	return rv;
320}
321 
322static inline int
323find_first_zero_bit(volatile void *p, int max)
324{
325	int b;
326	volatile u_int *ptr = (volatile u_int *)p;
327 
328	for (b = 0; b < max; b += 32) {
329		if (ptr[b >> 5] != ~0) {
330			for (;;) {
331				if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
332					return b;
333				b++;
334			}
335		}
336	}
337	return max;
338}
339 
340static inline int
341find_next_zero_bit(volatile void *p, int max, int b)
342{
343	volatile u_int *ptr = (volatile u_int *)p;
344 
345	for (; b < max; b += 32) {
346		if (ptr[b >> 5] != ~0) {
347			for (;;) {
348				if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
349					return b;
350				b++;
351			}
352		}
353	}
354	return max;
355}
356 
357static inline int
358find_first_bit(volatile void *p, int max)
359{
360	int b;
361	volatile u_int *ptr = (volatile u_int *)p;
362 
363	for (b = 0; b < max; b += 32) {
364		if (ptr[b >> 5] != 0) {
365			for (;;) {
366				if (ptr[b >> 5] & (1 << (b & 0x1f)))
367					return b;
368				b++;
369			}
370		}
371	}
372	return max;
373}
374 
375static inline int
376find_next_bit(const volatile void *p, int max, int b)
377{
378	volatile u_int *ptr = (volatile u_int *)p;
379 
380	for (; b < max; b+= 32) {
381		if (ptr[b >> 5] != 0) {
382			for (;;) {
383				if (ptr[b >> 5] & (1 << (b & 0x1f)))
384					return b;
385				b++;
386			}
387		}
388	}
389	return max;
390}
391 
392#define for_each_set_bit(b, p, max)for ((b) = find_first_bit((p), (max)); (b) < (max); (b) = find_next_bit
((p), (max), (b) + 1)) \
393	for ((b) = find_first_bit((p), (max));			\
394	     (b) < (max);					\
395	     (b) = find_next_bit((p), (max), (b) + 1))
396 
397#define for_each_clear_bit(b, p, max)for ((b) = find_first_zero_bit((p), (max)); (b) < (max); (
b) = find_next_zero_bit((p), (max), (b) + 1)) \
398	for ((b) = find_first_zero_bit((p), (max));		\
399	     (b) < (max);					\
400	     (b) = find_next_zero_bit((p), (max), (b) + 1))
401 
402#if defined(__i386__)
403#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
404#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
405#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
406#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
)	__asm volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
407#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("")do { __asm volatile("" ::: "memory"); } while (0)
408#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("")do { __asm volatile("" ::: "memory"); } while (0)
409#define __smp_store_mb(var, value)do { (void)__sync_lock_test_and_set(&var, value); } while
 (0)	do { (void)xchg(&var, value)__sync_lock_test_and_set(&var, value); } while (0)
410#define smp_mb__after_atomic()do { } while (0)	do { } while (0)
411#define smp_mb__before_atomic()do { } while (0)	do { } while (0)
412#elif defined(__amd64__1)
413#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__membar("lfence")do { __asm volatile("lfence" ::: "memory"); } while (0)
414#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__membar("sfence")do { __asm volatile("sfence" ::: "memory"); } while (0)
415#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__membar("mfence")do { __asm volatile("mfence" ::: "memory"); } while (0)
416#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
)	__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc")
417#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("")do { __asm volatile("" ::: "memory"); } while (0)
418#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("")do { __asm volatile("" ::: "memory"); } while (0)
419#define __smp_store_mb(var, value)do { (void)__sync_lock_test_and_set(&var, value); } while
 (0)	do { (void)xchg(&var, value)__sync_lock_test_and_set(&var, value); } while (0)
420#define smp_mb__after_atomic()do { } while (0)	do { } while (0)
421#define smp_mb__before_atomic()do { } while (0)	do { } while (0)
422#elif defined(__aarch64__)
423#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__membar("dsb ld")do { __asm volatile("dsb ld" ::: "memory"); } while (0)
424#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__membar("dsb st")do { __asm volatile("dsb st" ::: "memory"); } while (0)
425#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
426#define dma_rmb() __membar("dmb oshld")do { __asm volatile("dmb oshld" ::: "memory"); } while (0)
427#define dma_wmb() __membar("dmb oshst")do { __asm volatile("dmb oshst" ::: "memory"); } while (0)
428#define dma_mb() __membar("dmb osh")do { __asm volatile("dmb osh" ::: "memory"); } while (0)
429#elif defined(__arm__)
430#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
431#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
432#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__membar("dsb sy")do { __asm volatile("dsb sy" ::: "memory"); } while (0)
433#elif defined(__mips64__)
434#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	mips_sync() 
435#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	mips_sync()
436#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	mips_sync()
437#elif defined(__powerpc64__)
438#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
439#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
440#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
441#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("lwsync")do { __asm volatile("lwsync" ::: "memory"); } while (0)
442#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("lwsync")do { __asm volatile("lwsync" ::: "memory"); } while (0)
443#elif defined(__powerpc__)
444#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
445#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
446#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__membar("sync")do { __asm volatile("sync" ::: "memory"); } while (0)
447#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("eieio")do { __asm volatile("eieio" ::: "memory"); } while (0)
448#elif defined(__riscv)
449#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	__membar("fence ir,ir")do { __asm volatile("fence ir,ir" ::: "memory"); } while (0)
450#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	__membar("fence ow,ow")do { __asm volatile("fence ow,ow" ::: "memory"); } while (0)
451#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	__membar("fence iorw,iorw")do { __asm volatile("fence iorw,iorw" ::: "memory"); } while (
0)
452#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("fence r,r")do { __asm volatile("fence r,r" ::: "memory"); } while (0)
453#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0)	__membar("fence w,w")do { __asm volatile("fence w,w" ::: "memory"); } while (0)
454#define smp_mb()__asm volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc"
)	__membar("fence rw,rw")do { __asm volatile("fence rw,rw" ::: "memory"); } while (0)
455#elif defined(__sparc64__)
456#define rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)	membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
457#define wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)	membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
458#define mb()do { __asm volatile("mfence" ::: "memory"); } while (0)	membar_sync()do { __asm volatile("mfence" ::: "memory"); } while (0)
459#endif
460 
461#ifndef smp_rmb
462#define smp_rmb()do { __asm volatile("" ::: "memory"); } while (0)	rmb()do { __asm volatile("lfence" ::: "memory"); } while (0)
463#endif
464 
465#ifndef smp_wmb
466#define smp_wmb()do { __asm volatile("" ::: "memory"); } while (0)	wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)
467#endif
468 
469#ifndef mmiowb
470#define mmiowb()do { __asm volatile("sfence" ::: "memory"); } while (0)	wmb()do { __asm volatile("sfence" ::: "memory"); } while (0)
471#endif
472 
473#ifndef smp_mb__before_atomic
474#define smp_mb__before_atomic()do { } while (0)	mb()do { __asm volatile("mfence" ::: "memory"); } while (0)
475#endif
476 
477#ifndef smp_mb__after_atomic
478#define smp_mb__after_atomic()do { } while (0)	mb()do { __asm volatile("mfence" ::: "memory"); } while (0)
479#endif
480 
481#ifndef smp_store_mb
482#define smp_store_mb(x, v)do { x = v; do { __asm volatile("mfence" ::: "memory"); } while
 (0); } while (0)	do { x = v; mb()do { __asm volatile("mfence" ::: "memory"); } while (0); } while (0)
483#endif
484 
485#endif