| File: | dev/pci/drm/include/linux/list.h |
| Warning: | line 252, column 13 Access to field 'prev' results in a dereference of a null pointer (loaded from variable 'next') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | ||||
| 2 | * Copyright © 2008-2015 Intel Corporation | ||||
| 3 | * | ||||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||||
| 6 | * to deal in the Software without restriction, including without limitation | ||||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||||
| 10 | * | ||||
| 11 | * The above copyright notice and this permission notice (including the next | ||||
| 12 | * paragraph) shall be included in all copies or substantial portions of the | ||||
| 13 | * Software. | ||||
| 14 | * | ||||
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||||
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||||
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||||
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||||
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||||
| 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | ||||
| 21 | * IN THE SOFTWARE. | ||||
| 22 | * | ||||
| 23 | * Authors: | ||||
| 24 | * Eric Anholt <eric@anholt.net> | ||||
| 25 | * | ||||
| 26 | */ | ||||
| 27 | |||||
| 28 | #include <drm/drm_vma_manager.h> | ||||
| 29 | #include <linux/dma-fence-array.h> | ||||
| 30 | #include <linux/kthread.h> | ||||
| 31 | #include <linux/dma-resv.h> | ||||
| 32 | #include <linux/shmem_fs.h> | ||||
| 33 | #include <linux/slab.h> | ||||
| 34 | #include <linux/stop_machine.h> | ||||
| 35 | #include <linux/swap.h> | ||||
| 36 | #include <linux/pci.h> | ||||
| 37 | #include <linux/dma-buf.h> | ||||
| 38 | #include <linux/mman.h> | ||||
| 39 | |||||
| 40 | #include <dev/pci/agpvar.h> | ||||
| 41 | |||||
| 42 | #include "display/intel_display.h" | ||||
| 43 | #include "display/intel_frontbuffer.h" | ||||
| 44 | |||||
| 45 | #include "gem/i915_gem_clflush.h" | ||||
| 46 | #include "gem/i915_gem_context.h" | ||||
| 47 | #include "gem/i915_gem_ioctls.h" | ||||
| 48 | #include "gem/i915_gem_mman.h" | ||||
| 49 | #include "gem/i915_gem_region.h" | ||||
| 50 | #include "gt/intel_engine_user.h" | ||||
| 51 | #include "gt/intel_gt.h" | ||||
| 52 | #include "gt/intel_gt_pm.h" | ||||
| 53 | #include "gt/intel_workarounds.h" | ||||
| 54 | |||||
| 55 | #include "i915_drv.h" | ||||
| 56 | #include "i915_trace.h" | ||||
| 57 | #include "i915_vgpu.h" | ||||
| 58 | |||||
| 59 | #include "intel_pm.h" | ||||
| 60 | |||||
| 61 | static int | ||||
| 62 | insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size) | ||||
| 63 | { | ||||
| 64 | int err; | ||||
| 65 | |||||
| 66 | err = mutex_lock_interruptible(&ggtt->vm.mutex); | ||||
| 67 | if (err) | ||||
| 68 | return err; | ||||
| 69 | |||||
| 70 | memset(node, 0, sizeof(*node))__builtin_memset((node), (0), (sizeof(*node))); | ||||
| 71 | err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node, | ||||
| 72 | size, 0, I915_COLOR_UNEVICTABLE(-1), | ||||
| 73 | 0, ggtt->mappable_end, | ||||
| 74 | DRM_MM_INSERT_LOW); | ||||
| 75 | |||||
| 76 | mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex); | ||||
| 77 | |||||
| 78 | return err; | ||||
| 79 | } | ||||
| 80 | |||||
| 81 | static void | ||||
| 82 | remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node) | ||||
| 83 | { | ||||
| 84 | mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex); | ||||
| 85 | drm_mm_remove_node(node); | ||||
| 86 | mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex); | ||||
| 87 | } | ||||
| 88 | |||||
| 89 | int | ||||
| 90 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, | ||||
| 91 | struct drm_file *file) | ||||
| 92 | { | ||||
| 93 | struct i915_ggtt *ggtt = &to_i915(dev)->ggtt; | ||||
| 94 | struct drm_i915_gem_get_aperture *args = data; | ||||
| 95 | struct i915_vma *vma; | ||||
| 96 | u64 pinned; | ||||
| 97 | |||||
| 98 | if (mutex_lock_interruptible(&ggtt->vm.mutex)) | ||||
| 99 | return -EINTR4; | ||||
| 100 | |||||
| 101 | pinned = ggtt->vm.reserved; | ||||
| 102 | list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)for (vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link ) *__mptr = ((&ggtt->vm.bound_list)->next); (__typeof (*vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma) , vm_link) );}); &vma->vm_link != (&ggtt->vm.bound_list ); vma = ({ const __typeof( ((__typeof(*vma) *)0)->vm_link ) *__mptr = (vma->vm_link.next); (__typeof(*vma) *)( (char *)__mptr - __builtin_offsetof(__typeof(*vma), vm_link) );})) | ||||
| 103 | if (i915_vma_is_pinned(vma)) | ||||
| 104 | pinned += vma->node.size; | ||||
| 105 | |||||
| 106 | mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex); | ||||
| 107 | |||||
| 108 | args->aper_size = ggtt->vm.total; | ||||
| 109 | args->aper_available_size = args->aper_size - pinned; | ||||
| 110 | |||||
| 111 | return 0; | ||||
| 112 | } | ||||
| 113 | |||||
| 114 | int i915_gem_object_unbind(struct drm_i915_gem_object *obj, | ||||
| 115 | unsigned long flags) | ||||
| 116 | { | ||||
| 117 | struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm; | ||||
| 118 | DRM_LIST_HEAD(still_in_list)struct list_head still_in_list = { &(still_in_list), & (still_in_list) }; | ||||
| 119 | intel_wakeref_t wakeref; | ||||
| 120 | struct i915_vma *vma; | ||||
| 121 | int ret; | ||||
| 122 | |||||
| 123 | if (list_empty(&obj->vma.list)) | ||||
| |||||
| 124 | return 0; | ||||
| 125 | |||||
| 126 | /* | ||||
| 127 | * As some machines use ACPI to handle runtime-resume callbacks, and | ||||
| 128 | * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex | ||||
| 129 | * as they are required by the shrinker. Ergo, we wake the device up | ||||
| 130 | * first just in case. | ||||
| 131 | */ | ||||
| 132 | wakeref = intel_runtime_pm_get(rpm); | ||||
| 133 | |||||
| 134 | try_again: | ||||
| 135 | ret = 0; | ||||
| 136 | spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock); | ||||
| 137 | while (!ret
( ((struct i915_vma *)0)->obj_link ) *__mptr = ((&obj-> vma.list)->next); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof (struct i915_vma, obj_link) );})) | ||||
| 138 | struct i915_vma,(list_empty(&obj->vma.list) ? ((void *)0) : ({ const __typeof ( ((struct i915_vma *)0)->obj_link ) *__mptr = ((&obj-> vma.list)->next); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof (struct i915_vma, obj_link) );})) | ||||
| 139 | obj_link)(list_empty(&obj->vma.list) ? ((void *)0) : ({ const __typeof ( ((struct i915_vma *)0)->obj_link ) *__mptr = ((&obj-> vma.list)->next); (struct i915_vma *)( (char *)__mptr - __builtin_offsetof (struct i915_vma, obj_link) );})))) { | ||||
| 140 | struct i915_address_space *vm = vma->vm; | ||||
| 141 | |||||
| 142 | list_move_tail(&vma->obj_link, &still_in_list); | ||||
| 143 | if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK(((int)(1UL << (10))) | ((int)(1UL << (11)))))) | ||||
| 144 | continue; | ||||
| 145 | |||||
| 146 | if (flags & I915_GEM_OBJECT_UNBIND_TEST(1UL << (2))) { | ||||
| 147 | ret = -EBUSY16; | ||||
| 148 | break; | ||||
| 149 | } | ||||
| 150 | |||||
| 151 | ret = -EAGAIN35; | ||||
| 152 | if (!i915_vm_tryopen(vm)) | ||||
| 153 | break; | ||||
| 154 | |||||
| 155 | /* Prevent vma being freed by i915_vma_parked as we unbind */ | ||||
| 156 | vma = __i915_vma_get(vma); | ||||
| 157 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); | ||||
| 158 | |||||
| 159 | if (vma
| ||||
| 160 | ret = -EBUSY16; | ||||
| 161 | if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE(1UL << (0)) || | ||||
| 162 | !i915_vma_is_active(vma)) | ||||
| 163 | ret = i915_vma_unbind(vma); | ||||
| 164 | |||||
| 165 | __i915_vma_put(vma); | ||||
| 166 | } | ||||
| 167 | |||||
| 168 | i915_vm_close(vm); | ||||
| 169 | spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock); | ||||
| 170 | } | ||||
| 171 | list_splice_init(&still_in_list, &obj->vma.list); | ||||
| 172 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); | ||||
| 173 | |||||
| 174 | if (ret == -EAGAIN35 && flags & I915_GEM_OBJECT_UNBIND_BARRIER(1UL << (1))) { | ||||
| 175 | rcu_barrier()__asm volatile("" : : : "memory"); /* flush the i915_vm_release() */ | ||||
| 176 | goto try_again; | ||||
| 177 | } | ||||
| 178 | |||||
| 179 | intel_runtime_pm_put(rpm, wakeref); | ||||
| 180 | |||||
| 181 | return ret; | ||||
| 182 | } | ||||
| 183 | |||||
| 184 | static int | ||||
| 185 | i915_gem_create(struct drm_file *file, | ||||
| 186 | struct intel_memory_region *mr, | ||||
| 187 | u64 *size_p, | ||||
| 188 | u32 *handle_p) | ||||
| 189 | { | ||||
| 190 | struct drm_i915_gem_object *obj; | ||||
| 191 | u32 handle; | ||||
| 192 | u64 size; | ||||
| 193 | int ret; | ||||
| 194 | |||||
| 195 | GEM_BUG_ON(!is_power_of_2(mr->min_page_size))((void)0); | ||||
| 196 | size = round_up(*size_p, mr->min_page_size)((((*size_p) + ((mr->min_page_size) - 1)) / (mr->min_page_size )) * (mr->min_page_size)); | ||||
| 197 | if (size == 0) | ||||
| 198 | return -EINVAL22; | ||||
| 199 | |||||
| 200 | /* For most of the ABI (e.g. mmap) we think in system pages */ | ||||
| 201 | GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE))((void)0); | ||||
| 202 | |||||
| 203 | /* Allocate the new object */ | ||||
| 204 | obj = i915_gem_object_create_region(mr, size, 0); | ||||
| 205 | if (IS_ERR(obj)) | ||||
| 206 | return PTR_ERR(obj); | ||||
| 207 | |||||
| 208 | ret = drm_gem_handle_create(file, &obj->base, &handle); | ||||
| 209 | /* drop reference from allocate - handle holds it now */ | ||||
| 210 | i915_gem_object_put(obj); | ||||
| 211 | if (ret) | ||||
| 212 | return ret; | ||||
| 213 | |||||
| 214 | *handle_p = handle; | ||||
| 215 | *size_p = size; | ||||
| 216 | return 0; | ||||
| 217 | } | ||||
| 218 | |||||
| 219 | int | ||||
| 220 | i915_gem_dumb_create(struct drm_file *file, | ||||
| 221 | struct drm_device *dev, | ||||
| 222 | struct drm_mode_create_dumb *args) | ||||
| 223 | { | ||||
| 224 | enum intel_memory_type mem_type; | ||||
| 225 | int cpp = DIV_ROUND_UP(args->bpp, 8)(((args->bpp) + ((8) - 1)) / (8)); | ||||
| 226 | u32 format; | ||||
| 227 | |||||
| 228 | switch (cpp) { | ||||
| 229 | case 1: | ||||
| 230 | format = DRM_FORMAT_C8((__u32)('C') | ((__u32)('8') << 8) | ((__u32)(' ') << 16) | ((__u32)(' ') << 24)); | ||||
| 231 | break; | ||||
| 232 | case 2: | ||||
| 233 | format = DRM_FORMAT_RGB565((__u32)('R') | ((__u32)('G') << 8) | ((__u32)('1') << 16) | ((__u32)('6') << 24)); | ||||
| 234 | break; | ||||
| 235 | case 4: | ||||
| 236 | format = DRM_FORMAT_XRGB8888((__u32)('X') | ((__u32)('R') << 8) | ((__u32)('2') << 16) | ((__u32)('4') << 24)); | ||||
| 237 | break; | ||||
| 238 | default: | ||||
| 239 | return -EINVAL22; | ||||
| 240 | } | ||||
| 241 | |||||
| 242 | /* have to work out size/pitch and return them */ | ||||
| 243 | args->pitch = roundup2(args->width * cpp, 64)(((args->width * cpp) + ((64) - 1)) & (~((__typeof(args ->width * cpp))(64) - 1))); | ||||
| 244 | |||||
| 245 | /* align stride to page size so that we can remap */ | ||||
| 246 | if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format, | ||||
| 247 | DRM_FORMAT_MOD_LINEAR((((__u64)0) << 56) | ((0) & 0x00ffffffffffffffULL) ))) | ||||
| 248 | args->pitch = roundup2(args->pitch, 4096)(((args->pitch) + ((4096) - 1)) & (~((__typeof(args-> pitch))(4096) - 1))); | ||||
| 249 | |||||
| 250 | if (args->pitch < args->width) | ||||
| 251 | return -EINVAL22; | ||||
| 252 | |||||
| 253 | args->size = mul_u32_u32(args->pitch, args->height); | ||||
| 254 | |||||
| 255 | mem_type = INTEL_MEMORY_SYSTEM; | ||||
| 256 | if (HAS_LMEM(to_i915(dev))((&(to_i915(dev))->__info)->memory_regions & (( 1UL << (INTEL_REGION_LMEM))))) | ||||
| 257 | mem_type = INTEL_MEMORY_LOCAL; | ||||
| 258 | |||||
| 259 | return i915_gem_create(file, | ||||
| 260 | intel_memory_region_by_type(to_i915(dev), | ||||
| 261 | mem_type), | ||||
| 262 | &args->size, &args->handle); | ||||
| 263 | } | ||||
| 264 | |||||
| 265 | /** | ||||
| 266 | * Creates a new mm object and returns a handle to it. | ||||
| 267 | * @dev: drm device pointer | ||||
| 268 | * @data: ioctl data blob | ||||
| 269 | * @file: drm file pointer | ||||
| 270 | */ | ||||
| 271 | int | ||||
| 272 | i915_gem_create_ioctl(struct drm_device *dev, void *data, | ||||
| 273 | struct drm_file *file) | ||||
| 274 | { | ||||
| 275 | struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev); | ||||
| 276 | struct drm_i915_gem_create *args = data; | ||||
| 277 | |||||
| 278 | i915_gem_flush_free_objects(i915); | ||||
| 279 | |||||
| 280 | return i915_gem_create(file, | ||||
| 281 | intel_memory_region_by_type(i915, | ||||
| 282 | INTEL_MEMORY_SYSTEM), | ||||
| 283 | &args->size, &args->handle); | ||||
| 284 | } | ||||
| 285 | |||||
| 286 | static int | ||||
| 287 | shmem_pread(struct vm_page *page, int offset, int len, char __user *user_data, | ||||
| 288 | bool_Bool needs_clflush) | ||||
| 289 | { | ||||
| 290 | char *vaddr; | ||||
| 291 | int ret; | ||||
| 292 | |||||
| 293 | vaddr = kmap(page); | ||||
| 294 | |||||
| 295 | if (needs_clflush) | ||||
| 296 | drm_clflush_virt_range(vaddr + offset, len); | ||||
| 297 | |||||
| 298 | ret = __copy_to_user(user_data, vaddr + offset, len); | ||||
| 299 | |||||
| 300 | kunmap_va(vaddr); | ||||
| 301 | |||||
| 302 | return ret ? -EFAULT14 : 0; | ||||
| 303 | } | ||||
| 304 | |||||
| 305 | static int | ||||
| 306 | i915_gem_shmem_pread(struct drm_i915_gem_object *obj, | ||||
| 307 | struct drm_i915_gem_pread *args) | ||||
| 308 | { | ||||
| 309 | unsigned int needs_clflush; | ||||
| 310 | unsigned int idx, offset; | ||||
| 311 | struct dma_fence *fence; | ||||
| 312 | char __user *user_data; | ||||
| 313 | u64 remain; | ||||
| 314 | int ret; | ||||
| 315 | |||||
| 316 | ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0)); | ||||
| 317 | if (ret) | ||||
| 318 | return ret; | ||||
| 319 | |||||
| 320 | ret = i915_gem_object_prepare_read(obj, &needs_clflush); | ||||
| 321 | if (ret) { | ||||
| 322 | i915_gem_object_unlock(obj); | ||||
| 323 | return ret; | ||||
| 324 | } | ||||
| 325 | |||||
| 326 | fence = i915_gem_object_lock_fence(obj); | ||||
| 327 | i915_gem_object_finish_access(obj); | ||||
| 328 | i915_gem_object_unlock(obj); | ||||
| 329 | |||||
| 330 | if (!fence) | ||||
| 331 | return -ENOMEM12; | ||||
| 332 | |||||
| 333 | remain = args->size; | ||||
| 334 | user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)); | ||||
| 335 | offset = offset_in_page(args->offset)((vaddr_t)(args->offset) & ((1 << 12) - 1)); | ||||
| 336 | for (idx = args->offset >> PAGE_SHIFT12; remain; idx++) { | ||||
| 337 | struct vm_page *page = i915_gem_object_get_page(obj, idx); | ||||
| 338 | unsigned int length = min_t(u64, remain, PAGE_SIZE - offset)({ u64 __min_a = (remain); u64 __min_b = ((1 << 12) - offset ); __min_a < __min_b ? __min_a : __min_b; }); | ||||
| 339 | |||||
| 340 | ret = shmem_pread(page, offset, length, user_data, | ||||
| 341 | needs_clflush); | ||||
| 342 | if (ret) | ||||
| 343 | break; | ||||
| 344 | |||||
| 345 | remain -= length; | ||||
| 346 | user_data += length; | ||||
| 347 | offset = 0; | ||||
| 348 | } | ||||
| 349 | |||||
| 350 | i915_gem_object_unlock_fence(obj, fence); | ||||
| 351 | return ret; | ||||
| 352 | } | ||||
| 353 | |||||
| 354 | #ifdef __linux__ | ||||
| 355 | static inline bool_Bool | ||||
| 356 | gtt_user_read(struct io_mapping *mapping, | ||||
| 357 | loff_t base, int offset, | ||||
| 358 | char __user *user_data, int length) | ||||
| 359 | { | ||||
| 360 | void __iomem *vaddr; | ||||
| 361 | unsigned long unwritten; | ||||
| 362 | |||||
| 363 | /* We can use the cpu mem copy function because this is X86. */ | ||||
| 364 | vaddr = io_mapping_map_atomic_wc(mapping, base); | ||||
| 365 | unwritten = __copy_to_user_inatomic(user_data, | ||||
| 366 | (void __force *)vaddr + offset, | ||||
| 367 | length); | ||||
| 368 | io_mapping_unmap_atomic(vaddr); | ||||
| 369 | if (unwritten) { | ||||
| 370 | vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE(1 << 12)); | ||||
| 371 | unwritten = copy_to_user(user_data, | ||||
| 372 | (void __force *)vaddr + offset, | ||||
| 373 | length); | ||||
| 374 | io_mapping_unmap(vaddr); | ||||
| 375 | } | ||||
| 376 | return unwritten; | ||||
| 377 | } | ||||
| 378 | #else | ||||
| 379 | static inline bool_Bool | ||||
| 380 | gtt_user_read(struct drm_i915_privateinteldrm_softc *dev_priv, | ||||
| 381 | loff_t base, int offset, | ||||
| 382 | char __user *user_data, int length) | ||||
| 383 | { | ||||
| 384 | bus_space_handle_t bsh; | ||||
| 385 | void __iomem *vaddr; | ||||
| 386 | unsigned long unwritten; | ||||
| 387 | |||||
| 388 | /* We can use the cpu mem copy function because this is X86. */ | ||||
| 389 | agp_map_atomic(dev_priv->agph, base, &bsh); | ||||
| 390 | vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh))); | ||||
| 391 | unwritten = __copy_to_user_inatomic(user_data, | ||||
| 392 | (void __force *)vaddr + offset, | ||||
| 393 | length); | ||||
| 394 | agp_unmap_atomic(dev_priv->agph, bsh); | ||||
| 395 | if (unwritten) { | ||||
| 396 | agp_map_subregion(dev_priv->agph, base, PAGE_SIZE(1 << 12), &bsh); | ||||
| 397 | vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh))); | ||||
| 398 | unwritten = copy_to_user(user_data, | ||||
| 399 | (void __force *)vaddr + offset, | ||||
| 400 | length); | ||||
| 401 | agp_unmap_subregion(dev_priv->agph, bsh, PAGE_SIZE(1 << 12)); | ||||
| 402 | } | ||||
| 403 | return unwritten; | ||||
| 404 | } | ||||
| 405 | #endif | ||||
| 406 | |||||
| 407 | static int | ||||
| 408 | i915_gem_gtt_pread(struct drm_i915_gem_object *obj, | ||||
| 409 | const struct drm_i915_gem_pread *args) | ||||
| 410 | { | ||||
| 411 | struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev); | ||||
| 412 | struct i915_ggtt *ggtt = &i915->ggtt; | ||||
| 413 | intel_wakeref_t wakeref; | ||||
| 414 | struct drm_mm_node node; | ||||
| 415 | struct dma_fence *fence; | ||||
| 416 | void __user *user_data; | ||||
| 417 | struct i915_vma *vma; | ||||
| 418 | u64 remain, offset; | ||||
| 419 | int ret; | ||||
| 420 | |||||
| 421 | wakeref = intel_runtime_pm_get(&i915->runtime_pm); | ||||
| 422 | vma = ERR_PTR(-ENODEV19); | ||||
| 423 | if (!i915_gem_object_is_tiled(obj)) | ||||
| 424 | vma = i915_gem_object_ggtt_pin(obj, NULL((void *)0), 0, 0, | ||||
| 425 | PIN_MAPPABLE(1ULL << (3)) | | ||||
| 426 | PIN_NONBLOCK(1ULL << (2)) /* NOWARN */ | | ||||
| 427 | PIN_NOEVICT(1ULL << (0))); | ||||
| 428 | if (!IS_ERR(vma)) { | ||||
| 429 | node.start = i915_ggtt_offset(vma); | ||||
| 430 | node.flags = 0; | ||||
| 431 | } else { | ||||
| 432 | ret = insert_mappable_node(ggtt, &node, PAGE_SIZE(1 << 12)); | ||||
| 433 | if (ret) | ||||
| 434 | goto out_rpm; | ||||
| 435 | GEM_BUG_ON(!drm_mm_node_allocated(&node))((void)0); | ||||
| 436 | } | ||||
| 437 | |||||
| 438 | ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0)); | ||||
| 439 | if (ret) | ||||
| 440 | goto out_unpin; | ||||
| 441 | |||||
| 442 | ret = i915_gem_object_set_to_gtt_domain(obj, false0); | ||||
| 443 | if (ret) { | ||||
| 444 | i915_gem_object_unlock(obj); | ||||
| 445 | goto out_unpin; | ||||
| 446 | } | ||||
| 447 | |||||
| 448 | fence = i915_gem_object_lock_fence(obj); | ||||
| 449 | i915_gem_object_unlock(obj); | ||||
| 450 | if (!fence) { | ||||
| 451 | ret = -ENOMEM12; | ||||
| 452 | goto out_unpin; | ||||
| 453 | } | ||||
| 454 | |||||
| 455 | user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)); | ||||
| 456 | remain = args->size; | ||||
| 457 | offset = args->offset; | ||||
| 458 | |||||
| 459 | while (remain > 0) { | ||||
| 460 | /* Operation in this page | ||||
| 461 | * | ||||
| 462 | * page_base = page offset within aperture | ||||
| 463 | * page_offset = offset within page | ||||
| 464 | * page_length = bytes to copy for this page | ||||
| 465 | */ | ||||
| 466 | u32 page_base = node.start; | ||||
| 467 | unsigned page_offset = offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1)); | ||||
| 468 | unsigned page_length = PAGE_SIZE(1 << 12) - page_offset; | ||||
| 469 | page_length = remain < page_length ? remain : page_length; | ||||
| 470 | if (drm_mm_node_allocated(&node)) { | ||||
| 471 | ggtt->vm.insert_page(&ggtt->vm, | ||||
| 472 | i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT12), | ||||
| 473 | node.start, I915_CACHE_NONE, 0); | ||||
| 474 | } else { | ||||
| 475 | page_base += offset & LINUX_PAGE_MASK(~((1 << 12) - 1)); | ||||
| 476 | } | ||||
| 477 | |||||
| 478 | if (gtt_user_read(i915, page_base, page_offset, | ||||
| 479 | user_data, page_length)) { | ||||
| 480 | ret = -EFAULT14; | ||||
| 481 | break; | ||||
| 482 | } | ||||
| 483 | |||||
| 484 | remain -= page_length; | ||||
| 485 | user_data += page_length; | ||||
| 486 | offset += page_length; | ||||
| 487 | } | ||||
| 488 | |||||
| 489 | i915_gem_object_unlock_fence(obj, fence); | ||||
| 490 | out_unpin: | ||||
| 491 | if (drm_mm_node_allocated(&node)) { | ||||
| 492 | ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); | ||||
| 493 | remove_mappable_node(ggtt, &node); | ||||
| 494 | } else { | ||||
| 495 | i915_vma_unpin(vma); | ||||
| 496 | } | ||||
| 497 | out_rpm: | ||||
| 498 | intel_runtime_pm_put(&i915->runtime_pm, wakeref); | ||||
| 499 | return ret; | ||||
| 500 | } | ||||
| 501 | |||||
| 502 | /** | ||||
| 503 | * Reads data from the object referenced by handle. | ||||
| 504 | * @dev: drm device pointer | ||||
| 505 | * @data: ioctl data blob | ||||
| 506 | * @file: drm file pointer | ||||
| 507 | * | ||||
| 508 | * On error, the contents of *data are undefined. | ||||
| 509 | */ | ||||
| 510 | int | ||||
| 511 | i915_gem_pread_ioctl(struct drm_device *dev, void *data, | ||||
| 512 | struct drm_file *file) | ||||
| 513 | { | ||||
| 514 | struct drm_i915_gem_pread *args = data; | ||||
| 515 | struct drm_i915_gem_object *obj; | ||||
| 516 | int ret; | ||||
| 517 | |||||
| 518 | if (args->size == 0) | ||||
| 519 | return 0; | ||||
| 520 | |||||
| 521 | if (!access_ok(u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)), | ||||
| 522 | args->size)) | ||||
| 523 | return -EFAULT14; | ||||
| 524 | |||||
| 525 | obj = i915_gem_object_lookup(file, args->handle); | ||||
| 526 | if (!obj) | ||||
| 527 | return -ENOENT2; | ||||
| 528 | |||||
| 529 | /* Bounds check source. */ | ||||
| 530 | if (range_overflows_t(u64, args->offset, args->size, obj->base.size)({ typeof((u64)(args->offset)) start__ = ((u64)(args->offset )); typeof((u64)(args->size)) size__ = ((u64)(args->size )); typeof((u64)(obj->base.size)) max__ = ((u64)(obj->base .size)); (void)(&start__ == &size__); (void)(&start__ == &max__); start__ >= max__ || size__ > max__ - start__ ; })) { | ||||
| 531 | ret = -EINVAL22; | ||||
| 532 | goto out; | ||||
| 533 | } | ||||
| 534 | |||||
| 535 | trace_i915_gem_object_pread(obj, args->offset, args->size); | ||||
| 536 | |||||
| 537 | ret = -ENODEV19; | ||||
| 538 | if (obj->ops->pread) | ||||
| 539 | ret = obj->ops->pread(obj, args); | ||||
| 540 | if (ret != -ENODEV19) | ||||
| 541 | goto out; | ||||
| 542 | |||||
| 543 | ret = i915_gem_object_wait(obj, | ||||
| 544 | I915_WAIT_INTERRUPTIBLE(1UL << (0)), | ||||
| 545 | MAX_SCHEDULE_TIMEOUT(0x7fffffff)); | ||||
| 546 | if (ret) | ||||
| 547 | goto out; | ||||
| 548 | |||||
| 549 | ret = i915_gem_object_pin_pages(obj); | ||||
| 550 | if (ret) | ||||
| 551 | goto out; | ||||
| 552 | |||||
| 553 | ret = i915_gem_shmem_pread(obj, args); | ||||
| 554 | if (ret == -EFAULT14 || ret == -ENODEV19) | ||||
| 555 | ret = i915_gem_gtt_pread(obj, args); | ||||
| 556 | |||||
| 557 | i915_gem_object_unpin_pages(obj); | ||||
| 558 | out: | ||||
| 559 | i915_gem_object_put(obj); | ||||
| 560 | return ret; | ||||
| 561 | } | ||||
| 562 | |||||
| 563 | /* This is the fast write path which cannot handle | ||||
| 564 | * page faults in the source data | ||||
| 565 | */ | ||||
| 566 | #ifdef __linux__ | ||||
| 567 | static inline bool_Bool | ||||
| 568 | ggtt_write(struct io_mapping *mapping, | ||||
| 569 | loff_t base, int offset, | ||||
| 570 | char __user *user_data, int length) | ||||
| 571 | { | ||||
| 572 | void __iomem *vaddr; | ||||
| 573 | unsigned long unwritten; | ||||
| 574 | |||||
| 575 | /* We can use the cpu mem copy function because this is X86. */ | ||||
| 576 | vaddr = io_mapping_map_atomic_wc(mapping, base); | ||||
| 577 | unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, | ||||
| 578 | user_data, length); | ||||
| 579 | io_mapping_unmap_atomic(vaddr); | ||||
| 580 | if (unwritten) { | ||||
| 581 | vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE(1 << 12)); | ||||
| 582 | unwritten = copy_from_user((void __force *)vaddr + offset, | ||||
| 583 | user_data, length); | ||||
| 584 | io_mapping_unmap(vaddr); | ||||
| 585 | } | ||||
| 586 | |||||
| 587 | return unwritten; | ||||
| 588 | } | ||||
| 589 | #else | ||||
| 590 | static inline bool_Bool | ||||
| 591 | ggtt_write(struct drm_i915_privateinteldrm_softc *dev_priv, | ||||
| 592 | loff_t base, int offset, | ||||
| 593 | char __user *user_data, int length) | ||||
| 594 | { | ||||
| 595 | bus_space_handle_t bsh; | ||||
| 596 | void __iomem *vaddr; | ||||
| 597 | unsigned long unwritten; | ||||
| 598 | |||||
| 599 | /* We can use the cpu mem copy function because this is X86. */ | ||||
| 600 | agp_map_atomic(dev_priv->agph, base, &bsh); | ||||
| 601 | vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh))); | ||||
| 602 | unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, | ||||
| 603 | user_data, length); | ||||
| 604 | agp_unmap_atomic(dev_priv->agph, bsh); | ||||
| 605 | if (unwritten) { | ||||
| 606 | agp_map_subregion(dev_priv->agph, base, PAGE_SIZE(1 << 12), &bsh); | ||||
| 607 | vaddr = bus_space_vaddr(dev_priv->bst, bsh)((dev_priv->bst)->vaddr((bsh))); | ||||
| 608 | unwritten = copy_from_user((void __force *)vaddr + offset, | ||||
| 609 | user_data, length); | ||||
| 610 | agp_unmap_subregion(dev_priv->agph, bsh, PAGE_SIZE(1 << 12)); | ||||
| 611 | } | ||||
| 612 | |||||
| 613 | return unwritten; | ||||
| 614 | } | ||||
| 615 | #endif | ||||
| 616 | |||||
| 617 | /** | ||||
| 618 | * This is the fast pwrite path, where we copy the data directly from the | ||||
| 619 | * user into the GTT, uncached. | ||||
| 620 | * @obj: i915 GEM object | ||||
| 621 | * @args: pwrite arguments structure | ||||
| 622 | */ | ||||
| 623 | static int | ||||
| 624 | i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, | ||||
| 625 | const struct drm_i915_gem_pwrite *args) | ||||
| 626 | { | ||||
| 627 | struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev); | ||||
| 628 | struct i915_ggtt *ggtt = &i915->ggtt; | ||||
| 629 | struct intel_runtime_pm *rpm = &i915->runtime_pm; | ||||
| 630 | intel_wakeref_t wakeref; | ||||
| 631 | struct drm_mm_node node; | ||||
| 632 | struct dma_fence *fence; | ||||
| 633 | struct i915_vma *vma; | ||||
| 634 | u64 remain, offset; | ||||
| 635 | void __user *user_data; | ||||
| 636 | int ret; | ||||
| 637 | |||||
| 638 | if (i915_gem_object_has_struct_page(obj)) { | ||||
| 639 | /* | ||||
| 640 | * Avoid waking the device up if we can fallback, as | ||||
| 641 | * waking/resuming is very slow (worst-case 10-100 ms | ||||
| 642 | * depending on PCI sleeps and our own resume time). | ||||
| 643 | * This easily dwarfs any performance advantage from | ||||
| 644 | * using the cache bypass of indirect GGTT access. | ||||
| 645 | */ | ||||
| 646 | wakeref = intel_runtime_pm_get_if_in_use(rpm); | ||||
| 647 | if (!wakeref) | ||||
| 648 | return -EFAULT14; | ||||
| 649 | } else { | ||||
| 650 | /* No backing pages, no fallback, we must force GGTT access */ | ||||
| 651 | wakeref = intel_runtime_pm_get(rpm); | ||||
| 652 | } | ||||
| 653 | |||||
| 654 | vma = ERR_PTR(-ENODEV19); | ||||
| 655 | if (!i915_gem_object_is_tiled(obj)) | ||||
| 656 | vma = i915_gem_object_ggtt_pin(obj, NULL((void *)0), 0, 0, | ||||
| 657 | PIN_MAPPABLE(1ULL << (3)) | | ||||
| 658 | PIN_NONBLOCK(1ULL << (2)) /* NOWARN */ | | ||||
| 659 | PIN_NOEVICT(1ULL << (0))); | ||||
| 660 | if (!IS_ERR(vma)) { | ||||
| 661 | node.start = i915_ggtt_offset(vma); | ||||
| 662 | node.flags = 0; | ||||
| 663 | } else { | ||||
| 664 | ret = insert_mappable_node(ggtt, &node, PAGE_SIZE(1 << 12)); | ||||
| 665 | if (ret) | ||||
| 666 | goto out_rpm; | ||||
| 667 | GEM_BUG_ON(!drm_mm_node_allocated(&node))((void)0); | ||||
| 668 | } | ||||
| 669 | |||||
| 670 | ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0)); | ||||
| 671 | if (ret) | ||||
| 672 | goto out_unpin; | ||||
| 673 | |||||
| 674 | ret = i915_gem_object_set_to_gtt_domain(obj, true1); | ||||
| 675 | if (ret) { | ||||
| 676 | i915_gem_object_unlock(obj); | ||||
| 677 | goto out_unpin; | ||||
| 678 | } | ||||
| 679 | |||||
| 680 | fence = i915_gem_object_lock_fence(obj); | ||||
| 681 | i915_gem_object_unlock(obj); | ||||
| 682 | if (!fence) { | ||||
| 683 | ret = -ENOMEM12; | ||||
| 684 | goto out_unpin; | ||||
| 685 | } | ||||
| 686 | |||||
| 687 | i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); | ||||
| 688 | |||||
| 689 | user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)); | ||||
| 690 | offset = args->offset; | ||||
| 691 | remain = args->size; | ||||
| 692 | while (remain) { | ||||
| 693 | /* Operation in this page | ||||
| 694 | * | ||||
| 695 | * page_base = page offset within aperture | ||||
| 696 | * page_offset = offset within page | ||||
| 697 | * page_length = bytes to copy for this page | ||||
| 698 | */ | ||||
| 699 | u32 page_base = node.start; | ||||
| 700 | unsigned int page_offset = offset_in_page(offset)((vaddr_t)(offset) & ((1 << 12) - 1)); | ||||
| 701 | unsigned int page_length = PAGE_SIZE(1 << 12) - page_offset; | ||||
| 702 | page_length = remain < page_length ? remain : page_length; | ||||
| 703 | if (drm_mm_node_allocated(&node)) { | ||||
| 704 | /* flush the write before we modify the GGTT */ | ||||
| 705 | intel_gt_flush_ggtt_writes(ggtt->vm.gt); | ||||
| 706 | ggtt->vm.insert_page(&ggtt->vm, | ||||
| 707 | i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT12), | ||||
| 708 | node.start, I915_CACHE_NONE, 0); | ||||
| 709 | wmb()do { __asm volatile("sfence" ::: "memory"); } while (0); /* flush modifications to the GGTT (insert_page) */ | ||||
| 710 | } else { | ||||
| 711 | page_base += offset & LINUX_PAGE_MASK(~((1 << 12) - 1)); | ||||
| 712 | } | ||||
| 713 | /* If we get a fault while copying data, then (presumably) our | ||||
| 714 | * source page isn't available. Return the error and we'll | ||||
| 715 | * retry in the slow path. | ||||
| 716 | * If the object is non-shmem backed, we retry again with the | ||||
| 717 | * path that handles page fault. | ||||
| 718 | */ | ||||
| 719 | if (ggtt_write(i915, page_base, page_offset, | ||||
| 720 | user_data, page_length)) { | ||||
| 721 | ret = -EFAULT14; | ||||
| 722 | break; | ||||
| 723 | } | ||||
| 724 | |||||
| 725 | remain -= page_length; | ||||
| 726 | user_data += page_length; | ||||
| 727 | offset += page_length; | ||||
| 728 | } | ||||
| 729 | |||||
| 730 | intel_gt_flush_ggtt_writes(ggtt->vm.gt); | ||||
| 731 | i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); | ||||
| 732 | |||||
| 733 | i915_gem_object_unlock_fence(obj, fence); | ||||
| 734 | out_unpin: | ||||
| 735 | if (drm_mm_node_allocated(&node)) { | ||||
| 736 | ggtt->vm.clear_range(&ggtt->vm, node.start, node.size); | ||||
| 737 | remove_mappable_node(ggtt, &node); | ||||
| 738 | } else { | ||||
| 739 | i915_vma_unpin(vma); | ||||
| 740 | } | ||||
| 741 | out_rpm: | ||||
| 742 | intel_runtime_pm_put(rpm, wakeref); | ||||
| 743 | return ret; | ||||
| 744 | } | ||||
| 745 | |||||
| 746 | /* Per-page copy function for the shmem pwrite fastpath. | ||||
| 747 | * Flushes invalid cachelines before writing to the target if | ||||
| 748 | * needs_clflush_before is set and flushes out any written cachelines after | ||||
| 749 | * writing if needs_clflush is set. | ||||
| 750 | */ | ||||
| 751 | static int | ||||
| 752 | shmem_pwrite(struct vm_page *page, int offset, int len, char __user *user_data, | ||||
| 753 | bool_Bool needs_clflush_before, | ||||
| 754 | bool_Bool needs_clflush_after) | ||||
| 755 | { | ||||
| 756 | char *vaddr; | ||||
| 757 | int ret; | ||||
| 758 | |||||
| 759 | vaddr = kmap(page); | ||||
| 760 | |||||
| 761 | if (needs_clflush_before) | ||||
| 762 | drm_clflush_virt_range(vaddr + offset, len); | ||||
| 763 | |||||
| 764 | ret = __copy_from_user(vaddr + offset, user_data, len); | ||||
| 765 | if (!ret && needs_clflush_after) | ||||
| 766 | drm_clflush_virt_range(vaddr + offset, len); | ||||
| 767 | |||||
| 768 | kunmap_va(vaddr); | ||||
| 769 | |||||
| 770 | return ret ? -EFAULT14 : 0; | ||||
| 771 | } | ||||
| 772 | |||||
| 773 | static int | ||||
| 774 | i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, | ||||
| 775 | const struct drm_i915_gem_pwrite *args) | ||||
| 776 | { | ||||
| 777 | unsigned int partial_cacheline_write; | ||||
| 778 | unsigned int needs_clflush; | ||||
| 779 | unsigned int offset, idx; | ||||
| 780 | struct dma_fence *fence; | ||||
| 781 | void __user *user_data; | ||||
| 782 | u64 remain; | ||||
| 783 | int ret; | ||||
| 784 | |||||
| 785 | ret = i915_gem_object_lock_interruptible(obj, NULL((void *)0)); | ||||
| 786 | if (ret) | ||||
| 787 | return ret; | ||||
| 788 | |||||
| 789 | ret = i915_gem_object_prepare_write(obj, &needs_clflush); | ||||
| 790 | if (ret) { | ||||
| 791 | i915_gem_object_unlock(obj); | ||||
| 792 | return ret; | ||||
| 793 | } | ||||
| 794 | |||||
| 795 | fence = i915_gem_object_lock_fence(obj); | ||||
| 796 | i915_gem_object_finish_access(obj); | ||||
| 797 | i915_gem_object_unlock(obj); | ||||
| 798 | |||||
| 799 | if (!fence) | ||||
| 800 | return -ENOMEM12; | ||||
| 801 | |||||
| 802 | /* If we don't overwrite a cacheline completely we need to be | ||||
| 803 | * careful to have up-to-date data by first clflushing. Don't | ||||
| 804 | * overcomplicate things and flush the entire patch. | ||||
| 805 | */ | ||||
| 806 | partial_cacheline_write = 0; | ||||
| 807 | if (needs_clflush & CLFLUSH_BEFORE(1UL << (0))) | ||||
| 808 | partial_cacheline_write = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_cflushsz - 1; | ||||
| 809 | |||||
| 810 | user_data = u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)); | ||||
| 811 | remain = args->size; | ||||
| 812 | offset = offset_in_page(args->offset)((vaddr_t)(args->offset) & ((1 << 12) - 1)); | ||||
| 813 | for (idx = args->offset >> PAGE_SHIFT12; remain; idx++) { | ||||
| 814 | struct vm_page *page = i915_gem_object_get_page(obj, idx); | ||||
| 815 | unsigned int length = min_t(u64, remain, PAGE_SIZE - offset)({ u64 __min_a = (remain); u64 __min_b = ((1 << 12) - offset ); __min_a < __min_b ? __min_a : __min_b; }); | ||||
| 816 | |||||
| 817 | ret = shmem_pwrite(page, offset, length, user_data, | ||||
| 818 | (offset | length) & partial_cacheline_write, | ||||
| 819 | needs_clflush & CLFLUSH_AFTER(1UL << (1))); | ||||
| 820 | if (ret) | ||||
| 821 | break; | ||||
| 822 | |||||
| 823 | remain -= length; | ||||
| 824 | user_data += length; | ||||
| 825 | offset = 0; | ||||
| 826 | } | ||||
| 827 | |||||
| 828 | i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); | ||||
| 829 | i915_gem_object_unlock_fence(obj, fence); | ||||
| 830 | |||||
| 831 | return ret; | ||||
| 832 | } | ||||
| 833 | |||||
| 834 | /** | ||||
| 835 | * Writes data to the object referenced by handle. | ||||
| 836 | * @dev: drm device | ||||
| 837 | * @data: ioctl data blob | ||||
| 838 | * @file: drm file | ||||
| 839 | * | ||||
| 840 | * On error, the contents of the buffer that were to be modified are undefined. | ||||
| 841 | */ | ||||
| 842 | int | ||||
| 843 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, | ||||
| 844 | struct drm_file *file) | ||||
| 845 | { | ||||
| 846 | struct drm_i915_gem_pwrite *args = data; | ||||
| 847 | struct drm_i915_gem_object *obj; | ||||
| 848 | int ret; | ||||
| 849 | |||||
| 850 | if (args->size == 0) | ||||
| 851 | return 0; | ||||
| 852 | |||||
| 853 | if (!access_ok(u64_to_user_ptr(args->data_ptr)((void *)(uintptr_t)(args->data_ptr)), args->size)) | ||||
| 854 | return -EFAULT14; | ||||
| 855 | |||||
| 856 | obj = i915_gem_object_lookup(file, args->handle); | ||||
| 857 | if (!obj) | ||||
| 858 | return -ENOENT2; | ||||
| 859 | |||||
| 860 | /* Bounds check destination. */ | ||||
| 861 | if (range_overflows_t(u64, args->offset, args->size, obj->base.size)({ typeof((u64)(args->offset)) start__ = ((u64)(args->offset )); typeof((u64)(args->size)) size__ = ((u64)(args->size )); typeof((u64)(obj->base.size)) max__ = ((u64)(obj->base .size)); (void)(&start__ == &size__); (void)(&start__ == &max__); start__ >= max__ || size__ > max__ - start__ ; })) { | ||||
| 862 | ret = -EINVAL22; | ||||
| 863 | goto err; | ||||
| 864 | } | ||||
| 865 | |||||
| 866 | /* Writes not allowed into this read-only object */ | ||||
| 867 | if (i915_gem_object_is_readonly(obj)) { | ||||
| 868 | ret = -EINVAL22; | ||||
| 869 | goto err; | ||||
| 870 | } | ||||
| 871 | |||||
| 872 | trace_i915_gem_object_pwrite(obj, args->offset, args->size); | ||||
| 873 | |||||
| 874 | ret = -ENODEV19; | ||||
| 875 | if (obj->ops->pwrite) | ||||
| 876 | ret = obj->ops->pwrite(obj, args); | ||||
| 877 | if (ret != -ENODEV19) | ||||
| 878 | goto err; | ||||
| 879 | |||||
| 880 | ret = i915_gem_object_wait(obj, | ||||
| 881 | I915_WAIT_INTERRUPTIBLE(1UL << (0)) | | ||||
| 882 | I915_WAIT_ALL(1UL << (2)), | ||||
| 883 | MAX_SCHEDULE_TIMEOUT(0x7fffffff)); | ||||
| 884 | if (ret) | ||||
| 885 | goto err; | ||||
| 886 | |||||
| 887 | ret = i915_gem_object_pin_pages(obj); | ||||
| 888 | if (ret) | ||||
| 889 | goto err; | ||||
| 890 | |||||
| 891 | ret = -EFAULT14; | ||||
| 892 | /* We can only do the GTT pwrite on untiled buffers, as otherwise | ||||
| 893 | * it would end up going through the fenced access, and we'll get | ||||
| 894 | * different detiling behavior between reading and writing. | ||||
| 895 | * pread/pwrite currently are reading and writing from the CPU | ||||
| 896 | * perspective, requiring manual detiling by the client. | ||||
| 897 | */ | ||||
| 898 | if (!i915_gem_object_has_struct_page(obj) || | ||||
| 899 | cpu_write_needs_clflush(obj)) | ||||
| 900 | /* Note that the gtt paths might fail with non-page-backed user | ||||
| 901 | * pointers (e.g. gtt mappings when moving data between | ||||
| 902 | * textures). Fallback to the shmem path in that case. | ||||
| 903 | */ | ||||
| 904 | ret = i915_gem_gtt_pwrite_fast(obj, args); | ||||
| 905 | |||||
| 906 | if (ret == -EFAULT14 || ret == -ENOSPC28) { | ||||
| 907 | if (i915_gem_object_has_struct_page(obj)) | ||||
| 908 | ret = i915_gem_shmem_pwrite(obj, args); | ||||
| 909 | } | ||||
| 910 | |||||
| 911 | i915_gem_object_unpin_pages(obj); | ||||
| 912 | err: | ||||
| 913 | i915_gem_object_put(obj); | ||||
| 914 | return ret; | ||||
| 915 | } | ||||
| 916 | |||||
| 917 | /** | ||||
| 918 | * Called when user space has done writes to this buffer | ||||
| 919 | * @dev: drm device | ||||
| 920 | * @data: ioctl data blob | ||||
| 921 | * @file: drm file | ||||
| 922 | */ | ||||
| 923 | int | ||||
| 924 | i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, | ||||
| 925 | struct drm_file *file) | ||||
| 926 | { | ||||
| 927 | struct drm_i915_gem_sw_finish *args = data; | ||||
| 928 | struct drm_i915_gem_object *obj; | ||||
| 929 | |||||
| 930 | obj = i915_gem_object_lookup(file, args->handle); | ||||
| 931 | if (!obj) | ||||
| 932 | return -ENOENT2; | ||||
| 933 | |||||
| 934 | /* | ||||
| 935 | * Proxy objects are barred from CPU access, so there is no | ||||
| 936 | * need to ban sw_finish as it is a nop. | ||||
| 937 | */ | ||||
| 938 | |||||
| 939 | /* Pinned buffers may be scanout, so flush the cache */ | ||||
| 940 | i915_gem_object_flush_if_display(obj); | ||||
| 941 | i915_gem_object_put(obj); | ||||
| 942 | |||||
| 943 | return 0; | ||||
| 944 | } | ||||
| 945 | |||||
| 946 | void i915_gem_runtime_suspend(struct drm_i915_privateinteldrm_softc *i915) | ||||
| 947 | { | ||||
| 948 | struct drm_i915_gem_object *obj, *on; | ||||
| 949 | int i; | ||||
| 950 | |||||
| 951 | /* | ||||
| 952 | * Only called during RPM suspend. All users of the userfault_list | ||||
| 953 | * must be holding an RPM wakeref to ensure that this can not | ||||
| 954 | * run concurrently with themselves (and use the struct_mutex for | ||||
| 955 | * protection between themselves). | ||||
| 956 | */ | ||||
| 957 | |||||
| 958 | list_for_each_entry_safe(obj, on,for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->userfault_link ) *__mptr = ((&i915->ggtt.userfault_list)->next); ( __typeof(*obj) *)( (char *)__mptr - __builtin_offsetof(__typeof (*obj), userfault_link) );}), on = ({ const __typeof( ((__typeof (*obj) *)0)->userfault_link ) *__mptr = (obj->userfault_link .next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof (__typeof(*obj), userfault_link) );}); &obj->userfault_link != (&i915->ggtt.userfault_list); obj = on, on = ({ const __typeof( ((__typeof(*on) *)0)->userfault_link ) *__mptr = (on->userfault_link.next); (__typeof(*on) *)( (char *)__mptr - __builtin_offsetof(__typeof(*on), userfault_link) );})) | ||||
| 959 | &i915->ggtt.userfault_list, userfault_link)for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->userfault_link ) *__mptr = ((&i915->ggtt.userfault_list)->next); ( __typeof(*obj) *)( (char *)__mptr - __builtin_offsetof(__typeof (*obj), userfault_link) );}), on = ({ const __typeof( ((__typeof (*obj) *)0)->userfault_link ) *__mptr = (obj->userfault_link .next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof (__typeof(*obj), userfault_link) );}); &obj->userfault_link != (&i915->ggtt.userfault_list); obj = on, on = ({ const __typeof( ((__typeof(*on) *)0)->userfault_link ) *__mptr = (on->userfault_link.next); (__typeof(*on) *)( (char *)__mptr - __builtin_offsetof(__typeof(*on), userfault_link) );})) | ||||
| 960 | __i915_gem_object_release_mmap_gtt(obj); | ||||
| 961 | |||||
| 962 | /* | ||||
| 963 | * The fence will be lost when the device powers down. If any were | ||||
| 964 | * in use by hardware (i.e. they are pinned), we should not be powering | ||||
| 965 | * down! All other fences will be reacquired by the user upon waking. | ||||
| 966 | */ | ||||
| 967 | for (i = 0; i < i915->ggtt.num_fences; i++) { | ||||
| 968 | struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i]; | ||||
| 969 | |||||
| 970 | /* | ||||
| 971 | * Ideally we want to assert that the fence register is not | ||||
| 972 | * live at this point (i.e. that no piece of code will be | ||||
| 973 | * trying to write through fence + GTT, as that both violates | ||||
| 974 | * our tracking of activity and associated locking/barriers, | ||||
| 975 | * but also is illegal given that the hw is powered down). | ||||
| 976 | * | ||||
| 977 | * Previously we used reg->pin_count as a "liveness" indicator. | ||||
| 978 | * That is not sufficient, and we need a more fine-grained | ||||
| 979 | * tool if we want to have a sanity check here. | ||||
| 980 | */ | ||||
| 981 | |||||
| 982 | if (!reg->vma) | ||||
| 983 | continue; | ||||
| 984 | |||||
| 985 | GEM_BUG_ON(i915_vma_has_userfault(reg->vma))((void)0); | ||||
| 986 | reg->dirty = true1; | ||||
| 987 | } | ||||
| 988 | } | ||||
| 989 | |||||
| 990 | static void discard_ggtt_vma(struct i915_vma *vma) | ||||
| 991 | { | ||||
| 992 | struct drm_i915_gem_object *obj = vma->obj; | ||||
| 993 | |||||
| 994 | spin_lock(&obj->vma.lock)mtx_enter(&obj->vma.lock); | ||||
| 995 | if (!RB_EMPTY_NODE(&vma->obj_node)((&vma->obj_node)->__entry.rbe_parent == &vma-> obj_node)) { | ||||
| 996 | rb_erase(&vma->obj_node, &obj->vma.tree)linux_root_RB_REMOVE((struct linux_root *)(&obj->vma.tree ), (&vma->obj_node)); | ||||
| 997 | RB_CLEAR_NODE(&vma->obj_node)(((&vma->obj_node))->__entry.rbe_parent = (&vma ->obj_node)); | ||||
| 998 | } | ||||
| 999 | spin_unlock(&obj->vma.lock)mtx_leave(&obj->vma.lock); | ||||
| 1000 | } | ||||
| 1001 | |||||
| 1002 | struct i915_vma * | ||||
| 1003 | i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, | ||||
| 1004 | struct i915_gem_ww_ctx *ww, | ||||
| 1005 | const struct i915_ggtt_view *view, | ||||
| 1006 | u64 size, u64 alignment, u64 flags) | ||||
| 1007 | { | ||||
| 1008 | struct drm_i915_privateinteldrm_softc *i915 = to_i915(obj->base.dev); | ||||
| 1009 | struct i915_ggtt *ggtt = &i915->ggtt; | ||||
| 1010 | struct i915_vma *vma; | ||||
| 1011 | int ret; | ||||
| 1012 | |||||
| 1013 | if (flags & PIN_MAPPABLE(1ULL << (3)) && | ||||
| 1014 | (!view || view->type == I915_GGTT_VIEW_NORMAL)) { | ||||
| 1015 | /* | ||||
| 1016 | * If the required space is larger than the available | ||||
| 1017 | * aperture, we will not able to find a slot for the | ||||
| 1018 | * object and unbinding the object now will be in | ||||
| 1019 | * vain. Worse, doing so may cause us to ping-pong | ||||
| 1020 | * the object in and out of the Global GTT and | ||||
| 1021 | * waste a lot of cycles under the mutex. | ||||
| 1022 | */ | ||||
| 1023 | if (obj->base.size > ggtt->mappable_end) | ||||
| 1024 | return ERR_PTR(-E2BIG7); | ||||
| 1025 | |||||
| 1026 | /* | ||||
| 1027 | * If NONBLOCK is set the caller is optimistically | ||||
| 1028 | * trying to cache the full object within the mappable | ||||
| 1029 | * aperture, and *must* have a fallback in place for | ||||
| 1030 | * situations where we cannot bind the object. We | ||||
| 1031 | * can be a little more lax here and use the fallback | ||||
| 1032 | * more often to avoid costly migrations of ourselves | ||||
| 1033 | * and other objects within the aperture. | ||||
| 1034 | * | ||||
| 1035 | * Half-the-aperture is used as a simple heuristic. | ||||
| 1036 | * More interesting would to do search for a free | ||||
| 1037 | * block prior to making the commitment to unbind. | ||||
| 1038 | * That caters for the self-harm case, and with a | ||||
| 1039 | * little more heuristics (e.g. NOFAULT, NOEVICT) | ||||
| 1040 | * we could try to minimise harm to others. | ||||
| 1041 | */ | ||||
| 1042 | if (flags & PIN_NONBLOCK(1ULL << (2)) && | ||||
| 1043 | obj->base.size > ggtt->mappable_end / 2) | ||||
| 1044 | return ERR_PTR(-ENOSPC28); | ||||
| 1045 | } | ||||
| 1046 | |||||
| 1047 | new_vma: | ||||
| 1048 | vma = i915_vma_instance(obj, &ggtt->vm, view); | ||||
| 1049 | if (IS_ERR(vma)) | ||||
| 1050 | return vma; | ||||
| 1051 | |||||
| 1052 | if (i915_vma_misplaced(vma, size, alignment, flags)) { | ||||
| 1053 | if (flags & PIN_NONBLOCK(1ULL << (2))) { | ||||
| 1054 | if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) | ||||
| 1055 | return ERR_PTR(-ENOSPC28); | ||||
| 1056 | |||||
| 1057 | if (flags & PIN_MAPPABLE(1ULL << (3)) && | ||||
| 1058 | vma->fence_size > ggtt->mappable_end / 2) | ||||
| 1059 | return ERR_PTR(-ENOSPC28); | ||||
| 1060 | } | ||||
| 1061 | |||||
| 1062 | if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) { | ||||
| 1063 | discard_ggtt_vma(vma); | ||||
| 1064 | goto new_vma; | ||||
| 1065 | } | ||||
| 1066 | |||||
| 1067 | ret = i915_vma_unbind(vma); | ||||
| 1068 | if (ret) | ||||
| 1069 | return ERR_PTR(ret); | ||||
| 1070 | } | ||||
| 1071 | |||||
| 1072 | ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL(1ULL << (10))); | ||||
| 1073 | if (ret) | ||||
| 1074 | return ERR_PTR(ret); | ||||
| 1075 | |||||
| 1076 | if (vma->fence && !i915_gem_object_is_tiled(obj)) { | ||||
| 1077 | mutex_lock(&ggtt->vm.mutex)rw_enter_write(&ggtt->vm.mutex); | ||||
| 1078 | i915_vma_revoke_fence(vma); | ||||
| 1079 | mutex_unlock(&ggtt->vm.mutex)rw_exit_write(&ggtt->vm.mutex); | ||||
| 1080 | } | ||||
| 1081 | |||||
| 1082 | ret = i915_vma_wait_for_bind(vma); | ||||
| 1083 | if (ret) { | ||||
| 1084 | i915_vma_unpin(vma); | ||||
| 1085 | return ERR_PTR(ret); | ||||
| 1086 | } | ||||
| 1087 | |||||
| 1088 | return vma; | ||||
| 1089 | } | ||||
| 1090 | |||||
| 1091 | int | ||||
| 1092 | i915_gem_madvise_ioctl(struct drm_device *dev, void *data, | ||||
| 1093 | struct drm_file *file_priv) | ||||
| 1094 | { | ||||
| 1095 | struct drm_i915_privateinteldrm_softc *i915 = to_i915(dev); | ||||
| 1096 | struct drm_i915_gem_madvise *args = data; | ||||
| 1097 | struct drm_i915_gem_object *obj; | ||||
| 1098 | int err; | ||||
| 1099 | |||||
| 1100 | switch (args->madv) { | ||||
| 1101 | case I915_MADV_DONTNEED1: | ||||
| 1102 | case I915_MADV_WILLNEED0: | ||||
| 1103 | break; | ||||
| 1104 | default: | ||||
| 1105 | return -EINVAL22; | ||||
| 1106 | } | ||||
| 1107 | |||||
| 1108 | obj = i915_gem_object_lookup(file_priv, args->handle); | ||||
| 1109 | if (!obj) | ||||
| 1110 | return -ENOENT2; | ||||
| 1111 | |||||
| 1112 | err = mutex_lock_interruptible(&obj->mm.lock); | ||||
| 1113 | if (err) | ||||
| 1114 | goto out; | ||||
| 1115 | |||||
| 1116 | if (i915_gem_object_has_pages(obj) && | ||||
| 1117 | i915_gem_object_is_tiled(obj) && | ||||
| 1118 | i915->quirks & QUIRK_PIN_SWIZZLED_PAGES(1<<5)) { | ||||
| 1119 | if (obj->mm.madv == I915_MADV_WILLNEED0) { | ||||
| 1120 | GEM_BUG_ON(!obj->mm.quirked)((void)0); | ||||
| 1121 | __i915_gem_object_unpin_pages(obj); | ||||
| 1122 | obj->mm.quirked = false0; | ||||
| 1123 | } | ||||
| 1124 | if (args->madv == I915_MADV_WILLNEED0) { | ||||
| 1125 | GEM_BUG_ON(obj->mm.quirked)((void)0); | ||||
| 1126 | __i915_gem_object_pin_pages(obj); | ||||
| 1127 | obj->mm.quirked = true1; | ||||
| 1128 | } | ||||
| 1129 | } | ||||
| 1130 | |||||
| 1131 | if (obj->mm.madv != __I915_MADV_PURGED2) | ||||
| 1132 | obj->mm.madv = args->madv; | ||||
| 1133 | |||||
| 1134 | if (i915_gem_object_has_pages(obj)) { | ||||
| 1135 | struct list_head *list; | ||||
| 1136 | |||||
| 1137 | if (i915_gem_object_is_shrinkable(obj)) { | ||||
| 1138 | unsigned long flags; | ||||
| 1139 | |||||
| 1140 | spin_lock_irqsave(&i915->mm.obj_lock, flags)do { flags = 0; mtx_enter(&i915->mm.obj_lock); } while (0); | ||||
| 1141 | |||||
| 1142 | if (obj->mm.madv != I915_MADV_WILLNEED0) | ||||
| 1143 | list = &i915->mm.purge_list; | ||||
| 1144 | else | ||||
| 1145 | list = &i915->mm.shrink_list; | ||||
| 1146 | list_move_tail(&obj->mm.link, list); | ||||
| 1147 | |||||
| 1148 | spin_unlock_irqrestore(&i915->mm.obj_lock, flags)do { (void)(flags); mtx_leave(&i915->mm.obj_lock); } while (0); | ||||
| 1149 | } | ||||
| 1150 | } | ||||
| 1151 | |||||
| 1152 | /* if the object is no longer attached, discard its backing storage */ | ||||
| 1153 | if (obj->mm.madv == I915_MADV_DONTNEED1 && | ||||
| 1154 | !i915_gem_object_has_pages(obj)) | ||||
| 1155 | i915_gem_object_truncate(obj); | ||||
| 1156 | |||||
| 1157 | args->retained = obj->mm.madv != __I915_MADV_PURGED2; | ||||
| 1158 | mutex_unlock(&obj->mm.lock)rw_exit_write(&obj->mm.lock); | ||||
| 1159 | |||||
| 1160 | out: | ||||
| 1161 | i915_gem_object_put(obj); | ||||
| 1162 | return err; | ||||
| 1163 | } | ||||
| 1164 | |||||
| 1165 | int i915_gem_init(struct drm_i915_privateinteldrm_softc *dev_priv) | ||||
| 1166 | { | ||||
| 1167 | int ret; | ||||
| 1168 | |||||
| 1169 | /* We need to fallback to 4K pages if host doesn't support huge gtt. */ | ||||
| 1170 | if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv)) | ||||
| 1171 | mkwrite_device_info(dev_priv)->page_sizes = | ||||
| 1172 | I915_GTT_PAGE_SIZE_4K(1ULL << (12)); | ||||
| 1173 | |||||
| 1174 | ret = i915_gem_init_userptr(dev_priv); | ||||
| 1175 | if (ret) | ||||
| 1176 | return ret; | ||||
| 1177 | |||||
| 1178 | intel_uc_fetch_firmwares(&dev_priv->gt.uc); | ||||
| 1179 | intel_wopcm_init(&dev_priv->wopcm); | ||||
| 1180 | |||||
| 1181 | ret = i915_init_ggtt(dev_priv); | ||||
| 1182 | if (ret) { | ||||
| 1183 | GEM_BUG_ON(ret == -EIO)((void)0); | ||||
| 1184 | goto err_unlock; | ||||
| 1185 | } | ||||
| 1186 | |||||
| 1187 | /* | ||||
| 1188 | * Despite its name intel_init_clock_gating applies both display | ||||
| 1189 | * clock gating workarounds; GT mmio workarounds and the occasional | ||||
| 1190 | * GT power context workaround. Worse, sometimes it includes a context | ||||
| 1191 | * register workaround which we need to apply before we record the | ||||
| 1192 | * default HW state for all contexts. | ||||
| 1193 | * | ||||
| 1194 | * FIXME: break up the workarounds and apply them at the right time! | ||||
| 1195 | */ | ||||
| 1196 | intel_init_clock_gating(dev_priv); | ||||
| 1197 | |||||
| 1198 | ret = intel_gt_init(&dev_priv->gt); | ||||
| 1199 | if (ret) | ||||
| 1200 | goto err_unlock; | ||||
| 1201 | |||||
| 1202 | return 0; | ||||
| 1203 | |||||
| 1204 | /* | ||||
| 1205 | * Unwinding is complicated by that we want to handle -EIO to mean | ||||
| 1206 | * disable GPU submission but keep KMS alive. We want to mark the | ||||
| 1207 | * HW as irrevisibly wedged, but keep enough state around that the | ||||
| 1208 | * driver doesn't explode during runtime. | ||||
| 1209 | */ | ||||
| 1210 | err_unlock: | ||||
| 1211 | i915_gem_drain_workqueue(dev_priv); | ||||
| 1212 | |||||
| 1213 | if (ret != -EIO5) { | ||||
| 1214 | intel_uc_cleanup_firmwares(&dev_priv->gt.uc); | ||||
| 1215 | i915_gem_cleanup_userptr(dev_priv); | ||||
| 1216 | } | ||||
| 1217 | |||||
| 1218 | if (ret == -EIO5) { | ||||
| 1219 | /* | ||||
| 1220 | * Allow engines or uC initialisation to fail by marking the GPU | ||||
| 1221 | * as wedged. But we only want to do this when the GPU is angry, | ||||
| 1222 | * for all other failure, such as an allocation failure, bail. | ||||
| 1223 | */ | ||||
| 1224 | if (!intel_gt_is_wedged(&dev_priv->gt)) { | ||||
| 1225 | i915_probe_error(dev_priv,__i915_printk(dev_priv, 0 ? "\0017" : "\0013", "Failed to initialize GPU, declaring it wedged!\n" ) | ||||
| 1226 | "Failed to initialize GPU, declaring it wedged!\n")__i915_printk(dev_priv, 0 ? "\0017" : "\0013", "Failed to initialize GPU, declaring it wedged!\n" ); | ||||
| 1227 | intel_gt_set_wedged(&dev_priv->gt); | ||||
| 1228 | } | ||||
| 1229 | |||||
| 1230 | /* Minimal basic recovery for KMS */ | ||||
| 1231 | ret = i915_ggtt_enable_hw(dev_priv); | ||||
| 1232 | i915_ggtt_resume(&dev_priv->ggtt); | ||||
| 1233 | intel_init_clock_gating(dev_priv); | ||||
| 1234 | } | ||||
| 1235 | |||||
| 1236 | i915_gem_drain_freed_objects(dev_priv); | ||||
| 1237 | return ret; | ||||
| 1238 | } | ||||
| 1239 | |||||
| 1240 | void i915_gem_driver_register(struct drm_i915_privateinteldrm_softc *i915) | ||||
| 1241 | { | ||||
| 1242 | i915_gem_driver_register__shrinker(i915); | ||||
| 1243 | |||||
| 1244 | intel_engines_driver_register(i915); | ||||
| 1245 | } | ||||
| 1246 | |||||
| 1247 | void i915_gem_driver_unregister(struct drm_i915_privateinteldrm_softc *i915) | ||||
| 1248 | { | ||||
| 1249 | i915_gem_driver_unregister__shrinker(i915); | ||||
| 1250 | } | ||||
| 1251 | |||||
| 1252 | void i915_gem_driver_remove(struct drm_i915_privateinteldrm_softc *dev_priv) | ||||
| 1253 | { | ||||
| 1254 | intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref); | ||||
| 1255 | |||||
| 1256 | i915_gem_suspend_late(dev_priv); | ||||
| 1257 | intel_gt_driver_remove(&dev_priv->gt); | ||||
| 1258 | dev_priv->uabi_engines = RB_ROOT(struct rb_root) { ((void *)0) }; | ||||
| 1259 | |||||
| 1260 | /* Flush any outstanding unpin_work. */ | ||||
| 1261 | i915_gem_drain_workqueue(dev_priv); | ||||
| 1262 | |||||
| 1263 | i915_gem_drain_freed_objects(dev_priv); | ||||
| 1264 | } | ||||
| 1265 | |||||
| 1266 | void i915_gem_driver_release(struct drm_i915_privateinteldrm_softc *dev_priv) | ||||
| 1267 | { | ||||
| 1268 | i915_gem_driver_release__contexts(dev_priv); | ||||
| 1269 | |||||
| 1270 | intel_gt_driver_release(&dev_priv->gt); | ||||
| 1271 | |||||
| 1272 | intel_wa_list_free(&dev_priv->gt_wa_list); | ||||
| 1273 | |||||
| 1274 | intel_uc_cleanup_firmwares(&dev_priv->gt.uc); | ||||
| 1275 | i915_gem_cleanup_userptr(dev_priv); | ||||
| 1276 | |||||
| 1277 | i915_gem_drain_freed_objects(dev_priv); | ||||
| 1278 | |||||
| 1279 | drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list))({ int __ret = !!((!list_empty(&dev_priv->gem.contexts .list))); if (__ret) printf("%s %s: " "%s", dev_driver_string (((&dev_priv->drm))->dev), "", "drm_WARN_ON(" "!list_empty(&dev_priv->gem.contexts.list)" ")"); __builtin_expect(!!(__ret), 0); }); | ||||
| 1280 | } | ||||
| 1281 | |||||
| 1282 | static void i915_gem_init__mm(struct drm_i915_privateinteldrm_softc *i915) | ||||
| 1283 | { | ||||
| 1284 | mtx_init(&i915->mm.obj_lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&i915-> mm.obj_lock), ((((0x0)) > 0x0 && ((0x0)) < 0x9) ? 0x9 : ((0x0)))); } while (0); | ||||
| 1285 | |||||
| 1286 | init_llist_head(&i915->mm.free_list); | ||||
| 1287 | |||||
| 1288 | INIT_LIST_HEAD(&i915->mm.purge_list); | ||||
| 1289 | INIT_LIST_HEAD(&i915->mm.shrink_list); | ||||
| 1290 | |||||
| 1291 | i915_gem_init__objects(i915); | ||||
| 1292 | } | ||||
| 1293 | |||||
| 1294 | void i915_gem_init_early(struct drm_i915_privateinteldrm_softc *dev_priv) | ||||
| 1295 | { | ||||
| 1296 | i915_gem_init__mm(dev_priv); | ||||
| 1297 | i915_gem_init__contexts(dev_priv); | ||||
| 1298 | |||||
| 1299 | mtx_init(&dev_priv->fb_tracking.lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&dev_priv ->fb_tracking.lock), ((((0x0)) > 0x0 && ((0x0)) < 0x9) ? 0x9 : ((0x0)))); } while (0); | ||||
| 1300 | } | ||||
| 1301 | |||||
| 1302 | void i915_gem_cleanup_early(struct drm_i915_privateinteldrm_softc *dev_priv) | ||||
| 1303 | { | ||||
| 1304 | i915_gem_drain_freed_objects(dev_priv); | ||||
| 1305 | GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list))((void)0); | ||||
| 1306 | GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count))((void)0); | ||||
| 1307 | drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count)({ int __ret = !!((dev_priv->mm.shrink_count)); if (__ret) printf("%s %s: " "%s", dev_driver_string(((&dev_priv-> drm))->dev), "", "drm_WARN_ON(" "dev_priv->mm.shrink_count" ")"); __builtin_expect(!!(__ret), 0); }); | ||||
| 1308 | } | ||||
| 1309 | |||||
| 1310 | int i915_gem_freeze(struct drm_i915_privateinteldrm_softc *dev_priv) | ||||
| 1311 | { | ||||
| 1312 | /* Discard all purgeable objects, let userspace recover those as | ||||
| 1313 | * required after resuming. | ||||
| 1314 | */ | ||||
| 1315 | i915_gem_shrink_all(dev_priv); | ||||
| 1316 | |||||
| 1317 | return 0; | ||||
| 1318 | } | ||||
| 1319 | |||||
| 1320 | int i915_gem_freeze_late(struct drm_i915_privateinteldrm_softc *i915) | ||||
| 1321 | { | ||||
| 1322 | struct drm_i915_gem_object *obj; | ||||
| 1323 | intel_wakeref_t wakeref; | ||||
| 1324 | |||||
| 1325 | /* | ||||
| 1326 | * Called just before we write the hibernation image. | ||||
| 1327 | * | ||||
| 1328 | * We need to update the domain tracking to reflect that the CPU | ||||
| 1329 | * will be accessing all the pages to create and restore from the | ||||
| 1330 | * hibernation, and so upon restoration those pages will be in the | ||||
| 1331 | * CPU domain. | ||||
| 1332 | * | ||||
| 1333 | * To make sure the hibernation image contains the latest state, | ||||
| 1334 | * we update that state just before writing out the image. | ||||
| 1335 | * | ||||
| 1336 | * To try and reduce the hibernation image, we manually shrink | ||||
| 1337 | * the objects as well, see i915_gem_freeze() | ||||
| 1338 | */ | ||||
| 1339 | |||||
| 1340 | wakeref = intel_runtime_pm_get(&i915->runtime_pm); | ||||
| 1341 | |||||
| 1342 | i915_gem_shrink(i915, -1UL, NULL((void *)0), ~0); | ||||
| 1343 | i915_gem_drain_freed_objects(i915); | ||||
| 1344 | |||||
| 1345 | list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)for (obj = ({ const __typeof( ((__typeof(*obj) *)0)->mm.link ) *__mptr = ((&i915->mm.shrink_list)->next); (__typeof (*obj) *)( (char *)__mptr - __builtin_offsetof(__typeof(*obj) , mm.link) );}); &obj->mm.link != (&i915->mm.shrink_list ); obj = ({ const __typeof( ((__typeof(*obj) *)0)->mm.link ) *__mptr = (obj->mm.link.next); (__typeof(*obj) *)( (char *)__mptr - __builtin_offsetof(__typeof(*obj), mm.link) );})) { | ||||
| 1346 | i915_gem_object_lock(obj, NULL((void *)0)); | ||||
| 1347 | drm_WARN_ON(&i915->drm,({ int __ret = !!((i915_gem_object_set_to_cpu_domain(obj, 1)) ); if (__ret) printf("%s %s: " "%s", dev_driver_string(((& i915->drm))->dev), "", "drm_WARN_ON(" "i915_gem_object_set_to_cpu_domain(obj, 1)" ")"); __builtin_expect(!!(__ret), 0); }) | ||||
| 1348 | i915_gem_object_set_to_cpu_domain(obj, true))({ int __ret = !!((i915_gem_object_set_to_cpu_domain(obj, 1)) ); if (__ret) printf("%s %s: " "%s", dev_driver_string(((& i915->drm))->dev), "", "drm_WARN_ON(" "i915_gem_object_set_to_cpu_domain(obj, 1)" ")"); __builtin_expect(!!(__ret), 0); }); | ||||
| 1349 | i915_gem_object_unlock(obj); | ||||
| 1350 | } | ||||
| 1351 | |||||
| 1352 | intel_runtime_pm_put(&i915->runtime_pm, wakeref); | ||||
| 1353 | |||||
| 1354 | return 0; | ||||
| 1355 | } | ||||
| 1356 | |||||
| 1357 | int i915_gem_open(struct drm_i915_privateinteldrm_softc *i915, struct drm_file *file) | ||||
| 1358 | { | ||||
| 1359 | struct drm_i915_file_private *file_priv; | ||||
| 1360 | int ret; | ||||
| 1361 | |||||
| 1362 | DRM_DEBUG("\n")__drm_dbg(DRM_UT_CORE, "\n"); | ||||
| 1363 | |||||
| 1364 | file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL(0x0001 | 0x0004)); | ||||
| 1365 | if (!file_priv) | ||||
| 1366 | return -ENOMEM12; | ||||
| 1367 | |||||
| 1368 | file->driver_priv = file_priv; | ||||
| 1369 | file_priv->dev_priv = i915; | ||||
| 1370 | file_priv->file = file; | ||||
| 1371 | |||||
| 1372 | file_priv->bsd_engine = -1; | ||||
| 1373 | file_priv->hang_timestamp = jiffies; | ||||
| 1374 | |||||
| 1375 | ret = i915_gem_context_open(i915, file); | ||||
| 1376 | if (ret) | ||||
| 1377 | kfree(file_priv); | ||||
| 1378 | |||||
| 1379 | return ret; | ||||
| 1380 | } | ||||
| 1381 | |||||
| 1382 | void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool_Bool intr) | ||||
| 1383 | { | ||||
| 1384 | ww_acquire_init(&ww->ctx, &reservation_ww_class); | ||||
| 1385 | INIT_LIST_HEAD(&ww->obj_list); | ||||
| 1386 | ww->intr = intr; | ||||
| 1387 | ww->contended = NULL((void *)0); | ||||
| 1388 | } | ||||
| 1389 | |||||
| 1390 | static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww) | ||||
| 1391 | { | ||||
| 1392 | struct drm_i915_gem_object *obj; | ||||
| 1393 | |||||
| 1394 | while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link)(list_empty(&ww->obj_list) ? ((void *)0) : ({ const __typeof ( ((struct drm_i915_gem_object *)0)->obj_link ) *__mptr = ( (&ww->obj_list)->next); (struct drm_i915_gem_object *)( (char *)__mptr - __builtin_offsetof(struct drm_i915_gem_object , obj_link) );})))) { | ||||
| 1395 | list_del(&obj->obj_link); | ||||
| 1396 | i915_gem_object_unlock(obj); | ||||
| 1397 | } | ||||
| 1398 | } | ||||
| 1399 | |||||
| 1400 | void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj) | ||||
| 1401 | { | ||||
| 1402 | list_del(&obj->obj_link); | ||||
| 1403 | i915_gem_object_unlock(obj); | ||||
| 1404 | } | ||||
| 1405 | |||||
| 1406 | void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww) | ||||
| 1407 | { | ||||
| 1408 | i915_gem_ww_ctx_unlock_all(ww); | ||||
| 1409 | WARN_ON(ww->contended)({ int __ret = !!((ww->contended)); if (__ret) printf("%s" , "WARN_ON(" "ww->contended" ")"); __builtin_expect(!!(__ret ), 0); }); | ||||
| 1410 | ww_acquire_fini(&ww->ctx); | ||||
| 1411 | } | ||||
| 1412 | |||||
| 1413 | int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww) | ||||
| 1414 | { | ||||
| 1415 | int ret = 0; | ||||
| 1416 | |||||
| 1417 | if (WARN_ON(!ww->contended)({ int __ret = !!((!ww->contended)); if (__ret) printf("%s" , "WARN_ON(" "!ww->contended" ")"); __builtin_expect(!!(__ret ), 0); })) | ||||
| 1418 | return -EINVAL22; | ||||
| 1419 | |||||
| 1420 | i915_gem_ww_ctx_unlock_all(ww); | ||||
| 1421 | if (ww->intr) | ||||
| 1422 | ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx); | ||||
| 1423 | else | ||||
| 1424 | dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx); | ||||
| 1425 | |||||
| 1426 | if (!ret) | ||||
| 1427 | list_add_tail(&ww->contended->obj_link, &ww->obj_list); | ||||
| 1428 | |||||
| 1429 | ww->contended = NULL((void *)0); | ||||
| 1430 | |||||
| 1431 | return ret; | ||||
| 1432 | } | ||||
| 1433 | |||||
| 1434 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0 | ||||
| 1435 | #include "selftests/mock_gem_device.c" | ||||
| 1436 | #include "selftests/i915_gem.c" | ||||
| 1437 | #endif |
| 1 | /* $OpenBSD: list.h,v 1.4 2021/10/01 04:36:38 jsg Exp $ */ | |||
| 2 | /* drm_linux_list.h -- linux list functions for the BSDs. | |||
| 3 | * Created: Mon Apr 7 14:30:16 1999 by anholt@FreeBSD.org | |||
| 4 | */ | |||
| 5 | /*- | |||
| 6 | * Copyright 2003 Eric Anholt | |||
| 7 | * All Rights Reserved. | |||
| 8 | * | |||
| 9 | * Permission is hereby granted, free of charge, to any person obtaining a | |||
| 10 | * copy of this software and associated documentation files (the "Software"), | |||
| 11 | * to deal in the Software without restriction, including without limitation | |||
| 12 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |||
| 13 | * and/or sell copies of the Software, and to permit persons to whom the | |||
| 14 | * Software is furnished to do so, subject to the following conditions: | |||
| 15 | * | |||
| 16 | * The above copyright notice and this permission notice (including the next | |||
| 17 | * paragraph) shall be included in all copies or substantial portions of the | |||
| 18 | * Software. | |||
| 19 | * | |||
| 20 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
| 21 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
| 22 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |||
| 23 | * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR | |||
| 24 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |||
| 25 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |||
| 26 | * OTHER DEALINGS IN THE SOFTWARE. | |||
| 27 | * | |||
| 28 | * Authors: | |||
| 29 | * Eric Anholt <anholt@FreeBSD.org> | |||
| 30 | * | |||
| 31 | */ | |||
| 32 | ||||
| 33 | #ifndef _DRM_LINUX_LIST_H_ | |||
| 34 | #define _DRM_LINUX_LIST_H_ | |||
| 35 | ||||
| 36 | #include <sys/param.h> | |||
| 37 | #include <linux/kernel.h> | |||
| 38 | #include <linux/types.h> | |||
| 39 | #include <linux/poison.h> | |||
| 40 | ||||
| 41 | #define list_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); ( type *)( (char *)__mptr - __builtin_offsetof(type, member) ); }) container_of(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); ( type *)( (char *)__mptr - __builtin_offsetof(type, member) ); }) | |||
| 42 | ||||
| 43 | static inline void | |||
| 44 | INIT_LIST_HEAD(struct list_head *head) { | |||
| 45 | (head)->next = head; | |||
| 46 | (head)->prev = head; | |||
| 47 | } | |||
| 48 | ||||
| 49 | #define LIST_HEAD_INIT(name){ &(name), &(name) } { &(name), &(name) } | |||
| 50 | ||||
| 51 | #define DRM_LIST_HEAD(name)struct list_head name = { &(name), &(name) } \ | |||
| 52 | struct list_head name = LIST_HEAD_INIT(name){ &(name), &(name) } | |||
| 53 | ||||
| 54 | static inline int | |||
| 55 | list_empty(const struct list_head *head) { | |||
| 56 | return (head)->next == head; | |||
| 57 | } | |||
| 58 | ||||
| 59 | static inline int | |||
| 60 | list_is_singular(const struct list_head *head) { | |||
| 61 | return !list_empty(head) && ((head)->next == (head)->prev); | |||
| 62 | } | |||
| 63 | ||||
| 64 | static inline int | |||
| 65 | list_is_first(const struct list_head *list, | |||
| 66 | const struct list_head *head) | |||
| 67 | { | |||
| 68 | return list->prev == head; | |||
| 69 | } | |||
| 70 | ||||
| 71 | static inline int | |||
| 72 | list_is_last(const struct list_head *list, | |||
| 73 | const struct list_head *head) | |||
| 74 | { | |||
| 75 | return list->next == head; | |||
| 76 | } | |||
| 77 | ||||
| 78 | static inline void | |||
| 79 | list_add(struct list_head *new, struct list_head *head) { | |||
| 80 | (head)->next->prev = new; | |||
| 81 | (new)->next = (head)->next; | |||
| 82 | (new)->prev = head; | |||
| 83 | (head)->next = new; | |||
| 84 | } | |||
| 85 | ||||
| 86 | static inline void | |||
| 87 | list_add_tail(struct list_head *entry, struct list_head *head) { | |||
| 88 | (entry)->prev = (head)->prev; | |||
| 89 | (entry)->next = head; | |||
| 90 | (head)->prev->next = entry; | |||
| 91 | (head)->prev = entry; | |||
| 92 | } | |||
| 93 | ||||
| 94 | static inline void | |||
| 95 | list_del(struct list_head *entry) { | |||
| 96 | (entry)->next->prev = (entry)->prev; | |||
| 97 | (entry)->prev->next = (entry)->next; | |||
| 98 | } | |||
| 99 | ||||
| 100 | #define __list_del_entry(x)list_del(x) list_del(x) | |||
| 101 | ||||
| 102 | static inline void list_replace(struct list_head *old, | |||
| 103 | struct list_head *new) | |||
| 104 | { | |||
| 105 | new->next = old->next; | |||
| 106 | new->next->prev = new; | |||
| 107 | new->prev = old->prev; | |||
| 108 | new->prev->next = new; | |||
| 109 | } | |||
| 110 | ||||
| 111 | static inline void list_replace_init(struct list_head *old, | |||
| 112 | struct list_head *new) | |||
| 113 | { | |||
| 114 | list_replace(old, new); | |||
| 115 | INIT_LIST_HEAD(old); | |||
| 116 | } | |||
| 117 | ||||
| 118 | static inline void list_move(struct list_head *list, struct list_head *head) | |||
| 119 | { | |||
| 120 | list_del(list); | |||
| 121 | list_add(list, head); | |||
| 122 | } | |||
| 123 | ||||
| 124 | static inline void list_move_tail(struct list_head *list, | |||
| 125 | struct list_head *head) | |||
| 126 | { | |||
| 127 | list_del(list); | |||
| 128 | list_add_tail(list, head); | |||
| 129 | } | |||
| 130 | ||||
| 131 | static inline void | |||
| 132 | list_rotate_to_front(struct list_head *list, struct list_head *head) | |||
| 133 | { | |||
| 134 | list_del(head); | |||
| 135 | list_add_tail(head, list); | |||
| 136 | } | |||
| 137 | ||||
| 138 | static inline void | |||
| 139 | list_bulk_move_tail(struct list_head *head, struct list_head *first, | |||
| 140 | struct list_head *last) | |||
| 141 | { | |||
| 142 | first->prev->next = last->next; | |||
| 143 | last->next->prev = first->prev; | |||
| 144 | head->prev->next = first; | |||
| 145 | first->prev = head->prev; | |||
| 146 | last->next = head; | |||
| 147 | head->prev = last; | |||
| 148 | } | |||
| 149 | ||||
| 150 | static inline void | |||
| 151 | list_del_init(struct list_head *entry) { | |||
| 152 | (entry)->next->prev = (entry)->prev; | |||
| 153 | (entry)->prev->next = (entry)->next; | |||
| 154 | INIT_LIST_HEAD(entry); | |||
| 155 | } | |||
| 156 | ||||
| 157 | #define list_next_entry(pos, member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) \ | |||
| 158 | list_entry(((pos)->member.next), typeof(*(pos)), member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) | |||
| 159 | ||||
| 160 | #define list_prev_entry(pos, member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.prev)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) \ | |||
| 161 | list_entry(((pos)->member.prev), typeof(*(pos)), member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.prev)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) | |||
| 162 | ||||
| 163 | #define list_safe_reset_next(pos, n, member)n = ({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) \ | |||
| 164 | n = list_next_entry(pos, member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) | |||
| 165 | ||||
| 166 | #define list_for_each(entry, head)for (entry = (head)->next; entry != head; entry = (entry)-> next) \ | |||
| 167 | for (entry = (head)->next; entry != head; entry = (entry)->next) | |||
| 168 | ||||
| 169 | #define list_for_each_prev(entry, head)for (entry = (head)->prev; entry != (head); entry = entry-> prev) \ | |||
| 170 | for (entry = (head)->prev; entry != (head); \ | |||
| 171 | entry = entry->prev) | |||
| 172 | ||||
| 173 | #define list_for_each_safe(entry, temp, head)for (entry = (head)->next, temp = (entry)->next; entry != head; entry = temp, temp = entry->next) \ | |||
| 174 | for (entry = (head)->next, temp = (entry)->next; \ | |||
| 175 | entry != head; \ | |||
| 176 | entry = temp, temp = entry->next) | |||
| 177 | ||||
| 178 | #define list_for_each_entry_safe_reverse(pos, n, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}), n = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos )->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); &(pos)->member != (head ); pos = n, n = ({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.prev); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof(__typeof(*n), member) );})) \ | |||
| 179 | for (pos = list_entry((head)->prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}), \ | |||
| 180 | n = list_entry((pos)->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos)->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 181 | &(pos)->member != (head); \ | |||
| 182 | pos = n, n = list_entry(n->member.prev, __typeof(*n), member)({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.prev); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof (__typeof(*n), member) );})) | |||
| 183 | ||||
| 184 | #define list_for_each_entry_safe_from(pos, n, head, member)for (n = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char * )__mptr - __builtin_offsetof(__typeof(*pos), member) );}); & pos->member != (head); pos = n, n = ({ const __typeof( ((__typeof (*n) *)0)->member ) *__mptr = (n->member.next); (__typeof (*n) *)( (char *)__mptr - __builtin_offsetof(__typeof(*n), member ) );})) \ | |||
| 185 | for (n = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 186 | &pos->member != (head); \ | |||
| 187 | pos = n, n = list_entry(n->member.next, __typeof(*n), member)({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.next); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof (__typeof(*n), member) );})) | |||
| 188 | ||||
| 189 | #define list_for_each_entry(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); &pos-> member != (head); pos = ({ const __typeof( ((__typeof(*pos) * )0)->member ) *__mptr = (pos->member.next); (__typeof(* pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );})) \ | |||
| 190 | for (pos = list_entry((head)->next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); \ | |||
| 191 | &pos->member != (head); \ | |||
| 192 | pos = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 193 | ||||
| 194 | #define list_for_each_entry_from(pos, head, member)for (; &pos->member != (head); pos = ({ const __typeof ( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member .next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );})) \ | |||
| 195 | for (; \ | |||
| 196 | &pos->member != (head); \ | |||
| 197 | pos = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 198 | ||||
| 199 | #define list_for_each_entry_reverse(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); &pos-> member != (head); pos = ({ const __typeof( ((__typeof(*pos) * )0)->member ) *__mptr = (pos->member.prev); (__typeof(* pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );})) \ | |||
| 200 | for (pos = list_entry((head)->prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); \ | |||
| 201 | &pos->member != (head); \ | |||
| 202 | pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 203 | ||||
| 204 | #define list_for_each_entry_from_reverse(pos, head, member)for (; &pos->member != (head); pos = ({ const __typeof ( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member .prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );})) \ | |||
| 205 | for (; \ | |||
| 206 | &pos->member != (head); \ | |||
| 207 | pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 208 | ||||
| 209 | #define list_for_each_entry_continue(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos)->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); & pos->member != (head); pos = ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );})) \ | |||
| 210 | for (pos = list_entry((pos)->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos)->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 211 | &pos->member != (head); \ | |||
| 212 | pos = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 213 | ||||
| 214 | #define list_for_each_entry_continue_reverse(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); & pos->member != (head); pos = ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );})) \ | |||
| 215 | for (pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 216 | &pos->member != (head); \ | |||
| 217 | pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 218 | ||||
| 219 | /** | |||
| 220 | * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry | |||
| 221 | * @pos: the type * to use as a loop cursor. | |||
| 222 | * @n: another type * to use as temporary storage | |||
| 223 | * @head: the head for your list. | |||
| 224 | * @member: the name of the list_struct within the struct. | |||
| 225 | */ | |||
| 226 | #define list_for_each_entry_safe(pos, n, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}), n = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos-> member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); &pos->member != (head); pos = n, n = ({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.next); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof(__typeof(*n), member) );})) \ | |||
| 227 | for (pos = list_entry((head)->next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}), \ | |||
| 228 | n = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 229 | &pos->member != (head); \ | |||
| 230 | pos = n, n = list_entry(n->member.next, __typeof(*n), member)({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.next); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof (__typeof(*n), member) );})) | |||
| 231 | ||||
| 232 | #define list_first_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> next); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) \ | |||
| 233 | list_entry((ptr)->next, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> next); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) | |||
| 234 | ||||
| 235 | #define list_first_entry_or_null(ptr, type, member)(list_empty(ptr) ? ((void *)0) : ({ const __typeof( ((type *) 0)->member ) *__mptr = ((ptr)->next); (type *)( (char * )__mptr - __builtin_offsetof(type, member) );})) \ | |||
| 236 | (list_empty(ptr) ? NULL((void *)0) : list_first_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> next); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );})) | |||
| 237 | ||||
| 238 | #define list_last_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> prev); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) \ | |||
| 239 | list_entry((ptr)->prev, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> prev); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) | |||
| 240 | ||||
| 241 | static inline void | |||
| 242 | __list_splice(const struct list_head *list, struct list_head *prev, | |||
| 243 | struct list_head *next) | |||
| 244 | { | |||
| 245 | struct list_head *first = list->next; | |||
| 246 | struct list_head *last = list->prev; | |||
| 247 | ||||
| 248 | first->prev = prev; | |||
| 249 | prev->next = first; | |||
| 250 | ||||
| 251 | last->next = next; | |||
| 252 | next->prev = last; | |||
| ||||
| 253 | } | |||
| 254 | ||||
| 255 | static inline void | |||
| 256 | list_splice(const struct list_head *list, struct list_head *head) | |||
| 257 | { | |||
| 258 | if (list_empty(list)) | |||
| 259 | return; | |||
| 260 | ||||
| 261 | __list_splice(list, head, head->next); | |||
| 262 | } | |||
| 263 | ||||
| 264 | static inline void | |||
| 265 | list_splice_init(struct list_head *list, struct list_head *head) | |||
| 266 | { | |||
| 267 | if (list_empty(list)) | |||
| 268 | return; | |||
| 269 | ||||
| 270 | __list_splice(list, head, head->next); | |||
| 271 | INIT_LIST_HEAD(list); | |||
| 272 | } | |||
| 273 | ||||
| 274 | static inline void | |||
| 275 | list_splice_tail(const struct list_head *list, struct list_head *head) | |||
| 276 | { | |||
| 277 | if (list_empty(list)) | |||
| 278 | return; | |||
| 279 | ||||
| 280 | __list_splice(list, head->prev, head); | |||
| 281 | } | |||
| 282 | ||||
| 283 | static inline void | |||
| 284 | list_splice_tail_init(struct list_head *list, struct list_head *head) | |||
| 285 | { | |||
| 286 | if (list_empty(list)) | |||
| 287 | return; | |||
| 288 | ||||
| 289 | __list_splice(list, head->prev, head); | |||
| 290 | INIT_LIST_HEAD(list); | |||
| 291 | } | |||
| 292 | ||||
| 293 | void list_sort(void *, struct list_head *, | |||
| 294 | int (*)(void *, const struct list_head *, const struct list_head *)); | |||
| 295 | ||||
| 296 | #define hlist_entry(ptr, type, member)((ptr) ? ({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) : ((void *)0)) \ | |||
| 297 | ((ptr) ? container_of(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); ( type *)( (char *)__mptr - __builtin_offsetof(type, member) ); }) : NULL((void *)0)) | |||
| 298 | ||||
| 299 | static inline void | |||
| 300 | INIT_HLIST_HEAD(struct hlist_head *head) { | |||
| 301 | head->first = NULL((void *)0); | |||
| 302 | } | |||
| 303 | ||||
| 304 | static inline int | |||
| 305 | hlist_empty(const struct hlist_head *head) { | |||
| 306 | return head->first == NULL((void *)0); | |||
| 307 | } | |||
| 308 | ||||
| 309 | static inline void | |||
| 310 | hlist_add_head(struct hlist_node *new, struct hlist_head *head) | |||
| 311 | { | |||
| 312 | if ((new->next = head->first) != NULL((void *)0)) | |||
| 313 | head->first->prev = &new->next; | |||
| 314 | head->first = new; | |||
| 315 | new->prev = &head->first; | |||
| 316 | } | |||
| 317 | ||||
| 318 | static inline void | |||
| 319 | hlist_del_init(struct hlist_node *node) | |||
| 320 | { | |||
| 321 | if (node->next != NULL((void *)0)) | |||
| 322 | node->next->prev = node->prev; | |||
| 323 | *(node->prev) = node->next; | |||
| 324 | node->next = NULL((void *)0); | |||
| 325 | node->prev = NULL((void *)0); | |||
| 326 | } | |||
| 327 | ||||
| 328 | #define hlist_for_each(pos, head)for (pos = (head)->first; pos != ((void *)0); pos = pos-> next) \ | |||
| 329 | for (pos = (head)->first; pos != NULL((void *)0); pos = pos->next) | |||
| 330 | ||||
| 331 | #define hlist_for_each_entry(pos, head, member)for (pos = (((head)->first) ? ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = ((head)->first); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );}) : ((void *)0)); pos != ((void *)0); pos = (((pos )->member.next) ? ({ const __typeof( ((__typeof(*pos) *)0) ->member ) *__mptr = ((pos)->member.next); (__typeof(*pos ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0))) \ | |||
| 332 | for (pos = hlist_entry((head)->first, __typeof(*pos), member)(((head)->first) ? ({ const __typeof( ((__typeof(*pos) *)0 )->member ) *__mptr = ((head)->first); (__typeof(*pos) * )( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0)); \ | |||
| 333 | pos != NULL((void *)0); \ | |||
| 334 | pos = hlist_entry((pos)->member.next, __typeof(*pos), member)(((pos)->member.next) ? ({ const __typeof( ((__typeof(*pos ) *)0)->member ) *__mptr = ((pos)->member.next); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );}) : ((void *)0))) | |||
| 335 | ||||
| 336 | #define hlist_for_each_entry_safe(pos, n, head, member)for (pos = (((head)->first) ? ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = ((head)->first); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );}) : ((void *)0)); pos != ((void *)0) && ( n = pos->member.next, 1); pos = ((n) ? ({ const __typeof( ( (__typeof(*pos) *)0)->member ) *__mptr = (n); (__typeof(*pos ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0))) \ | |||
| 337 | for (pos = hlist_entry((head)->first, __typeof(*pos), member)(((head)->first) ? ({ const __typeof( ((__typeof(*pos) *)0 )->member ) *__mptr = ((head)->first); (__typeof(*pos) * )( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0)); \ | |||
| 338 | pos != NULL((void *)0) && (n = pos->member.next, 1); \ | |||
| 339 | pos = hlist_entry(n, __typeof(*pos), member)((n) ? ({ const __typeof( ((__typeof(*pos) *)0)->member ) * __mptr = (n); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}) : ((void *)0))) | |||
| 340 | ||||
| 341 | #endif /* _DRM_LINUX_LIST_H_ */ |