| File: | dev/pci/drm/include/linux/list.h |
| Warning: | line 80, column 28 Access to field 'prev' results in a dereference of an undefined pointer value (loaded from field 'next') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | // SPDX-License-Identifier: MIT | ||||||
| 2 | /* | ||||||
| 3 | * Copyright © 2019 Intel Corporation | ||||||
| 4 | */ | ||||||
| 5 | |||||||
| 6 | #include <linux/kmemleak.h> | ||||||
| 7 | #include <linux/slab.h> | ||||||
| 8 | |||||||
| 9 | #include "i915_buddy.h" | ||||||
| 10 | |||||||
| 11 | #include "i915_gem.h" | ||||||
| 12 | #include "i915_globals.h" | ||||||
| 13 | #include "i915_utils.h" | ||||||
| 14 | |||||||
| 15 | static struct i915_global_block { | ||||||
| 16 | struct i915_global base; | ||||||
| 17 | #ifdef __linux__ | ||||||
| 18 | struct kmem_cache *slab_blocks; | ||||||
| 19 | #else | ||||||
| 20 | struct pool slab_blocks; | ||||||
| 21 | #endif | ||||||
| 22 | } global; | ||||||
| 23 | |||||||
| 24 | static void i915_global_buddy_shrink(void) | ||||||
| 25 | { | ||||||
| 26 | #ifdef notyet | ||||||
| 27 | kmem_cache_shrink(global.slab_blocks); | ||||||
| 28 | #endif | ||||||
| 29 | } | ||||||
| 30 | |||||||
| 31 | static void i915_global_buddy_exit(void) | ||||||
| 32 | { | ||||||
| 33 | #ifdef __linux__ | ||||||
| 34 | kmem_cache_destroy(global.slab_blocks); | ||||||
| 35 | #else | ||||||
| 36 | pool_destroy(&global.slab_blocks); | ||||||
| 37 | #endif | ||||||
| 38 | } | ||||||
| 39 | |||||||
| 40 | static struct i915_global_block global = { { | ||||||
| 41 | .shrink = i915_global_buddy_shrink, | ||||||
| 42 | .exit = i915_global_buddy_exit, | ||||||
| 43 | } }; | ||||||
| 44 | |||||||
| 45 | int __init i915_global_buddy_init(void) | ||||||
| 46 | { | ||||||
| 47 | #ifdef __linux__ | ||||||
| 48 | global.slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN); | ||||||
| 49 | if (!global.slab_blocks) | ||||||
| 50 | return -ENOMEM12; | ||||||
| 51 | #else | ||||||
| 52 | pool_init(&global.slab_blocks, sizeof(struct i915_buddy_block), | ||||||
| 53 | CACHELINESIZE64, IPL_NONE0x0, 0, "i915bb", NULL((void *)0)); | ||||||
| 54 | #endif | ||||||
| 55 | |||||||
| 56 | i915_global_register(&global.base); | ||||||
| 57 | return 0; | ||||||
| 58 | } | ||||||
| 59 | |||||||
| 60 | static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_block *parent, | ||||||
| 61 | unsigned int order, | ||||||
| 62 | u64 offset) | ||||||
| 63 | { | ||||||
| 64 | struct i915_buddy_block *block; | ||||||
| 65 | |||||||
| 66 | #ifdef __linux__ | ||||||
| 67 | block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL(0x0001 | 0x0004)); | ||||||
| 68 | #else | ||||||
| 69 | block = pool_get(&global.slab_blocks, PR_WAITOK0x0001 | PR_ZERO0x0008); | ||||||
| 70 | #endif | ||||||
| 71 | if (!block) | ||||||
| 72 | return NULL((void *)0); | ||||||
| 73 | |||||||
| 74 | block->header = offset; | ||||||
| 75 | block->header |= order; | ||||||
| 76 | block->parent = parent; | ||||||
| 77 | |||||||
| 78 | return block; | ||||||
| 79 | } | ||||||
| 80 | |||||||
| 81 | static void i915_block_free(struct i915_buddy_block *block) | ||||||
| 82 | { | ||||||
| 83 | #ifdef __linux__ | ||||||
| 84 | kmem_cache_free(global.slab_blocks, block); | ||||||
| 85 | #else | ||||||
| 86 | pool_put(&global.slab_blocks, block); | ||||||
| 87 | #endif | ||||||
| 88 | } | ||||||
| 89 | |||||||
| 90 | static void mark_allocated(struct i915_buddy_block *block) | ||||||
| 91 | { | ||||||
| 92 | block->header &= ~I915_BUDDY_HEADER_STATE(((~0ULL) >> (64 - (11) - 1)) & ((~0ULL) << ( 10))); | ||||||
| 93 | block->header |= I915_BUDDY_ALLOCATED(1 << 10); | ||||||
| 94 | |||||||
| 95 | list_del(&block->link); | ||||||
| 96 | } | ||||||
| 97 | |||||||
| 98 | static void mark_free(struct i915_buddy_mm *mm, | ||||||
| 99 | struct i915_buddy_block *block) | ||||||
| 100 | { | ||||||
| 101 | block->header &= ~I915_BUDDY_HEADER_STATE(((~0ULL) >> (64 - (11) - 1)) & ((~0ULL) << ( 10))); | ||||||
| 102 | block->header |= I915_BUDDY_FREE(2 << 10); | ||||||
| 103 | |||||||
| 104 | list_add(&block->link, | ||||||
| 105 | &mm->free_list[i915_buddy_block_order(block)]); | ||||||
| 106 | } | ||||||
| 107 | |||||||
| 108 | static void mark_split(struct i915_buddy_block *block) | ||||||
| 109 | { | ||||||
| 110 | block->header &= ~I915_BUDDY_HEADER_STATE(((~0ULL) >> (64 - (11) - 1)) & ((~0ULL) << ( 10))); | ||||||
| 111 | block->header |= I915_BUDDY_SPLIT(3 << 10); | ||||||
| 112 | |||||||
| 113 | list_del(&block->link); | ||||||
| 114 | } | ||||||
| 115 | |||||||
| 116 | int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size) | ||||||
| 117 | { | ||||||
| 118 | unsigned int i; | ||||||
| 119 | u64 offset; | ||||||
| 120 | |||||||
| 121 | if (size < chunk_size) | ||||||
| |||||||
| 122 | return -EINVAL22; | ||||||
| 123 | |||||||
| 124 | if (chunk_size < PAGE_SIZE(1 << 12)) | ||||||
| 125 | return -EINVAL22; | ||||||
| 126 | |||||||
| 127 | if (!is_power_of_2(chunk_size)(((chunk_size) != 0) && (((chunk_size) - 1) & (chunk_size )) == 0)) | ||||||
| 128 | return -EINVAL22; | ||||||
| 129 | |||||||
| 130 | size = round_down(size, chunk_size)(((size) / (chunk_size)) * (chunk_size)); | ||||||
| 131 | |||||||
| 132 | mm->size = size; | ||||||
| 133 | mm->chunk_size = chunk_size; | ||||||
| 134 | mm->max_order = ilog2(size)((sizeof(size) <= 4) ? (fls(size) - 1) : (flsl(size) - 1)) - ilog2(chunk_size)((sizeof(chunk_size) <= 4) ? (fls(chunk_size) - 1) : (flsl (chunk_size) - 1)); | ||||||
| 135 | |||||||
| 136 | GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER)((void)0); | ||||||
| 137 | |||||||
| 138 | mm->free_list = kmalloc_array(mm->max_order + 1, | ||||||
| 139 | sizeof(struct list_head), | ||||||
| 140 | GFP_KERNEL(0x0001 | 0x0004)); | ||||||
| 141 | if (!mm->free_list) | ||||||
| 142 | return -ENOMEM12; | ||||||
| 143 | |||||||
| 144 | for (i = 0; i
| ||||||
| 145 | INIT_LIST_HEAD(&mm->free_list[i]); | ||||||
| 146 | |||||||
| 147 | mm->n_roots = hweight64(size); | ||||||
| 148 | |||||||
| 149 | mm->roots = kmalloc_array(mm->n_roots, | ||||||
| 150 | sizeof(struct i915_buddy_block *), | ||||||
| 151 | GFP_KERNEL(0x0001 | 0x0004)); | ||||||
| 152 | if (!mm->roots) | ||||||
| 153 | goto out_free_list; | ||||||
| 154 | |||||||
| 155 | offset = 0; | ||||||
| 156 | i = 0; | ||||||
| 157 | |||||||
| 158 | /* | ||||||
| 159 | * Split into power-of-two blocks, in case we are given a size that is | ||||||
| 160 | * not itself a power-of-two. | ||||||
| 161 | */ | ||||||
| 162 | do { | ||||||
| 163 | struct i915_buddy_block *root; | ||||||
| 164 | unsigned int order; | ||||||
| 165 | u64 root_size; | ||||||
| 166 | |||||||
| 167 | root_size = rounddown_pow_of_two(size); | ||||||
| 168 | order = ilog2(root_size)((sizeof(root_size) <= 4) ? (fls(root_size) - 1) : (flsl(root_size ) - 1)) - ilog2(chunk_size)((sizeof(chunk_size) <= 4) ? (fls(chunk_size) - 1) : (flsl (chunk_size) - 1)); | ||||||
| 169 | |||||||
| 170 | root = i915_block_alloc(NULL((void *)0), order, offset); | ||||||
| 171 | if (!root
| ||||||
| 172 | goto out_free_roots; | ||||||
| 173 | |||||||
| 174 | mark_free(mm, root); | ||||||
| 175 | |||||||
| 176 | GEM_BUG_ON(i > mm->max_order)((void)0); | ||||||
| 177 | GEM_BUG_ON(i915_buddy_block_size(mm, root) < chunk_size)((void)0); | ||||||
| 178 | |||||||
| 179 | mm->roots[i] = root; | ||||||
| 180 | |||||||
| 181 | offset += root_size; | ||||||
| 182 | size -= root_size; | ||||||
| 183 | i++; | ||||||
| 184 | } while (size); | ||||||
| 185 | |||||||
| 186 | return 0; | ||||||
| 187 | |||||||
| 188 | out_free_roots: | ||||||
| 189 | while (i--) | ||||||
| 190 | i915_block_free(mm->roots[i]); | ||||||
| 191 | kfree(mm->roots); | ||||||
| 192 | out_free_list: | ||||||
| 193 | kfree(mm->free_list); | ||||||
| 194 | return -ENOMEM12; | ||||||
| 195 | } | ||||||
| 196 | |||||||
| 197 | void i915_buddy_fini(struct i915_buddy_mm *mm) | ||||||
| 198 | { | ||||||
| 199 | int i; | ||||||
| 200 | |||||||
| 201 | for (i = 0; i < mm->n_roots; ++i) { | ||||||
| 202 | GEM_WARN_ON(!i915_buddy_block_is_free(mm->roots[i]))({ __builtin_expect(!!(!!(!i915_buddy_block_is_free(mm->roots [i]))), 0); }); | ||||||
| 203 | i915_block_free(mm->roots[i]); | ||||||
| 204 | } | ||||||
| 205 | |||||||
| 206 | kfree(mm->roots); | ||||||
| 207 | kfree(mm->free_list); | ||||||
| 208 | } | ||||||
| 209 | |||||||
| 210 | static int split_block(struct i915_buddy_mm *mm, | ||||||
| 211 | struct i915_buddy_block *block) | ||||||
| 212 | { | ||||||
| 213 | unsigned int block_order = i915_buddy_block_order(block) - 1; | ||||||
| 214 | u64 offset = i915_buddy_block_offset(block); | ||||||
| 215 | |||||||
| 216 | GEM_BUG_ON(!i915_buddy_block_is_free(block))((void)0); | ||||||
| 217 | GEM_BUG_ON(!i915_buddy_block_order(block))((void)0); | ||||||
| 218 | |||||||
| 219 | block->left = i915_block_alloc(block, block_order, offset); | ||||||
| 220 | if (!block->left) | ||||||
| 221 | return -ENOMEM12; | ||||||
| 222 | |||||||
| 223 | block->right = i915_block_alloc(block, block_order, | ||||||
| 224 | offset + (mm->chunk_size << block_order)); | ||||||
| 225 | if (!block->right) { | ||||||
| 226 | i915_block_free(block->left); | ||||||
| 227 | return -ENOMEM12; | ||||||
| 228 | } | ||||||
| 229 | |||||||
| 230 | mark_free(mm, block->left); | ||||||
| 231 | mark_free(mm, block->right); | ||||||
| 232 | |||||||
| 233 | mark_split(block); | ||||||
| 234 | |||||||
| 235 | return 0; | ||||||
| 236 | } | ||||||
| 237 | |||||||
| 238 | static struct i915_buddy_block * | ||||||
| 239 | get_buddy(struct i915_buddy_block *block) | ||||||
| 240 | { | ||||||
| 241 | struct i915_buddy_block *parent; | ||||||
| 242 | |||||||
| 243 | parent = block->parent; | ||||||
| 244 | if (!parent) | ||||||
| 245 | return NULL((void *)0); | ||||||
| 246 | |||||||
| 247 | if (parent->left == block) | ||||||
| 248 | return parent->right; | ||||||
| 249 | |||||||
| 250 | return parent->left; | ||||||
| 251 | } | ||||||
| 252 | |||||||
| 253 | static void __i915_buddy_free(struct i915_buddy_mm *mm, | ||||||
| 254 | struct i915_buddy_block *block) | ||||||
| 255 | { | ||||||
| 256 | struct i915_buddy_block *parent; | ||||||
| 257 | |||||||
| 258 | while ((parent = block->parent)) { | ||||||
| 259 | struct i915_buddy_block *buddy; | ||||||
| 260 | |||||||
| 261 | buddy = get_buddy(block); | ||||||
| 262 | |||||||
| 263 | if (!i915_buddy_block_is_free(buddy)) | ||||||
| 264 | break; | ||||||
| 265 | |||||||
| 266 | list_del(&buddy->link); | ||||||
| 267 | |||||||
| 268 | i915_block_free(block); | ||||||
| 269 | i915_block_free(buddy); | ||||||
| 270 | |||||||
| 271 | block = parent; | ||||||
| 272 | } | ||||||
| 273 | |||||||
| 274 | mark_free(mm, block); | ||||||
| 275 | } | ||||||
| 276 | |||||||
| 277 | void i915_buddy_free(struct i915_buddy_mm *mm, | ||||||
| 278 | struct i915_buddy_block *block) | ||||||
| 279 | { | ||||||
| 280 | GEM_BUG_ON(!i915_buddy_block_is_allocated(block))((void)0); | ||||||
| 281 | __i915_buddy_free(mm, block); | ||||||
| 282 | } | ||||||
| 283 | |||||||
| 284 | void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects) | ||||||
| 285 | { | ||||||
| 286 | struct i915_buddy_block *block, *on; | ||||||
| 287 | |||||||
| 288 | list_for_each_entry_safe(block, on, objects, link)for (block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = ((objects)->next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );}), on = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = (block->link.next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );}); &block ->link != (objects); block = on, on = ({ const __typeof( ( (__typeof(*on) *)0)->link ) *__mptr = (on->link.next); ( __typeof(*on) *)( (char *)__mptr - __builtin_offsetof(__typeof (*on), link) );})) { | ||||||
| 289 | i915_buddy_free(mm, block); | ||||||
| 290 | cond_resched()do { if (({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self ))); __ci;})->ci_schedstate.spc_schedflags & 0x0002) yield (); } while (0); | ||||||
| 291 | } | ||||||
| 292 | INIT_LIST_HEAD(objects); | ||||||
| 293 | } | ||||||
| 294 | |||||||
| 295 | /* | ||||||
| 296 | * Allocate power-of-two block. The order value here translates to: | ||||||
| 297 | * | ||||||
| 298 | * 0 = 2^0 * mm->chunk_size | ||||||
| 299 | * 1 = 2^1 * mm->chunk_size | ||||||
| 300 | * 2 = 2^2 * mm->chunk_size | ||||||
| 301 | * ... | ||||||
| 302 | */ | ||||||
| 303 | struct i915_buddy_block * | ||||||
| 304 | i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order) | ||||||
| 305 | { | ||||||
| 306 | struct i915_buddy_block *block = NULL((void *)0); | ||||||
| 307 | unsigned int i; | ||||||
| 308 | int err; | ||||||
| 309 | |||||||
| 310 | for (i = order; i <= mm->max_order; ++i) { | ||||||
| 311 | block = list_first_entry_or_null(&mm->free_list[i],(list_empty(&mm->free_list[i]) ? ((void *)0) : ({ const __typeof( ((struct i915_buddy_block *)0)->link ) *__mptr = ((&mm->free_list[i])->next); (struct i915_buddy_block *)( (char *)__mptr - __builtin_offsetof(struct i915_buddy_block , link) );})) | ||||||
| 312 | struct i915_buddy_block,(list_empty(&mm->free_list[i]) ? ((void *)0) : ({ const __typeof( ((struct i915_buddy_block *)0)->link ) *__mptr = ((&mm->free_list[i])->next); (struct i915_buddy_block *)( (char *)__mptr - __builtin_offsetof(struct i915_buddy_block , link) );})) | ||||||
| 313 | link)(list_empty(&mm->free_list[i]) ? ((void *)0) : ({ const __typeof( ((struct i915_buddy_block *)0)->link ) *__mptr = ((&mm->free_list[i])->next); (struct i915_buddy_block *)( (char *)__mptr - __builtin_offsetof(struct i915_buddy_block , link) );})); | ||||||
| 314 | if (block) | ||||||
| 315 | break; | ||||||
| 316 | } | ||||||
| 317 | |||||||
| 318 | if (!block) | ||||||
| 319 | return ERR_PTR(-ENOSPC28); | ||||||
| 320 | |||||||
| 321 | GEM_BUG_ON(!i915_buddy_block_is_free(block))((void)0); | ||||||
| 322 | |||||||
| 323 | while (i != order) { | ||||||
| 324 | err = split_block(mm, block); | ||||||
| 325 | if (unlikely(err)__builtin_expect(!!(err), 0)) | ||||||
| 326 | goto out_free; | ||||||
| 327 | |||||||
| 328 | /* Go low */ | ||||||
| 329 | block = block->left; | ||||||
| 330 | i--; | ||||||
| 331 | } | ||||||
| 332 | |||||||
| 333 | mark_allocated(block); | ||||||
| 334 | kmemleak_update_trace(block); | ||||||
| 335 | return block; | ||||||
| 336 | |||||||
| 337 | out_free: | ||||||
| 338 | if (i != order) | ||||||
| 339 | __i915_buddy_free(mm, block); | ||||||
| 340 | return ERR_PTR(err); | ||||||
| 341 | } | ||||||
| 342 | |||||||
| 343 | static inline bool_Bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) | ||||||
| 344 | { | ||||||
| 345 | return s1 <= e2 && e1 >= s2; | ||||||
| 346 | } | ||||||
| 347 | |||||||
| 348 | static inline bool_Bool contains(u64 s1, u64 e1, u64 s2, u64 e2) | ||||||
| 349 | { | ||||||
| 350 | return s1 <= s2 && e1 >= e2; | ||||||
| 351 | } | ||||||
| 352 | |||||||
| 353 | /* | ||||||
| 354 | * Allocate range. Note that it's safe to chain together multiple alloc_ranges | ||||||
| 355 | * with the same blocks list. | ||||||
| 356 | * | ||||||
| 357 | * Intended for pre-allocating portions of the address space, for example to | ||||||
| 358 | * reserve a block for the initial framebuffer or similar, hence the expectation | ||||||
| 359 | * here is that i915_buddy_alloc() is still the main vehicle for | ||||||
| 360 | * allocations, so if that's not the case then the drm_mm range allocator is | ||||||
| 361 | * probably a much better fit, and so you should probably go use that instead. | ||||||
| 362 | */ | ||||||
| 363 | int i915_buddy_alloc_range(struct i915_buddy_mm *mm, | ||||||
| 364 | struct list_head *blocks, | ||||||
| 365 | u64 start, u64 size) | ||||||
| 366 | { | ||||||
| 367 | struct i915_buddy_block *block; | ||||||
| 368 | struct i915_buddy_block *buddy; | ||||||
| 369 | DRM_LIST_HEAD(allocated)struct list_head allocated = { &(allocated), &(allocated ) }; | ||||||
| 370 | DRM_LIST_HEAD(dfs)struct list_head dfs = { &(dfs), &(dfs) }; | ||||||
| 371 | u64 end; | ||||||
| 372 | int err; | ||||||
| 373 | int i; | ||||||
| 374 | |||||||
| 375 | if (size < mm->chunk_size) | ||||||
| 376 | return -EINVAL22; | ||||||
| 377 | |||||||
| 378 | if (!IS_ALIGNED(size | start, mm->chunk_size)(((size | start) & ((mm->chunk_size) - 1)) == 0)) | ||||||
| 379 | return -EINVAL22; | ||||||
| 380 | |||||||
| 381 | if (range_overflows(start, size, mm->size)({ typeof(start) start__ = (start); typeof(size) size__ = (size ); typeof(mm->size) max__ = (mm->size); (void)(&start__ == &size__); (void)(&start__ == &max__); start__ >= max__ || size__ > max__ - start__; })) | ||||||
| 382 | return -EINVAL22; | ||||||
| 383 | |||||||
| 384 | for (i = 0; i < mm->n_roots; ++i) | ||||||
| 385 | list_add_tail(&mm->roots[i]->tmp_link, &dfs); | ||||||
| 386 | |||||||
| 387 | end = start + size - 1; | ||||||
| 388 | |||||||
| 389 | do { | ||||||
| 390 | u64 block_start; | ||||||
| 391 | u64 block_end; | ||||||
| 392 | |||||||
| 393 | block = list_first_entry_or_null(&dfs,(list_empty(&dfs) ? ((void *)0) : ({ const __typeof( ((struct i915_buddy_block *)0)->tmp_link ) *__mptr = ((&dfs)-> next); (struct i915_buddy_block *)( (char *)__mptr - __builtin_offsetof (struct i915_buddy_block, tmp_link) );})) | ||||||
| 394 | struct i915_buddy_block,(list_empty(&dfs) ? ((void *)0) : ({ const __typeof( ((struct i915_buddy_block *)0)->tmp_link ) *__mptr = ((&dfs)-> next); (struct i915_buddy_block *)( (char *)__mptr - __builtin_offsetof (struct i915_buddy_block, tmp_link) );})) | ||||||
| 395 | tmp_link)(list_empty(&dfs) ? ((void *)0) : ({ const __typeof( ((struct i915_buddy_block *)0)->tmp_link ) *__mptr = ((&dfs)-> next); (struct i915_buddy_block *)( (char *)__mptr - __builtin_offsetof (struct i915_buddy_block, tmp_link) );})); | ||||||
| 396 | if (!block) | ||||||
| 397 | break; | ||||||
| 398 | |||||||
| 399 | list_del(&block->tmp_link); | ||||||
| 400 | |||||||
| 401 | block_start = i915_buddy_block_offset(block); | ||||||
| 402 | block_end = block_start + i915_buddy_block_size(mm, block) - 1; | ||||||
| 403 | |||||||
| 404 | if (!overlaps(start, end, block_start, block_end)) | ||||||
| 405 | continue; | ||||||
| 406 | |||||||
| 407 | if (i915_buddy_block_is_allocated(block)) { | ||||||
| 408 | err = -ENOSPC28; | ||||||
| 409 | goto err_free; | ||||||
| 410 | } | ||||||
| 411 | |||||||
| 412 | if (contains(start, end, block_start, block_end)) { | ||||||
| 413 | if (!i915_buddy_block_is_free(block)) { | ||||||
| 414 | err = -ENOSPC28; | ||||||
| 415 | goto err_free; | ||||||
| 416 | } | ||||||
| 417 | |||||||
| 418 | mark_allocated(block); | ||||||
| 419 | list_add_tail(&block->link, &allocated); | ||||||
| 420 | continue; | ||||||
| 421 | } | ||||||
| 422 | |||||||
| 423 | if (!i915_buddy_block_is_split(block)) { | ||||||
| 424 | err = split_block(mm, block); | ||||||
| 425 | if (unlikely(err)__builtin_expect(!!(err), 0)) | ||||||
| 426 | goto err_undo; | ||||||
| 427 | } | ||||||
| 428 | |||||||
| 429 | list_add(&block->right->tmp_link, &dfs); | ||||||
| 430 | list_add(&block->left->tmp_link, &dfs); | ||||||
| 431 | } while (1); | ||||||
| 432 | |||||||
| 433 | list_splice_tail(&allocated, blocks); | ||||||
| 434 | return 0; | ||||||
| 435 | |||||||
| 436 | err_undo: | ||||||
| 437 | /* | ||||||
| 438 | * We really don't want to leave around a bunch of split blocks, since | ||||||
| 439 | * bigger is better, so make sure we merge everything back before we | ||||||
| 440 | * free the allocated blocks. | ||||||
| 441 | */ | ||||||
| 442 | buddy = get_buddy(block); | ||||||
| 443 | if (buddy && | ||||||
| 444 | (i915_buddy_block_is_free(block) && | ||||||
| 445 | i915_buddy_block_is_free(buddy))) | ||||||
| 446 | __i915_buddy_free(mm, block); | ||||||
| 447 | |||||||
| 448 | err_free: | ||||||
| 449 | i915_buddy_free_list(mm, &allocated); | ||||||
| 450 | return err; | ||||||
| 451 | } | ||||||
| 452 | |||||||
| 453 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)0 | ||||||
| 454 | #include "selftests/i915_buddy.c" | ||||||
| 455 | #endif |
| 1 | /* Public domain. */ |
| 2 | |
| 3 | #ifndef _LINUX_SLAB_H |
| 4 | #define _LINUX_SLAB_H |
| 5 | |
| 6 | #include <sys/types.h> |
| 7 | #include <sys/malloc.h> |
| 8 | |
| 9 | #include <linux/types.h> |
| 10 | #include <linux/workqueue.h> |
| 11 | #include <linux/gfp.h> |
| 12 | |
| 13 | #include <linux/processor.h> /* for CACHELINESIZE */ |
| 14 | |
| 15 | static inline void * |
| 16 | kmalloc(size_t size, int flags) |
| 17 | { |
| 18 | return malloc(size, M_DRM145, flags); |
| 19 | } |
| 20 | |
| 21 | static inline void * |
| 22 | kmalloc_array(size_t n, size_t size, int flags) |
| 23 | { |
| 24 | if (n != 0 && SIZE_MAX0xffffffffffffffffUL / n < size) |
| 25 | return NULL((void *)0); |
| 26 | return malloc(n * size, M_DRM145, flags); |
| 27 | } |
| 28 | |
| 29 | static inline void * |
| 30 | kcalloc(size_t n, size_t size, int flags) |
| 31 | { |
| 32 | if (n != 0 && SIZE_MAX0xffffffffffffffffUL / n < size) |
| 33 | return NULL((void *)0); |
| 34 | return malloc(n * size, M_DRM145, flags | M_ZERO0x0008); |
| 35 | } |
| 36 | |
| 37 | static inline void * |
| 38 | kzalloc(size_t size, int flags) |
| 39 | { |
| 40 | return malloc(size, M_DRM145, flags | M_ZERO0x0008); |
| 41 | } |
| 42 | |
| 43 | static inline void |
| 44 | kfree(const void *objp) |
| 45 | { |
| 46 | free((void *)objp, M_DRM145, 0); |
| 47 | } |
| 48 | |
| 49 | #endif |
| 1 | /* $OpenBSD: list.h,v 1.4 2021/10/01 04:36:38 jsg Exp $ */ | |||
| 2 | /* drm_linux_list.h -- linux list functions for the BSDs. | |||
| 3 | * Created: Mon Apr 7 14:30:16 1999 by anholt@FreeBSD.org | |||
| 4 | */ | |||
| 5 | /*- | |||
| 6 | * Copyright 2003 Eric Anholt | |||
| 7 | * All Rights Reserved. | |||
| 8 | * | |||
| 9 | * Permission is hereby granted, free of charge, to any person obtaining a | |||
| 10 | * copy of this software and associated documentation files (the "Software"), | |||
| 11 | * to deal in the Software without restriction, including without limitation | |||
| 12 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |||
| 13 | * and/or sell copies of the Software, and to permit persons to whom the | |||
| 14 | * Software is furnished to do so, subject to the following conditions: | |||
| 15 | * | |||
| 16 | * The above copyright notice and this permission notice (including the next | |||
| 17 | * paragraph) shall be included in all copies or substantial portions of the | |||
| 18 | * Software. | |||
| 19 | * | |||
| 20 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
| 21 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
| 22 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |||
| 23 | * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR | |||
| 24 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |||
| 25 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |||
| 26 | * OTHER DEALINGS IN THE SOFTWARE. | |||
| 27 | * | |||
| 28 | * Authors: | |||
| 29 | * Eric Anholt <anholt@FreeBSD.org> | |||
| 30 | * | |||
| 31 | */ | |||
| 32 | ||||
| 33 | #ifndef _DRM_LINUX_LIST_H_ | |||
| 34 | #define _DRM_LINUX_LIST_H_ | |||
| 35 | ||||
| 36 | #include <sys/param.h> | |||
| 37 | #include <linux/kernel.h> | |||
| 38 | #include <linux/types.h> | |||
| 39 | #include <linux/poison.h> | |||
| 40 | ||||
| 41 | #define list_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); ( type *)( (char *)__mptr - __builtin_offsetof(type, member) ); }) container_of(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); ( type *)( (char *)__mptr - __builtin_offsetof(type, member) ); }) | |||
| 42 | ||||
| 43 | static inline void | |||
| 44 | INIT_LIST_HEAD(struct list_head *head) { | |||
| 45 | (head)->next = head; | |||
| 46 | (head)->prev = head; | |||
| 47 | } | |||
| 48 | ||||
| 49 | #define LIST_HEAD_INIT(name){ &(name), &(name) } { &(name), &(name) } | |||
| 50 | ||||
| 51 | #define DRM_LIST_HEAD(name)struct list_head name = { &(name), &(name) } \ | |||
| 52 | struct list_head name = LIST_HEAD_INIT(name){ &(name), &(name) } | |||
| 53 | ||||
| 54 | static inline int | |||
| 55 | list_empty(const struct list_head *head) { | |||
| 56 | return (head)->next == head; | |||
| 57 | } | |||
| 58 | ||||
| 59 | static inline int | |||
| 60 | list_is_singular(const struct list_head *head) { | |||
| 61 | return !list_empty(head) && ((head)->next == (head)->prev); | |||
| 62 | } | |||
| 63 | ||||
| 64 | static inline int | |||
| 65 | list_is_first(const struct list_head *list, | |||
| 66 | const struct list_head *head) | |||
| 67 | { | |||
| 68 | return list->prev == head; | |||
| 69 | } | |||
| 70 | ||||
| 71 | static inline int | |||
| 72 | list_is_last(const struct list_head *list, | |||
| 73 | const struct list_head *head) | |||
| 74 | { | |||
| 75 | return list->next == head; | |||
| 76 | } | |||
| 77 | ||||
| 78 | static inline void | |||
| 79 | list_add(struct list_head *new, struct list_head *head) { | |||
| 80 | (head)->next->prev = new; | |||
| ||||
| 81 | (new)->next = (head)->next; | |||
| 82 | (new)->prev = head; | |||
| 83 | (head)->next = new; | |||
| 84 | } | |||
| 85 | ||||
| 86 | static inline void | |||
| 87 | list_add_tail(struct list_head *entry, struct list_head *head) { | |||
| 88 | (entry)->prev = (head)->prev; | |||
| 89 | (entry)->next = head; | |||
| 90 | (head)->prev->next = entry; | |||
| 91 | (head)->prev = entry; | |||
| 92 | } | |||
| 93 | ||||
| 94 | static inline void | |||
| 95 | list_del(struct list_head *entry) { | |||
| 96 | (entry)->next->prev = (entry)->prev; | |||
| 97 | (entry)->prev->next = (entry)->next; | |||
| 98 | } | |||
| 99 | ||||
| 100 | #define __list_del_entry(x)list_del(x) list_del(x) | |||
| 101 | ||||
| 102 | static inline void list_replace(struct list_head *old, | |||
| 103 | struct list_head *new) | |||
| 104 | { | |||
| 105 | new->next = old->next; | |||
| 106 | new->next->prev = new; | |||
| 107 | new->prev = old->prev; | |||
| 108 | new->prev->next = new; | |||
| 109 | } | |||
| 110 | ||||
| 111 | static inline void list_replace_init(struct list_head *old, | |||
| 112 | struct list_head *new) | |||
| 113 | { | |||
| 114 | list_replace(old, new); | |||
| 115 | INIT_LIST_HEAD(old); | |||
| 116 | } | |||
| 117 | ||||
| 118 | static inline void list_move(struct list_head *list, struct list_head *head) | |||
| 119 | { | |||
| 120 | list_del(list); | |||
| 121 | list_add(list, head); | |||
| 122 | } | |||
| 123 | ||||
| 124 | static inline void list_move_tail(struct list_head *list, | |||
| 125 | struct list_head *head) | |||
| 126 | { | |||
| 127 | list_del(list); | |||
| 128 | list_add_tail(list, head); | |||
| 129 | } | |||
| 130 | ||||
| 131 | static inline void | |||
| 132 | list_rotate_to_front(struct list_head *list, struct list_head *head) | |||
| 133 | { | |||
| 134 | list_del(head); | |||
| 135 | list_add_tail(head, list); | |||
| 136 | } | |||
| 137 | ||||
| 138 | static inline void | |||
| 139 | list_bulk_move_tail(struct list_head *head, struct list_head *first, | |||
| 140 | struct list_head *last) | |||
| 141 | { | |||
| 142 | first->prev->next = last->next; | |||
| 143 | last->next->prev = first->prev; | |||
| 144 | head->prev->next = first; | |||
| 145 | first->prev = head->prev; | |||
| 146 | last->next = head; | |||
| 147 | head->prev = last; | |||
| 148 | } | |||
| 149 | ||||
| 150 | static inline void | |||
| 151 | list_del_init(struct list_head *entry) { | |||
| 152 | (entry)->next->prev = (entry)->prev; | |||
| 153 | (entry)->prev->next = (entry)->next; | |||
| 154 | INIT_LIST_HEAD(entry); | |||
| 155 | } | |||
| 156 | ||||
| 157 | #define list_next_entry(pos, member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) \ | |||
| 158 | list_entry(((pos)->member.next), typeof(*(pos)), member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) | |||
| 159 | ||||
| 160 | #define list_prev_entry(pos, member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.prev)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) \ | |||
| 161 | list_entry(((pos)->member.prev), typeof(*(pos)), member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.prev)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) | |||
| 162 | ||||
| 163 | #define list_safe_reset_next(pos, n, member)n = ({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) \ | |||
| 164 | n = list_next_entry(pos, member)({ const __typeof( ((typeof(*(pos)) *)0)->member ) *__mptr = (((pos)->member.next)); (typeof(*(pos)) *)( (char *)__mptr - __builtin_offsetof(typeof(*(pos)), member) );}) | |||
| 165 | ||||
| 166 | #define list_for_each(entry, head)for (entry = (head)->next; entry != head; entry = (entry)-> next) \ | |||
| 167 | for (entry = (head)->next; entry != head; entry = (entry)->next) | |||
| 168 | ||||
| 169 | #define list_for_each_prev(entry, head)for (entry = (head)->prev; entry != (head); entry = entry-> prev) \ | |||
| 170 | for (entry = (head)->prev; entry != (head); \ | |||
| 171 | entry = entry->prev) | |||
| 172 | ||||
| 173 | #define list_for_each_safe(entry, temp, head)for (entry = (head)->next, temp = (entry)->next; entry != head; entry = temp, temp = entry->next) \ | |||
| 174 | for (entry = (head)->next, temp = (entry)->next; \ | |||
| 175 | entry != head; \ | |||
| 176 | entry = temp, temp = entry->next) | |||
| 177 | ||||
| 178 | #define list_for_each_entry_safe_reverse(pos, n, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}), n = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos )->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); &(pos)->member != (head ); pos = n, n = ({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.prev); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof(__typeof(*n), member) );})) \ | |||
| 179 | for (pos = list_entry((head)->prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}), \ | |||
| 180 | n = list_entry((pos)->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos)->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 181 | &(pos)->member != (head); \ | |||
| 182 | pos = n, n = list_entry(n->member.prev, __typeof(*n), member)({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.prev); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof (__typeof(*n), member) );})) | |||
| 183 | ||||
| 184 | #define list_for_each_entry_safe_from(pos, n, head, member)for (n = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char * )__mptr - __builtin_offsetof(__typeof(*pos), member) );}); & pos->member != (head); pos = n, n = ({ const __typeof( ((__typeof (*n) *)0)->member ) *__mptr = (n->member.next); (__typeof (*n) *)( (char *)__mptr - __builtin_offsetof(__typeof(*n), member ) );})) \ | |||
| 185 | for (n = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 186 | &pos->member != (head); \ | |||
| 187 | pos = n, n = list_entry(n->member.next, __typeof(*n), member)({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.next); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof (__typeof(*n), member) );})) | |||
| 188 | ||||
| 189 | #define list_for_each_entry(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); &pos-> member != (head); pos = ({ const __typeof( ((__typeof(*pos) * )0)->member ) *__mptr = (pos->member.next); (__typeof(* pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );})) \ | |||
| 190 | for (pos = list_entry((head)->next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); \ | |||
| 191 | &pos->member != (head); \ | |||
| 192 | pos = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 193 | ||||
| 194 | #define list_for_each_entry_from(pos, head, member)for (; &pos->member != (head); pos = ({ const __typeof ( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member .next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );})) \ | |||
| 195 | for (; \ | |||
| 196 | &pos->member != (head); \ | |||
| 197 | pos = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 198 | ||||
| 199 | #define list_for_each_entry_reverse(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); &pos-> member != (head); pos = ({ const __typeof( ((__typeof(*pos) * )0)->member ) *__mptr = (pos->member.prev); (__typeof(* pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );})) \ | |||
| 200 | for (pos = list_entry((head)->prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); \ | |||
| 201 | &pos->member != (head); \ | |||
| 202 | pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 203 | ||||
| 204 | #define list_for_each_entry_from_reverse(pos, head, member)for (; &pos->member != (head); pos = ({ const __typeof ( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member .prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );})) \ | |||
| 205 | for (; \ | |||
| 206 | &pos->member != (head); \ | |||
| 207 | pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 208 | ||||
| 209 | #define list_for_each_entry_continue(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos)->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); & pos->member != (head); pos = ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );})) \ | |||
| 210 | for (pos = list_entry((pos)->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((pos)->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 211 | &pos->member != (head); \ | |||
| 212 | pos = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 213 | ||||
| 214 | #define list_for_each_entry_continue_reverse(pos, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); & pos->member != (head); pos = ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );})) \ | |||
| 215 | for (pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 216 | &pos->member != (head); \ | |||
| 217 | pos = list_entry(pos->member.prev, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.prev); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );})) | |||
| 218 | ||||
| 219 | /** | |||
| 220 | * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry | |||
| 221 | * @pos: the type * to use as a loop cursor. | |||
| 222 | * @n: another type * to use as temporary storage | |||
| 223 | * @head: the head for your list. | |||
| 224 | * @member: the name of the list_struct within the struct. | |||
| 225 | */ | |||
| 226 | #define list_for_each_entry_safe(pos, n, head, member)for (pos = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}), n = ({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos-> member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}); &pos->member != (head); pos = n, n = ({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.next); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof(__typeof(*n), member) );})) \ | |||
| 227 | for (pos = list_entry((head)->next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = ((head)->next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}), \ | |||
| 228 | n = list_entry(pos->member.next, __typeof(*pos), member)({ const __typeof( ((__typeof(*pos) *)0)->member ) *__mptr = (pos->member.next); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member) );}); \ | |||
| 229 | &pos->member != (head); \ | |||
| 230 | pos = n, n = list_entry(n->member.next, __typeof(*n), member)({ const __typeof( ((__typeof(*n) *)0)->member ) *__mptr = (n->member.next); (__typeof(*n) *)( (char *)__mptr - __builtin_offsetof (__typeof(*n), member) );})) | |||
| 231 | ||||
| 232 | #define list_first_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> next); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) \ | |||
| 233 | list_entry((ptr)->next, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> next); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) | |||
| 234 | ||||
| 235 | #define list_first_entry_or_null(ptr, type, member)(list_empty(ptr) ? ((void *)0) : ({ const __typeof( ((type *) 0)->member ) *__mptr = ((ptr)->next); (type *)( (char * )__mptr - __builtin_offsetof(type, member) );})) \ | |||
| 236 | (list_empty(ptr) ? NULL((void *)0) : list_first_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> next); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );})) | |||
| 237 | ||||
| 238 | #define list_last_entry(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> prev); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) \ | |||
| 239 | list_entry((ptr)->prev, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = ((ptr)-> prev); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) | |||
| 240 | ||||
| 241 | static inline void | |||
| 242 | __list_splice(const struct list_head *list, struct list_head *prev, | |||
| 243 | struct list_head *next) | |||
| 244 | { | |||
| 245 | struct list_head *first = list->next; | |||
| 246 | struct list_head *last = list->prev; | |||
| 247 | ||||
| 248 | first->prev = prev; | |||
| 249 | prev->next = first; | |||
| 250 | ||||
| 251 | last->next = next; | |||
| 252 | next->prev = last; | |||
| 253 | } | |||
| 254 | ||||
| 255 | static inline void | |||
| 256 | list_splice(const struct list_head *list, struct list_head *head) | |||
| 257 | { | |||
| 258 | if (list_empty(list)) | |||
| 259 | return; | |||
| 260 | ||||
| 261 | __list_splice(list, head, head->next); | |||
| 262 | } | |||
| 263 | ||||
| 264 | static inline void | |||
| 265 | list_splice_init(struct list_head *list, struct list_head *head) | |||
| 266 | { | |||
| 267 | if (list_empty(list)) | |||
| 268 | return; | |||
| 269 | ||||
| 270 | __list_splice(list, head, head->next); | |||
| 271 | INIT_LIST_HEAD(list); | |||
| 272 | } | |||
| 273 | ||||
| 274 | static inline void | |||
| 275 | list_splice_tail(const struct list_head *list, struct list_head *head) | |||
| 276 | { | |||
| 277 | if (list_empty(list)) | |||
| 278 | return; | |||
| 279 | ||||
| 280 | __list_splice(list, head->prev, head); | |||
| 281 | } | |||
| 282 | ||||
| 283 | static inline void | |||
| 284 | list_splice_tail_init(struct list_head *list, struct list_head *head) | |||
| 285 | { | |||
| 286 | if (list_empty(list)) | |||
| 287 | return; | |||
| 288 | ||||
| 289 | __list_splice(list, head->prev, head); | |||
| 290 | INIT_LIST_HEAD(list); | |||
| 291 | } | |||
| 292 | ||||
| 293 | void list_sort(void *, struct list_head *, | |||
| 294 | int (*)(void *, const struct list_head *, const struct list_head *)); | |||
| 295 | ||||
| 296 | #define hlist_entry(ptr, type, member)((ptr) ? ({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); (type *)( (char *)__mptr - __builtin_offsetof(type, member ) );}) : ((void *)0)) \ | |||
| 297 | ((ptr) ? container_of(ptr, type, member)({ const __typeof( ((type *)0)->member ) *__mptr = (ptr); ( type *)( (char *)__mptr - __builtin_offsetof(type, member) ); }) : NULL((void *)0)) | |||
| 298 | ||||
| 299 | static inline void | |||
| 300 | INIT_HLIST_HEAD(struct hlist_head *head) { | |||
| 301 | head->first = NULL((void *)0); | |||
| 302 | } | |||
| 303 | ||||
| 304 | static inline int | |||
| 305 | hlist_empty(const struct hlist_head *head) { | |||
| 306 | return head->first == NULL((void *)0); | |||
| 307 | } | |||
| 308 | ||||
| 309 | static inline void | |||
| 310 | hlist_add_head(struct hlist_node *new, struct hlist_head *head) | |||
| 311 | { | |||
| 312 | if ((new->next = head->first) != NULL((void *)0)) | |||
| 313 | head->first->prev = &new->next; | |||
| 314 | head->first = new; | |||
| 315 | new->prev = &head->first; | |||
| 316 | } | |||
| 317 | ||||
| 318 | static inline void | |||
| 319 | hlist_del_init(struct hlist_node *node) | |||
| 320 | { | |||
| 321 | if (node->next != NULL((void *)0)) | |||
| 322 | node->next->prev = node->prev; | |||
| 323 | *(node->prev) = node->next; | |||
| 324 | node->next = NULL((void *)0); | |||
| 325 | node->prev = NULL((void *)0); | |||
| 326 | } | |||
| 327 | ||||
| 328 | #define hlist_for_each(pos, head)for (pos = (head)->first; pos != ((void *)0); pos = pos-> next) \ | |||
| 329 | for (pos = (head)->first; pos != NULL((void *)0); pos = pos->next) | |||
| 330 | ||||
| 331 | #define hlist_for_each_entry(pos, head, member)for (pos = (((head)->first) ? ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = ((head)->first); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );}) : ((void *)0)); pos != ((void *)0); pos = (((pos )->member.next) ? ({ const __typeof( ((__typeof(*pos) *)0) ->member ) *__mptr = ((pos)->member.next); (__typeof(*pos ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0))) \ | |||
| 332 | for (pos = hlist_entry((head)->first, __typeof(*pos), member)(((head)->first) ? ({ const __typeof( ((__typeof(*pos) *)0 )->member ) *__mptr = ((head)->first); (__typeof(*pos) * )( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0)); \ | |||
| 333 | pos != NULL((void *)0); \ | |||
| 334 | pos = hlist_entry((pos)->member.next, __typeof(*pos), member)(((pos)->member.next) ? ({ const __typeof( ((__typeof(*pos ) *)0)->member ) *__mptr = ((pos)->member.next); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );}) : ((void *)0))) | |||
| 335 | ||||
| 336 | #define hlist_for_each_entry_safe(pos, n, head, member)for (pos = (((head)->first) ? ({ const __typeof( ((__typeof (*pos) *)0)->member ) *__mptr = ((head)->first); (__typeof (*pos) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos) , member) );}) : ((void *)0)); pos != ((void *)0) && ( n = pos->member.next, 1); pos = ((n) ? ({ const __typeof( ( (__typeof(*pos) *)0)->member ) *__mptr = (n); (__typeof(*pos ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0))) \ | |||
| 337 | for (pos = hlist_entry((head)->first, __typeof(*pos), member)(((head)->first) ? ({ const __typeof( ((__typeof(*pos) *)0 )->member ) *__mptr = ((head)->first); (__typeof(*pos) * )( (char *)__mptr - __builtin_offsetof(__typeof(*pos), member ) );}) : ((void *)0)); \ | |||
| 338 | pos != NULL((void *)0) && (n = pos->member.next, 1); \ | |||
| 339 | pos = hlist_entry(n, __typeof(*pos), member)((n) ? ({ const __typeof( ((__typeof(*pos) *)0)->member ) * __mptr = (n); (__typeof(*pos) *)( (char *)__mptr - __builtin_offsetof (__typeof(*pos), member) );}) : ((void *)0))) | |||
| 340 | ||||
| 341 | #endif /* _DRM_LINUX_LIST_H_ */ |