File: | dev/pci/drm/ttm/ttm_memory.c |
Warning: | line 448, column 3 Value stored to 'zone' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
2 | /************************************************************************** |
3 | * |
4 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
5 | * All Rights Reserved. |
6 | * |
7 | * Permission is hereby granted, free of charge, to any person obtaining a |
8 | * copy of this software and associated documentation files (the |
9 | * "Software"), to deal in the Software without restriction, including |
10 | * without limitation the rights to use, copy, modify, merge, publish, |
11 | * distribute, sub license, and/or sell copies of the Software, and to |
12 | * permit persons to whom the Software is furnished to do so, subject to |
13 | * the following conditions: |
14 | * |
15 | * The above copyright notice and this permission notice (including the |
16 | * next paragraph) shall be included in all copies or substantial portions |
17 | * of the Software. |
18 | * |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
26 | * |
27 | **************************************************************************/ |
28 | |
29 | #define pr_fmt(fmt)"[TTM] " fmt "[TTM] " fmt |
30 | |
31 | #include <drm/ttm/ttm_memory.h> |
32 | #include <drm/ttm/ttm_module.h> |
33 | #include <drm/ttm/ttm_page_alloc.h> |
34 | #include <linux/spinlock.h> |
35 | #include <linux/sched.h> |
36 | #include <linux/wait.h> |
37 | #include <linux/mm.h> |
38 | #include <linux/module.h> |
39 | #include <linux/slab.h> |
40 | #include <linux/swap.h> |
41 | |
42 | #define TTM_MEMORY_ALLOC_RETRIES4 4 |
43 | |
44 | struct ttm_mem_global ttm_mem_glob; |
45 | EXPORT_SYMBOL(ttm_mem_glob); |
46 | |
47 | struct ttm_mem_zone { |
48 | struct kobject kobj; |
49 | struct ttm_mem_global *glob; |
50 | const char *name; |
51 | uint64_t zone_mem; |
52 | uint64_t emer_mem; |
53 | uint64_t max_mem; |
54 | uint64_t swap_limit; |
55 | uint64_t used_mem; |
56 | }; |
57 | |
58 | #ifdef notyet |
59 | static struct attribute ttm_mem_sys = { |
60 | .name = "zone_memory", |
61 | .mode = S_IRUGO |
62 | }; |
63 | static struct attribute ttm_mem_emer = { |
64 | .name = "emergency_memory", |
65 | .mode = S_IRUGO | S_IWUSR |
66 | }; |
67 | static struct attribute ttm_mem_max = { |
68 | .name = "available_memory", |
69 | .mode = S_IRUGO | S_IWUSR |
70 | }; |
71 | static struct attribute ttm_mem_swap = { |
72 | .name = "swap_limit", |
73 | .mode = S_IRUGO | S_IWUSR |
74 | }; |
75 | static struct attribute ttm_mem_used = { |
76 | .name = "used_memory", |
77 | .mode = S_IRUGO |
78 | }; |
79 | #endif |
80 | |
81 | static void ttm_mem_zone_kobj_release(struct kobject *kobj) |
82 | { |
83 | struct ttm_mem_zone *zone = |
84 | container_of(kobj, struct ttm_mem_zone, kobj)({ const __typeof( ((struct ttm_mem_zone *)0)->kobj ) *__mptr = (kobj); (struct ttm_mem_zone *)( (char *)__mptr - __builtin_offsetof (struct ttm_mem_zone, kobj) );}); |
85 | |
86 | pr_info("Zone %7s: Used memory at exit: %llu KiB\n",do { } while(0) |
87 | zone->name, (unsigned long long)zone->used_mem >> 10)do { } while(0); |
88 | kfree(zone); |
89 | } |
90 | |
91 | #ifdef notyet |
92 | static ssize_t ttm_mem_zone_show(struct kobject *kobj, |
93 | struct attribute *attr, |
94 | char *buffer) |
95 | { |
96 | struct ttm_mem_zone *zone = |
97 | container_of(kobj, struct ttm_mem_zone, kobj)({ const __typeof( ((struct ttm_mem_zone *)0)->kobj ) *__mptr = (kobj); (struct ttm_mem_zone *)( (char *)__mptr - __builtin_offsetof (struct ttm_mem_zone, kobj) );}); |
98 | uint64_t val = 0; |
99 | |
100 | spin_lock(&zone->glob->lock)mtx_enter(&zone->glob->lock); |
101 | if (attr == &ttm_mem_sys) |
102 | val = zone->zone_mem; |
103 | else if (attr == &ttm_mem_emer) |
104 | val = zone->emer_mem; |
105 | else if (attr == &ttm_mem_max) |
106 | val = zone->max_mem; |
107 | else if (attr == &ttm_mem_swap) |
108 | val = zone->swap_limit; |
109 | else if (attr == &ttm_mem_used) |
110 | val = zone->used_mem; |
111 | spin_unlock(&zone->glob->lock)mtx_leave(&zone->glob->lock); |
112 | |
113 | return snprintf(buffer, PAGE_SIZE(1 << 12), "%llu\n", |
114 | (unsigned long long) val >> 10); |
115 | } |
116 | |
117 | static void ttm_check_swapping(struct ttm_mem_global *glob); |
118 | |
119 | static ssize_t ttm_mem_zone_store(struct kobject *kobj, |
120 | struct attribute *attr, |
121 | const char *buffer, |
122 | size_t size) |
123 | { |
124 | struct ttm_mem_zone *zone = |
125 | container_of(kobj, struct ttm_mem_zone, kobj)({ const __typeof( ((struct ttm_mem_zone *)0)->kobj ) *__mptr = (kobj); (struct ttm_mem_zone *)( (char *)__mptr - __builtin_offsetof (struct ttm_mem_zone, kobj) );}); |
126 | int chars; |
127 | unsigned long val; |
128 | uint64_t val64; |
129 | |
130 | chars = sscanf(buffer, "%lu", &val); |
131 | if (chars == 0) |
132 | return size; |
133 | |
134 | val64 = val; |
135 | val64 <<= 10; |
136 | |
137 | spin_lock(&zone->glob->lock)mtx_enter(&zone->glob->lock); |
138 | if (val64 > zone->zone_mem) |
139 | val64 = zone->zone_mem; |
140 | if (attr == &ttm_mem_emer) { |
141 | zone->emer_mem = val64; |
142 | if (zone->max_mem > val64) |
143 | zone->max_mem = val64; |
144 | } else if (attr == &ttm_mem_max) { |
145 | zone->max_mem = val64; |
146 | if (zone->emer_mem < val64) |
147 | zone->emer_mem = val64; |
148 | } else if (attr == &ttm_mem_swap) |
149 | zone->swap_limit = val64; |
150 | spin_unlock(&zone->glob->lock)mtx_leave(&zone->glob->lock); |
151 | |
152 | ttm_check_swapping(zone->glob); |
153 | |
154 | return size; |
155 | } |
156 | |
157 | static struct attribute *ttm_mem_zone_attrs[] = { |
158 | &ttm_mem_sys, |
159 | &ttm_mem_emer, |
160 | &ttm_mem_max, |
161 | &ttm_mem_swap, |
162 | &ttm_mem_used, |
163 | NULL((void *)0) |
164 | }; |
165 | |
166 | static const struct sysfs_ops ttm_mem_zone_ops = { |
167 | .show = &ttm_mem_zone_show, |
168 | .store = &ttm_mem_zone_store |
169 | }; |
170 | #endif |
171 | |
172 | static struct kobj_type ttm_mem_zone_kobj_type = { |
173 | .release = &ttm_mem_zone_kobj_release, |
174 | #ifdef __linux__ |
175 | .sysfs_ops = &ttm_mem_zone_ops, |
176 | .default_attrs = ttm_mem_zone_attrs, |
177 | #endif |
178 | }; |
179 | |
180 | #ifdef notyet |
181 | static struct attribute ttm_mem_global_lower_mem_limit = { |
182 | .name = "lower_mem_limit", |
183 | .mode = S_IRUGO | S_IWUSR |
184 | }; |
185 | |
186 | static ssize_t ttm_mem_global_show(struct kobject *kobj, |
187 | struct attribute *attr, |
188 | char *buffer) |
189 | { |
190 | struct ttm_mem_global *glob = |
191 | container_of(kobj, struct ttm_mem_global, kobj)({ const __typeof( ((struct ttm_mem_global *)0)->kobj ) *__mptr = (kobj); (struct ttm_mem_global *)( (char *)__mptr - __builtin_offsetof (struct ttm_mem_global, kobj) );}); |
192 | uint64_t val = 0; |
193 | |
194 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
195 | val = glob->lower_mem_limit; |
196 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
197 | /* convert from number of pages to KB */ |
198 | val <<= (PAGE_SHIFT12 - 10); |
199 | return snprintf(buffer, PAGE_SIZE(1 << 12), "%llu\n", |
200 | (unsigned long long) val); |
201 | } |
202 | |
203 | static ssize_t ttm_mem_global_store(struct kobject *kobj, |
204 | struct attribute *attr, |
205 | const char *buffer, |
206 | size_t size) |
207 | { |
208 | int chars; |
209 | uint64_t val64; |
210 | unsigned long val; |
211 | struct ttm_mem_global *glob = |
212 | container_of(kobj, struct ttm_mem_global, kobj)({ const __typeof( ((struct ttm_mem_global *)0)->kobj ) *__mptr = (kobj); (struct ttm_mem_global *)( (char *)__mptr - __builtin_offsetof (struct ttm_mem_global, kobj) );}); |
213 | |
214 | chars = sscanf(buffer, "%lu", &val); |
215 | if (chars == 0) |
216 | return size; |
217 | |
218 | val64 = val; |
219 | /* convert from KB to number of pages */ |
220 | val64 >>= (PAGE_SHIFT12 - 10); |
221 | |
222 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
223 | glob->lower_mem_limit = val64; |
224 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
225 | |
226 | return size; |
227 | } |
228 | |
229 | static struct attribute *ttm_mem_global_attrs[] = { |
230 | &ttm_mem_global_lower_mem_limit, |
231 | NULL((void *)0) |
232 | }; |
233 | |
234 | static const struct sysfs_ops ttm_mem_global_ops = { |
235 | .show = &ttm_mem_global_show, |
236 | .store = &ttm_mem_global_store, |
237 | }; |
238 | #endif /* notyet */ |
239 | |
240 | static struct kobj_type ttm_mem_glob_kobj_type = { |
241 | #ifdef __linux__ |
242 | .sysfs_ops = &ttm_mem_global_ops, |
243 | .default_attrs = ttm_mem_global_attrs, |
244 | #endif |
245 | }; |
246 | |
247 | static bool_Bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, |
248 | bool_Bool from_wq, uint64_t extra) |
249 | { |
250 | unsigned int i; |
251 | struct ttm_mem_zone *zone; |
252 | uint64_t target; |
253 | |
254 | for (i = 0; i < glob->num_zones; ++i) { |
255 | zone = glob->zones[i]; |
256 | |
257 | if (from_wq) |
258 | target = zone->swap_limit; |
259 | else if (capable(CAP_SYS_ADMIN0x1)) |
260 | target = zone->emer_mem; |
261 | else |
262 | target = zone->max_mem; |
263 | |
264 | target = (extra > target) ? 0ULL : target; |
265 | |
266 | if (zone->used_mem > target) |
267 | return true1; |
268 | } |
269 | return false0; |
270 | } |
271 | |
272 | /* |
273 | * At this point we only support a single shrink callback. |
274 | * Extend this if needed, perhaps using a linked list of callbacks. |
275 | * Note that this function is reentrant: |
276 | * many threads may try to swap out at any given time. |
277 | */ |
278 | |
279 | static void ttm_shrink(struct ttm_mem_global *glob, bool_Bool from_wq, |
280 | uint64_t extra, struct ttm_operation_ctx *ctx) |
281 | { |
282 | int ret; |
283 | |
284 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
285 | |
286 | while (ttm_zones_above_swap_target(glob, from_wq, extra)) { |
287 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
288 | ret = ttm_bo_swapout(&ttm_bo_glob, ctx); |
289 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
290 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) |
291 | break; |
292 | } |
293 | |
294 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
295 | } |
296 | |
297 | static void ttm_shrink_work(struct work_struct *work) |
298 | { |
299 | struct ttm_operation_ctx ctx = { |
300 | .interruptible = false0, |
301 | .no_wait_gpu = false0 |
302 | }; |
303 | struct ttm_mem_global *glob = |
304 | container_of(work, struct ttm_mem_global, work)({ const __typeof( ((struct ttm_mem_global *)0)->work ) *__mptr = (work); (struct ttm_mem_global *)( (char *)__mptr - __builtin_offsetof (struct ttm_mem_global, work) );}); |
305 | |
306 | ttm_shrink(glob, true1, 0ULL, &ctx); |
307 | } |
308 | |
309 | static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, |
310 | uint64_t mem) |
311 | { |
312 | struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL(0x0001 | 0x0004)); |
313 | int ret; |
314 | |
315 | if (unlikely(!zone)__builtin_expect(!!(!zone), 0)) |
316 | return -ENOMEM12; |
317 | |
318 | zone->name = "kernel"; |
319 | zone->zone_mem = mem; |
320 | zone->max_mem = mem >> 1; |
321 | zone->emer_mem = (mem >> 1) + (mem >> 2); |
322 | zone->swap_limit = zone->max_mem - (mem >> 3); |
323 | zone->used_mem = 0; |
324 | zone->glob = glob; |
325 | glob->zone_kernel = zone; |
326 | ret = kobject_init_and_add( |
327 | &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
328 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) { |
329 | kobject_put(&zone->kobj); |
330 | return ret; |
331 | } |
332 | glob->zones[glob->num_zones++] = zone; |
333 | return 0; |
334 | } |
335 | |
336 | #ifdef CONFIG_HIGHMEM |
337 | static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, |
338 | uint64_t mem) |
339 | { |
340 | struct ttm_mem_zone *zone; |
341 | int ret; |
342 | |
343 | if (si->totalhigh == 0) |
344 | return 0; |
345 | |
346 | zone = kzalloc(sizeof(*zone), GFP_KERNEL(0x0001 | 0x0004)); |
347 | if (unlikely(!zone)__builtin_expect(!!(!zone), 0)) |
348 | return -ENOMEM12; |
349 | |
350 | zone->name = "highmem"; |
351 | zone->zone_mem = mem; |
352 | zone->max_mem = mem >> 1; |
353 | zone->emer_mem = (mem >> 1) + (mem >> 2); |
354 | zone->swap_limit = zone->max_mem - (mem >> 3); |
355 | zone->used_mem = 0; |
356 | zone->glob = glob; |
357 | glob->zone_highmem = zone; |
358 | ret = kobject_init_and_add( |
359 | &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", |
360 | zone->name); |
361 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) { |
362 | kobject_put(&zone->kobj); |
363 | return ret; |
364 | } |
365 | glob->zones[glob->num_zones++] = zone; |
366 | return 0; |
367 | } |
368 | #else |
369 | static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, |
370 | uint64_t mem) |
371 | { |
372 | struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL(0x0001 | 0x0004)); |
373 | int ret; |
374 | |
375 | if (unlikely(!zone)__builtin_expect(!!(!zone), 0)) |
376 | return -ENOMEM12; |
377 | |
378 | /** |
379 | * No special dma32 zone needed. |
380 | */ |
381 | |
382 | if (mem <= ((uint64_t) 1ULL << 32)) { |
383 | kfree(zone); |
384 | return 0; |
385 | } |
386 | |
387 | /* |
388 | * Limit max dma32 memory to 4GB for now |
389 | * until we can figure out how big this |
390 | * zone really is. |
391 | */ |
392 | |
393 | mem = ((uint64_t) 1ULL << 32); |
394 | zone->name = "dma32"; |
395 | zone->zone_mem = mem; |
396 | zone->max_mem = mem >> 1; |
397 | zone->emer_mem = (mem >> 1) + (mem >> 2); |
398 | zone->swap_limit = zone->max_mem - (mem >> 3); |
399 | zone->used_mem = 0; |
400 | zone->glob = glob; |
401 | glob->zone_dma32 = zone; |
402 | ret = kobject_init_and_add( |
403 | &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
404 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) { |
405 | kobject_put(&zone->kobj); |
406 | return ret; |
407 | } |
408 | glob->zones[glob->num_zones++] = zone; |
409 | return 0; |
410 | } |
411 | #endif |
412 | |
413 | int ttm_mem_global_init(struct ttm_mem_global *glob) |
414 | { |
415 | uint64_t mem; |
416 | int ret; |
417 | int i; |
418 | struct ttm_mem_zone *zone; |
419 | |
420 | mtx_init(&glob->lock, IPL_NONE)do { (void)(((void *)0)); (void)(0); __mtx_init((&glob-> lock), ((((0x0)) > 0x0 && ((0x0)) < 0x9) ? 0x9 : ((0x0)))); } while (0); |
421 | glob->swap_queue = create_singlethread_workqueue("ttm_swap"); |
422 | INIT_WORK(&glob->work, ttm_shrink_work); |
423 | ret = kobject_init_and_add( |
424 | &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); |
425 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) { |
426 | kobject_put(&glob->kobj); |
427 | return ret; |
428 | } |
429 | |
430 | mem = ptoa(physmem)((paddr_t)(physmem) << 12); |
431 | |
432 | /* set it as 0 by default to keep original behavior of OOM */ |
433 | glob->lower_mem_limit = 0; |
434 | |
435 | ret = ttm_mem_init_kernel_zone(glob, mem); |
436 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) |
437 | goto out_no_zone; |
438 | #ifdef CONFIG_HIGHMEM |
439 | ret = ttm_mem_init_highmem_zone(glob, mem); |
440 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) |
441 | goto out_no_zone; |
442 | #else |
443 | ret = ttm_mem_init_dma32_zone(glob, mem); |
444 | if (unlikely(ret != 0)__builtin_expect(!!(ret != 0), 0)) |
445 | goto out_no_zone; |
446 | #endif |
447 | for (i = 0; i < glob->num_zones; ++i) { |
448 | zone = glob->zones[i]; |
Value stored to 'zone' is never read | |
449 | pr_info("Zone %7s: Available graphics memory: %llu KiB\n",do { } while(0) |
450 | zone->name, (unsigned long long)zone->max_mem >> 10)do { } while(0); |
451 | } |
452 | ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE(1 << 12))); |
453 | ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE(1 << 12))); |
454 | return 0; |
455 | out_no_zone: |
456 | ttm_mem_global_release(glob); |
457 | return ret; |
458 | } |
459 | |
460 | void ttm_mem_global_release(struct ttm_mem_global *glob) |
461 | { |
462 | struct ttm_mem_zone *zone; |
463 | unsigned int i; |
464 | |
465 | /* let the page allocator first stop the shrink work. */ |
466 | ttm_page_alloc_fini(); |
467 | ttm_dma_page_alloc_fini(); |
468 | |
469 | flush_workqueue(glob->swap_queue); |
470 | destroy_workqueue(glob->swap_queue); |
471 | glob->swap_queue = NULL((void *)0); |
472 | for (i = 0; i < glob->num_zones; ++i) { |
473 | zone = glob->zones[i]; |
474 | kobject_del(&zone->kobj); |
475 | kobject_put(&zone->kobj); |
476 | } |
477 | kobject_del(&glob->kobj); |
478 | kobject_put(&glob->kobj); |
479 | memset(glob, 0, sizeof(*glob))__builtin_memset((glob), (0), (sizeof(*glob))); |
480 | } |
481 | |
482 | static void ttm_check_swapping(struct ttm_mem_global *glob) |
483 | { |
484 | bool_Bool needs_swapping = false0; |
485 | unsigned int i; |
486 | struct ttm_mem_zone *zone; |
487 | |
488 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
489 | for (i = 0; i < glob->num_zones; ++i) { |
490 | zone = glob->zones[i]; |
491 | if (zone->used_mem > zone->swap_limit) { |
492 | needs_swapping = true1; |
493 | break; |
494 | } |
495 | } |
496 | |
497 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
498 | |
499 | if (unlikely(needs_swapping)__builtin_expect(!!(needs_swapping), 0)) |
500 | (void)queue_work(glob->swap_queue, &glob->work); |
501 | |
502 | } |
503 | |
504 | static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, |
505 | struct ttm_mem_zone *single_zone, |
506 | uint64_t amount) |
507 | { |
508 | unsigned int i; |
509 | struct ttm_mem_zone *zone; |
510 | |
511 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
512 | for (i = 0; i < glob->num_zones; ++i) { |
513 | zone = glob->zones[i]; |
514 | if (single_zone && zone != single_zone) |
515 | continue; |
516 | zone->used_mem -= amount; |
517 | } |
518 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
519 | } |
520 | |
521 | void ttm_mem_global_free(struct ttm_mem_global *glob, |
522 | uint64_t amount) |
523 | { |
524 | return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount); |
525 | } |
526 | EXPORT_SYMBOL(ttm_mem_global_free); |
527 | |
528 | /* |
529 | * check if the available mem is under lower memory limit |
530 | * |
531 | * a. if no swap disk at all or free swap space is under swap_mem_limit |
532 | * but available system mem is bigger than sys_mem_limit, allow TTM |
533 | * allocation; |
534 | * |
535 | * b. if the available system mem is less than sys_mem_limit but free |
536 | * swap disk is bigger than swap_mem_limit, allow TTM allocation. |
537 | */ |
538 | bool_Bool |
539 | ttm_check_under_lowerlimit(struct ttm_mem_global *glob, |
540 | uint64_t num_pages, |
541 | struct ttm_operation_ctx *ctx) |
542 | { |
543 | int64_t available; |
544 | |
545 | if (ctx->flags & TTM_OPT_FLAG_FORCE_ALLOC0x2) |
546 | return false0; |
547 | |
548 | available = get_nr_swap_pages() + si_mem_available(); |
549 | available -= num_pages; |
550 | if (available < glob->lower_mem_limit) |
551 | return true1; |
552 | |
553 | return false0; |
554 | } |
555 | |
556 | static int ttm_mem_global_reserve(struct ttm_mem_global *glob, |
557 | struct ttm_mem_zone *single_zone, |
558 | uint64_t amount, bool_Bool reserve) |
559 | { |
560 | uint64_t limit; |
561 | int ret = -ENOMEM12; |
562 | unsigned int i; |
563 | struct ttm_mem_zone *zone; |
564 | |
565 | spin_lock(&glob->lock)mtx_enter(&glob->lock); |
566 | for (i = 0; i < glob->num_zones; ++i) { |
567 | zone = glob->zones[i]; |
568 | if (single_zone && zone != single_zone) |
569 | continue; |
570 | |
571 | limit = (capable(CAP_SYS_ADMIN0x1)) ? |
572 | zone->emer_mem : zone->max_mem; |
573 | |
574 | if (zone->used_mem > limit) |
575 | goto out_unlock; |
576 | } |
577 | |
578 | if (reserve) { |
579 | for (i = 0; i < glob->num_zones; ++i) { |
580 | zone = glob->zones[i]; |
581 | if (single_zone && zone != single_zone) |
582 | continue; |
583 | zone->used_mem += amount; |
584 | } |
585 | } |
586 | |
587 | ret = 0; |
588 | out_unlock: |
589 | spin_unlock(&glob->lock)mtx_leave(&glob->lock); |
590 | ttm_check_swapping(glob); |
591 | |
592 | return ret; |
593 | } |
594 | |
595 | |
596 | static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, |
597 | struct ttm_mem_zone *single_zone, |
598 | uint64_t memory, |
599 | struct ttm_operation_ctx *ctx) |
600 | { |
601 | int count = TTM_MEMORY_ALLOC_RETRIES4; |
602 | |
603 | while (unlikely(ttm_mem_global_reserve(glob,__builtin_expect(!!(ttm_mem_global_reserve(glob, single_zone, memory, 1) != 0), 0) |
604 | single_zone,__builtin_expect(!!(ttm_mem_global_reserve(glob, single_zone, memory, 1) != 0), 0) |
605 | memory, true)__builtin_expect(!!(ttm_mem_global_reserve(glob, single_zone, memory, 1) != 0), 0) |
606 | != 0)__builtin_expect(!!(ttm_mem_global_reserve(glob, single_zone, memory, 1) != 0), 0)) { |
607 | if (ctx->no_wait_gpu) |
608 | return -ENOMEM12; |
609 | if (unlikely(count-- == 0)__builtin_expect(!!(count-- == 0), 0)) |
610 | return -ENOMEM12; |
611 | ttm_shrink(glob, false0, memory + (memory >> 2) + 16, ctx); |
612 | } |
613 | |
614 | return 0; |
615 | } |
616 | |
617 | int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, |
618 | struct ttm_operation_ctx *ctx) |
619 | { |
620 | /** |
621 | * Normal allocations of kernel memory are registered in |
622 | * the kernel zone. |
623 | */ |
624 | |
625 | return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx); |
626 | } |
627 | EXPORT_SYMBOL(ttm_mem_global_alloc); |
628 | |
629 | int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, |
630 | struct vm_page *page, uint64_t size, |
631 | struct ttm_operation_ctx *ctx) |
632 | { |
633 | struct ttm_mem_zone *zone = NULL((void *)0); |
634 | |
635 | /** |
636 | * Page allocations may be registed in a single zone |
637 | * only if highmem or !dma32. |
638 | */ |
639 | |
640 | #ifdef CONFIG_HIGHMEM |
641 | if (PageHighMem(page)0 && glob->zone_highmem != NULL((void *)0)) |
642 | zone = glob->zone_highmem; |
643 | #else |
644 | if (glob->zone_dma32 && page_to_pfn(page)(((page)->phys_addr) / (1 << 12)) > 0x00100000UL) |
645 | zone = glob->zone_kernel; |
646 | #endif |
647 | return ttm_mem_global_alloc_zone(glob, zone, size, ctx); |
648 | } |
649 | |
650 | void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct vm_page *page, |
651 | uint64_t size) |
652 | { |
653 | struct ttm_mem_zone *zone = NULL((void *)0); |
654 | |
655 | #ifdef CONFIG_HIGHMEM |
656 | if (PageHighMem(page)0 && glob->zone_highmem != NULL((void *)0)) |
657 | zone = glob->zone_highmem; |
658 | #else |
659 | if (glob->zone_dma32 && page_to_pfn(page)(((page)->phys_addr) / (1 << 12)) > 0x00100000UL) |
660 | zone = glob->zone_kernel; |
661 | #endif |
662 | ttm_mem_global_free_zone(glob, zone, size); |
663 | } |
664 | |
665 | size_t ttm_round_pot(size_t size) |
666 | { |
667 | if ((size & (size - 1)) == 0) |
668 | return size; |
669 | else if (size > PAGE_SIZE(1 << 12)) |
670 | return PAGE_ALIGN(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1 )); |
671 | else { |
672 | size_t tmp_size = 4; |
673 | |
674 | while (tmp_size < size) |
675 | tmp_size <<= 1; |
676 | |
677 | return tmp_size; |
678 | } |
679 | return 0; |
680 | } |
681 | EXPORT_SYMBOL(ttm_round_pot); |