File: | dev/pci/drm/amd/amdgpu/amdgpu_vram_mgr.c |
Warning: | line 494, column 15 The left operand of '!=' is a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* | |||
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |||
3 | * | |||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |||
5 | * copy of this software and associated documentation files (the "Software"), | |||
6 | * to deal in the Software without restriction, including without limitation | |||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |||
8 | * and/or sell copies of the Software, and to permit persons to whom the | |||
9 | * Software is furnished to do so, subject to the following conditions: | |||
10 | * | |||
11 | * The above copyright notice and this permission notice shall be included in | |||
12 | * all copies or substantial portions of the Software. | |||
13 | * | |||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |||
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |||
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |||
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |||
20 | * OTHER DEALINGS IN THE SOFTWARE. | |||
21 | * | |||
22 | * Authors: Christian König | |||
23 | */ | |||
24 | ||||
25 | #include <linux/dma-mapping.h> | |||
26 | #include <drm/ttm/ttm_range_manager.h> | |||
27 | ||||
28 | #include "amdgpu.h" | |||
29 | #include "amdgpu_vm.h" | |||
30 | #include "amdgpu_res_cursor.h" | |||
31 | #include "amdgpu_atomfirmware.h" | |||
32 | #include "atom.h" | |||
33 | ||||
34 | struct amdgpu_vram_reservation { | |||
35 | u64 start; | |||
36 | u64 size; | |||
37 | struct list_head allocated; | |||
38 | struct list_head blocks; | |||
39 | }; | |||
40 | ||||
41 | static inline struct amdgpu_vram_mgr * | |||
42 | to_vram_mgr(struct ttm_resource_manager *man) | |||
43 | { | |||
44 | return container_of(man, struct amdgpu_vram_mgr, manager)({ const __typeof( ((struct amdgpu_vram_mgr *)0)->manager ) *__mptr = (man); (struct amdgpu_vram_mgr *)( (char *)__mptr - __builtin_offsetof(struct amdgpu_vram_mgr, manager) );}); | |||
45 | } | |||
46 | ||||
47 | static inline struct amdgpu_device * | |||
48 | to_amdgpu_device(struct amdgpu_vram_mgr *mgr) | |||
49 | { | |||
50 | return container_of(mgr, struct amdgpu_device, mman.vram_mgr)({ const __typeof( ((struct amdgpu_device *)0)->mman.vram_mgr ) *__mptr = (mgr); (struct amdgpu_device *)( (char *)__mptr - __builtin_offsetof(struct amdgpu_device, mman.vram_mgr) );}); | |||
51 | } | |||
52 | ||||
53 | static inline struct drm_buddy_block * | |||
54 | amdgpu_vram_mgr_first_block(struct list_head *list) | |||
55 | { | |||
56 | return list_first_entry_or_null(list, struct drm_buddy_block, link)(list_empty(list) ? ((void *)0) : ({ const __typeof( ((struct drm_buddy_block *)0)->link ) *__mptr = ((list)->next); (struct drm_buddy_block *)( (char *)__mptr - __builtin_offsetof (struct drm_buddy_block, link) );})); | |||
57 | } | |||
58 | ||||
59 | static inline bool_Bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) | |||
60 | { | |||
61 | struct drm_buddy_block *block; | |||
62 | u64 start, size; | |||
63 | ||||
64 | block = amdgpu_vram_mgr_first_block(head); | |||
65 | if (!block) | |||
66 | return false0; | |||
67 | ||||
68 | while (head != block->link.next) { | |||
69 | start = amdgpu_vram_mgr_block_start(block); | |||
70 | size = amdgpu_vram_mgr_block_size(block); | |||
71 | ||||
72 | block = list_entry(block->link.next, struct drm_buddy_block, link)({ const __typeof( ((struct drm_buddy_block *)0)->link ) * __mptr = (block->link.next); (struct drm_buddy_block *)( ( char *)__mptr - __builtin_offsetof(struct drm_buddy_block, link ) );}); | |||
73 | if (start + size != amdgpu_vram_mgr_block_start(block)) | |||
74 | return false0; | |||
75 | } | |||
76 | ||||
77 | return true1; | |||
78 | } | |||
79 | ||||
80 | ||||
81 | ||||
82 | /** | |||
83 | * DOC: mem_info_vram_total | |||
84 | * | |||
85 | * The amdgpu driver provides a sysfs API for reporting current total VRAM | |||
86 | * available on the device | |||
87 | * The file mem_info_vram_total is used for this and returns the total | |||
88 | * amount of VRAM in bytes | |||
89 | */ | |||
90 | static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, | |||
91 | struct device_attribute *attr, char *buf) | |||
92 | { | |||
93 | struct drm_device *ddev = dev_get_drvdata(dev); | |||
94 | struct amdgpu_device *adev = drm_to_adev(ddev); | |||
95 | ||||
96 | return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size); | |||
97 | } | |||
98 | ||||
99 | /** | |||
100 | * DOC: mem_info_vis_vram_total | |||
101 | * | |||
102 | * The amdgpu driver provides a sysfs API for reporting current total | |||
103 | * visible VRAM available on the device | |||
104 | * The file mem_info_vis_vram_total is used for this and returns the total | |||
105 | * amount of visible VRAM in bytes | |||
106 | */ | |||
107 | static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, | |||
108 | struct device_attribute *attr, char *buf) | |||
109 | { | |||
110 | struct drm_device *ddev = dev_get_drvdata(dev); | |||
111 | struct amdgpu_device *adev = drm_to_adev(ddev); | |||
112 | ||||
113 | return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size); | |||
114 | } | |||
115 | ||||
116 | /** | |||
117 | * DOC: mem_info_vram_used | |||
118 | * | |||
119 | * The amdgpu driver provides a sysfs API for reporting current total VRAM | |||
120 | * available on the device | |||
121 | * The file mem_info_vram_used is used for this and returns the total | |||
122 | * amount of currently used VRAM in bytes | |||
123 | */ | |||
124 | static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, | |||
125 | struct device_attribute *attr, | |||
126 | char *buf) | |||
127 | { | |||
128 | struct drm_device *ddev = dev_get_drvdata(dev); | |||
129 | struct amdgpu_device *adev = drm_to_adev(ddev); | |||
130 | struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager; | |||
131 | ||||
132 | return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man)); | |||
133 | } | |||
134 | ||||
135 | /** | |||
136 | * DOC: mem_info_vis_vram_used | |||
137 | * | |||
138 | * The amdgpu driver provides a sysfs API for reporting current total of | |||
139 | * used visible VRAM | |||
140 | * The file mem_info_vis_vram_used is used for this and returns the total | |||
141 | * amount of currently used visible VRAM in bytes | |||
142 | */ | |||
143 | static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, | |||
144 | struct device_attribute *attr, | |||
145 | char *buf) | |||
146 | { | |||
147 | struct drm_device *ddev = dev_get_drvdata(dev); | |||
148 | struct amdgpu_device *adev = drm_to_adev(ddev); | |||
149 | ||||
150 | return sysfs_emit(buf, "%llu\n", | |||
151 | amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr)); | |||
152 | } | |||
153 | ||||
154 | /** | |||
155 | * DOC: mem_info_vram_vendor | |||
156 | * | |||
157 | * The amdgpu driver provides a sysfs API for reporting the vendor of the | |||
158 | * installed VRAM | |||
159 | * The file mem_info_vram_vendor is used for this and returns the name of the | |||
160 | * vendor. | |||
161 | */ | |||
162 | static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, | |||
163 | struct device_attribute *attr, | |||
164 | char *buf) | |||
165 | { | |||
166 | struct drm_device *ddev = dev_get_drvdata(dev); | |||
167 | struct amdgpu_device *adev = drm_to_adev(ddev); | |||
168 | ||||
169 | switch (adev->gmc.vram_vendor) { | |||
170 | case SAMSUNG0x1: | |||
171 | return sysfs_emit(buf, "samsung\n"); | |||
172 | case INFINEON0x2: | |||
173 | return sysfs_emit(buf, "infineon\n"); | |||
174 | case ELPIDA0x3: | |||
175 | return sysfs_emit(buf, "elpida\n"); | |||
176 | case ETRON0x4: | |||
177 | return sysfs_emit(buf, "etron\n"); | |||
178 | case NANYA0x5: | |||
179 | return sysfs_emit(buf, "nanya\n"); | |||
180 | case HYNIX0x6: | |||
181 | return sysfs_emit(buf, "hynix\n"); | |||
182 | case MOSEL0x7: | |||
183 | return sysfs_emit(buf, "mosel\n"); | |||
184 | case WINBOND0x8: | |||
185 | return sysfs_emit(buf, "winbond\n"); | |||
186 | case ESMT0x9: | |||
187 | return sysfs_emit(buf, "esmt\n"); | |||
188 | case MICRON0xF: | |||
189 | return sysfs_emit(buf, "micron\n"); | |||
190 | default: | |||
191 | return sysfs_emit(buf, "unknown\n"); | |||
192 | } | |||
193 | } | |||
194 | ||||
195 | static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,struct device_attribute dev_attr_mem_info_vram_total | |||
196 | amdgpu_mem_info_vram_total_show, NULL)struct device_attribute dev_attr_mem_info_vram_total; | |||
197 | static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,struct device_attribute dev_attr_mem_info_vis_vram_total | |||
198 | amdgpu_mem_info_vis_vram_total_show,NULL)struct device_attribute dev_attr_mem_info_vis_vram_total; | |||
199 | static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,struct device_attribute dev_attr_mem_info_vram_used | |||
200 | amdgpu_mem_info_vram_used_show, NULL)struct device_attribute dev_attr_mem_info_vram_used; | |||
201 | static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,struct device_attribute dev_attr_mem_info_vis_vram_used | |||
202 | amdgpu_mem_info_vis_vram_used_show, NULL)struct device_attribute dev_attr_mem_info_vis_vram_used; | |||
203 | static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,struct device_attribute dev_attr_mem_info_vram_vendor | |||
204 | amdgpu_mem_info_vram_vendor, NULL)struct device_attribute dev_attr_mem_info_vram_vendor; | |||
205 | ||||
206 | static struct attribute *amdgpu_vram_mgr_attributes[] = { | |||
207 | &dev_attr_mem_info_vram_total.attr, | |||
208 | &dev_attr_mem_info_vis_vram_total.attr, | |||
209 | &dev_attr_mem_info_vram_used.attr, | |||
210 | &dev_attr_mem_info_vis_vram_used.attr, | |||
211 | &dev_attr_mem_info_vram_vendor.attr, | |||
212 | NULL((void *)0) | |||
213 | }; | |||
214 | ||||
215 | const struct attribute_group amdgpu_vram_mgr_attr_group = { | |||
216 | .attrs = amdgpu_vram_mgr_attributes | |||
217 | }; | |||
218 | ||||
219 | /** | |||
220 | * amdgpu_vram_mgr_vis_size - Calculate visible block size | |||
221 | * | |||
222 | * @adev: amdgpu_device pointer | |||
223 | * @block: DRM BUDDY block structure | |||
224 | * | |||
225 | * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM | |||
226 | */ | |||
227 | static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, | |||
228 | struct drm_buddy_block *block) | |||
229 | { | |||
230 | u64 start = amdgpu_vram_mgr_block_start(block); | |||
231 | u64 end = start + amdgpu_vram_mgr_block_size(block); | |||
232 | ||||
233 | if (start >= adev->gmc.visible_vram_size) | |||
234 | return 0; | |||
235 | ||||
236 | return (end > adev->gmc.visible_vram_size ? | |||
237 | adev->gmc.visible_vram_size : end) - start; | |||
238 | } | |||
239 | ||||
240 | /** | |||
241 | * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size | |||
242 | * | |||
243 | * @bo: &amdgpu_bo buffer object (must be in VRAM) | |||
244 | * | |||
245 | * Returns: | |||
246 | * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. | |||
247 | */ | |||
248 | u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) | |||
249 | { | |||
250 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |||
251 | struct ttm_resource *res = bo->tbo.resource; | |||
252 | struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); | |||
253 | struct drm_buddy_block *block; | |||
254 | u64 usage = 0; | |||
255 | ||||
256 | if (amdgpu_gmc_vram_full_visible(&adev->gmc)) | |||
257 | return amdgpu_bo_size(bo); | |||
258 | ||||
259 | if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT12) | |||
260 | return 0; | |||
261 | ||||
262 | list_for_each_entry(block, &vres->blocks, link)for (block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = ((&vres->blocks)->next); (__typeof(*block ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link ) );}); &block->link != (&vres->blocks); block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = (block->link.next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );})) | |||
263 | usage += amdgpu_vram_mgr_vis_size(adev, block); | |||
264 | ||||
265 | return usage; | |||
266 | } | |||
267 | ||||
268 | /* Commit the reservation of VRAM pages */ | |||
269 | static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) | |||
270 | { | |||
271 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); | |||
272 | struct amdgpu_device *adev = to_amdgpu_device(mgr); | |||
273 | struct drm_buddy *mm = &mgr->mm; | |||
274 | struct amdgpu_vram_reservation *rsv, *temp; | |||
275 | struct drm_buddy_block *block; | |||
276 | uint64_t vis_usage; | |||
277 | ||||
278 | list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)for (rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = ((&mgr->reservations_pending)->next); ( __typeof(*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof (*rsv), blocks) );}), temp = ({ const __typeof( ((__typeof(*rsv ) *)0)->blocks ) *__mptr = (rsv->blocks.next); (__typeof (*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rsv) , blocks) );}); &rsv->blocks != (&mgr->reservations_pending ); rsv = temp, temp = ({ const __typeof( ((__typeof(*temp) *) 0)->blocks ) *__mptr = (temp->blocks.next); (__typeof(* temp) *)( (char *)__mptr - __builtin_offsetof(__typeof(*temp) , blocks) );})) { | |||
279 | if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, | |||
280 | rsv->size, mm->chunk_size, &rsv->allocated, | |||
281 | DRM_BUDDY_RANGE_ALLOCATION(1 << 0))) | |||
282 | continue; | |||
283 | ||||
284 | block = amdgpu_vram_mgr_first_block(&rsv->allocated); | |||
285 | if (!block) | |||
286 | continue; | |||
287 | ||||
288 | dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",do { } while(0) | |||
289 | rsv->start, rsv->size)do { } while(0); | |||
290 | ||||
291 | vis_usage = amdgpu_vram_mgr_vis_size(adev, block); | |||
292 | atomic64_add(vis_usage, &mgr->vis_usage)__sync_fetch_and_add_8(&mgr->vis_usage, vis_usage); | |||
293 | spin_lock(&man->bdev->lru_lock)mtx_enter(&man->bdev->lru_lock); | |||
294 | man->usage += rsv->size; | |||
295 | spin_unlock(&man->bdev->lru_lock)mtx_leave(&man->bdev->lru_lock); | |||
296 | list_move(&rsv->blocks, &mgr->reserved_pages); | |||
297 | } | |||
298 | } | |||
299 | ||||
300 | /** | |||
301 | * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM | |||
302 | * | |||
303 | * @mgr: amdgpu_vram_mgr pointer | |||
304 | * @start: start address of the range in VRAM | |||
305 | * @size: size of the range | |||
306 | * | |||
307 | * Reserve memory from start address with the specified size in VRAM | |||
308 | */ | |||
309 | int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, | |||
310 | uint64_t start, uint64_t size) | |||
311 | { | |||
312 | struct amdgpu_vram_reservation *rsv; | |||
313 | ||||
314 | rsv = kzalloc(sizeof(*rsv), GFP_KERNEL(0x0001 | 0x0004)); | |||
315 | if (!rsv) | |||
316 | return -ENOMEM12; | |||
317 | ||||
318 | INIT_LIST_HEAD(&rsv->allocated); | |||
319 | INIT_LIST_HEAD(&rsv->blocks); | |||
320 | ||||
321 | rsv->start = start; | |||
322 | rsv->size = size; | |||
323 | ||||
324 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
325 | list_add_tail(&rsv->blocks, &mgr->reservations_pending); | |||
326 | amdgpu_vram_mgr_do_reserve(&mgr->manager); | |||
327 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
328 | ||||
329 | return 0; | |||
330 | } | |||
331 | ||||
332 | /** | |||
333 | * amdgpu_vram_mgr_query_page_status - query the reservation status | |||
334 | * | |||
335 | * @mgr: amdgpu_vram_mgr pointer | |||
336 | * @start: start address of a page in VRAM | |||
337 | * | |||
338 | * Returns: | |||
339 | * -EBUSY: the page is still hold and in pending list | |||
340 | * 0: the page has been reserved | |||
341 | * -ENOENT: the input page is not a reservation | |||
342 | */ | |||
343 | int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, | |||
344 | uint64_t start) | |||
345 | { | |||
346 | struct amdgpu_vram_reservation *rsv; | |||
347 | int ret; | |||
348 | ||||
349 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
350 | ||||
351 | list_for_each_entry(rsv, &mgr->reservations_pending, blocks)for (rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = ((&mgr->reservations_pending)->next); ( __typeof(*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof (*rsv), blocks) );}); &rsv->blocks != (&mgr->reservations_pending ); rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = (rsv->blocks.next); (__typeof(*rsv) *)( (char * )__mptr - __builtin_offsetof(__typeof(*rsv), blocks) );})) { | |||
352 | if (rsv->start <= start && | |||
353 | (start < (rsv->start + rsv->size))) { | |||
354 | ret = -EBUSY16; | |||
355 | goto out; | |||
356 | } | |||
357 | } | |||
358 | ||||
359 | list_for_each_entry(rsv, &mgr->reserved_pages, blocks)for (rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = ((&mgr->reserved_pages)->next); (__typeof (*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rsv) , blocks) );}); &rsv->blocks != (&mgr->reserved_pages ); rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = (rsv->blocks.next); (__typeof(*rsv) *)( (char * )__mptr - __builtin_offsetof(__typeof(*rsv), blocks) );})) { | |||
360 | if (rsv->start <= start && | |||
361 | (start < (rsv->start + rsv->size))) { | |||
362 | ret = 0; | |||
363 | goto out; | |||
364 | } | |||
365 | } | |||
366 | ||||
367 | ret = -ENOENT2; | |||
368 | out: | |||
369 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
370 | return ret; | |||
371 | } | |||
372 | ||||
373 | /** | |||
374 | * amdgpu_vram_mgr_new - allocate new ranges | |||
375 | * | |||
376 | * @man: TTM memory type manager | |||
377 | * @tbo: TTM BO we need this range for | |||
378 | * @place: placement flags and restrictions | |||
379 | * @res: the resulting mem object | |||
380 | * | |||
381 | * Allocate VRAM for the given BO. | |||
382 | */ | |||
383 | static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, | |||
384 | struct ttm_buffer_object *tbo, | |||
385 | const struct ttm_place *place, | |||
386 | struct ttm_resource **res) | |||
387 | { | |||
388 | u64 vis_usage = 0, max_bytes, cur_size, min_block_size; | |||
| ||||
389 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); | |||
390 | struct amdgpu_device *adev = to_amdgpu_device(mgr); | |||
391 | struct amdgpu_vram_mgr_resource *vres; | |||
392 | u64 size, remaining_size, lpfn, fpfn; | |||
393 | struct drm_buddy *mm = &mgr->mm; | |||
394 | struct drm_buddy_block *block; | |||
395 | unsigned long pages_per_block; | |||
396 | int r; | |||
397 | ||||
398 | lpfn = (u64)place->lpfn << PAGE_SHIFT12; | |||
399 | if (!lpfn) | |||
400 | lpfn = man->size; | |||
401 | ||||
402 | fpfn = (u64)place->fpfn << PAGE_SHIFT12; | |||
403 | ||||
404 | max_bytes = adev->gmc.mc_vram_size; | |||
405 | if (tbo->type != ttm_bo_type_kernel) | |||
406 | max_bytes -= AMDGPU_VM_RESERVED_VRAM(8ULL << 20); | |||
407 | ||||
408 | if (place->flags & TTM_PL_FLAG_CONTIGUOUS(1 << 0)) { | |||
409 | pages_per_block = ~0ul; | |||
410 | } else { | |||
411 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |||
412 | pages_per_block = HPAGE_PMD_NR; | |||
413 | #else | |||
414 | /* default to 2MB */ | |||
415 | pages_per_block = 2UL << (20UL - PAGE_SHIFT12); | |||
416 | #endif | |||
417 | pages_per_block = max_t(uint32_t, pages_per_block,({ uint32_t __max_a = (pages_per_block); uint32_t __max_b = ( tbo->page_alignment); __max_a > __max_b ? __max_a : __max_b ; }) | |||
418 | tbo->page_alignment)({ uint32_t __max_a = (pages_per_block); uint32_t __max_b = ( tbo->page_alignment); __max_a > __max_b ? __max_a : __max_b ; }); | |||
419 | } | |||
420 | ||||
421 | vres = kzalloc(sizeof(*vres), GFP_KERNEL(0x0001 | 0x0004)); | |||
422 | if (!vres) | |||
423 | return -ENOMEM12; | |||
424 | ||||
425 | ttm_resource_init(tbo, place, &vres->base); | |||
426 | ||||
427 | /* bail out quickly if there's likely not enough VRAM for this BO */ | |||
428 | if (ttm_resource_manager_usage(man) > max_bytes) { | |||
429 | r = -ENOSPC28; | |||
430 | goto error_fini; | |||
431 | } | |||
432 | ||||
433 | INIT_LIST_HEAD(&vres->blocks); | |||
434 | ||||
435 | if (place->flags & TTM_PL_FLAG_TOPDOWN(1 << 1)) | |||
436 | vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION(1 << 1); | |||
437 | ||||
438 | if (fpfn || lpfn != mgr->mm.size) | |||
439 | /* Allocate blocks in desired range */ | |||
440 | vres->flags |= DRM_BUDDY_RANGE_ALLOCATION(1 << 0); | |||
441 | ||||
442 | remaining_size = (u64)vres->base.num_pages << PAGE_SHIFT12; | |||
443 | ||||
444 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
445 | while (remaining_size) { | |||
446 | if (tbo->page_alignment) | |||
447 | min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT12; | |||
448 | else | |||
449 | min_block_size = mgr->default_page_size; | |||
450 | ||||
451 | BUG_ON(min_block_size < mm->chunk_size)((!(min_block_size < mm->chunk_size)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/dev/pci/drm/amd/amdgpu/amdgpu_vram_mgr.c" , 451, "!(min_block_size < mm->chunk_size)")); | |||
452 | ||||
453 | /* Limit maximum size to 2GiB due to SG table limitations */ | |||
454 | size = min(remaining_size, 2ULL << 30)(((remaining_size)<(2ULL << 30))?(remaining_size):(2ULL << 30)); | |||
455 | ||||
456 | if (size >= (u64)pages_per_block << PAGE_SHIFT12) | |||
457 | min_block_size = (u64)pages_per_block << PAGE_SHIFT12; | |||
458 | ||||
459 | cur_size = size; | |||
460 | ||||
461 | if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT12) { | |||
462 | /* | |||
463 | * Except for actual range allocation, modify the size and | |||
464 | * min_block_size conforming to continuous flag enablement | |||
465 | */ | |||
466 | if (place->flags & TTM_PL_FLAG_CONTIGUOUS(1 << 0)) { | |||
467 | size = roundup_pow_of_two(size); | |||
468 | min_block_size = size; | |||
469 | /* | |||
470 | * Modify the size value if size is not | |||
471 | * aligned with min_block_size | |||
472 | */ | |||
473 | } else if (!IS_ALIGNED(size, min_block_size)(((size) & ((min_block_size) - 1)) == 0)) { | |||
474 | size = round_up(size, min_block_size)((((size) + ((min_block_size) - 1)) / (min_block_size)) * (min_block_size )); | |||
475 | } | |||
476 | } | |||
477 | ||||
478 | r = drm_buddy_alloc_blocks(mm, fpfn, | |||
479 | lpfn, | |||
480 | size, | |||
481 | min_block_size, | |||
482 | &vres->blocks, | |||
483 | vres->flags); | |||
484 | if (unlikely(r)__builtin_expect(!!(r), 0)) | |||
485 | goto error_free_blocks; | |||
486 | ||||
487 | if (size > remaining_size) | |||
488 | remaining_size = 0; | |||
489 | else | |||
490 | remaining_size -= size; | |||
491 | } | |||
492 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
493 | ||||
494 | if (cur_size != size) { | |||
| ||||
495 | struct drm_buddy_block *block; | |||
496 | struct list_head *trim_list; | |||
497 | u64 original_size; | |||
498 | DRM_LIST_HEAD(temp)struct list_head temp = { &(temp), &(temp) }; | |||
499 | ||||
500 | trim_list = &vres->blocks; | |||
501 | original_size = (u64)vres->base.num_pages << PAGE_SHIFT12; | |||
502 | ||||
503 | /* | |||
504 | * If size value is rounded up to min_block_size, trim the last | |||
505 | * block to the required size | |||
506 | */ | |||
507 | if (!list_is_singular(&vres->blocks)) { | |||
508 | block = list_last_entry(&vres->blocks, typeof(*block), link)({ const __typeof( ((typeof(*block) *)0)->link ) *__mptr = ((&vres->blocks)->prev); (typeof(*block) *)( (char *)__mptr - __builtin_offsetof(typeof(*block), link) );}); | |||
509 | list_move_tail(&block->link, &temp); | |||
510 | trim_list = &temp; | |||
511 | /* | |||
512 | * Compute the original_size value by subtracting the | |||
513 | * last block size with (aligned size - original size) | |||
514 | */ | |||
515 | original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size); | |||
516 | } | |||
517 | ||||
518 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
519 | drm_buddy_block_trim(mm, | |||
520 | original_size, | |||
521 | trim_list); | |||
522 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
523 | ||||
524 | if (!list_empty(&temp)) | |||
525 | list_splice_tail(trim_list, &vres->blocks); | |||
526 | } | |||
527 | ||||
528 | vres->base.start = 0; | |||
529 | list_for_each_entry(block, &vres->blocks, link)for (block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = ((&vres->blocks)->next); (__typeof(*block ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link ) );}); &block->link != (&vres->blocks); block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = (block->link.next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );})) { | |||
530 | unsigned long start; | |||
531 | ||||
532 | start = amdgpu_vram_mgr_block_start(block) + | |||
533 | amdgpu_vram_mgr_block_size(block); | |||
534 | start >>= PAGE_SHIFT12; | |||
535 | ||||
536 | if (start > vres->base.num_pages) | |||
537 | start -= vres->base.num_pages; | |||
538 | else | |||
539 | start = 0; | |||
540 | vres->base.start = max(vres->base.start, start)(((vres->base.start)>(start))?(vres->base.start):(start )); | |||
541 | ||||
542 | vis_usage += amdgpu_vram_mgr_vis_size(adev, block); | |||
543 | } | |||
544 | ||||
545 | if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks)) | |||
546 | vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS(1 << 0); | |||
547 | ||||
548 | if (adev->gmc.xgmi.connected_to_cpu) | |||
549 | vres->base.bus.caching = ttm_cached; | |||
550 | else | |||
551 | vres->base.bus.caching = ttm_write_combined; | |||
552 | ||||
553 | atomic64_add(vis_usage, &mgr->vis_usage)__sync_fetch_and_add_8(&mgr->vis_usage, vis_usage); | |||
554 | *res = &vres->base; | |||
555 | return 0; | |||
556 | ||||
557 | error_free_blocks: | |||
558 | drm_buddy_free_list(mm, &vres->blocks); | |||
559 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
560 | error_fini: | |||
561 | ttm_resource_fini(man, &vres->base); | |||
562 | kfree(vres); | |||
563 | ||||
564 | return r; | |||
565 | } | |||
566 | ||||
567 | /** | |||
568 | * amdgpu_vram_mgr_del - free ranges | |||
569 | * | |||
570 | * @man: TTM memory type manager | |||
571 | * @res: TTM memory object | |||
572 | * | |||
573 | * Free the allocated VRAM again. | |||
574 | */ | |||
575 | static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, | |||
576 | struct ttm_resource *res) | |||
577 | { | |||
578 | struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); | |||
579 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); | |||
580 | struct amdgpu_device *adev = to_amdgpu_device(mgr); | |||
581 | struct drm_buddy *mm = &mgr->mm; | |||
582 | struct drm_buddy_block *block; | |||
583 | uint64_t vis_usage = 0; | |||
584 | ||||
585 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
586 | list_for_each_entry(block, &vres->blocks, link)for (block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = ((&vres->blocks)->next); (__typeof(*block ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link ) );}); &block->link != (&vres->blocks); block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = (block->link.next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );})) | |||
587 | vis_usage += amdgpu_vram_mgr_vis_size(adev, block); | |||
588 | ||||
589 | amdgpu_vram_mgr_do_reserve(man); | |||
590 | ||||
591 | drm_buddy_free_list(mm, &vres->blocks); | |||
592 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
593 | ||||
594 | atomic64_sub(vis_usage, &mgr->vis_usage)__sync_fetch_and_sub_8(&mgr->vis_usage, vis_usage); | |||
595 | ||||
596 | ttm_resource_fini(man, res); | |||
597 | kfree(vres); | |||
598 | } | |||
599 | ||||
600 | /** | |||
601 | * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table | |||
602 | * | |||
603 | * @adev: amdgpu device pointer | |||
604 | * @res: TTM memory object | |||
605 | * @offset: byte offset from the base of VRAM BO | |||
606 | * @length: number of bytes to export in sg_table | |||
607 | * @dev: the other device | |||
608 | * @dir: dma direction | |||
609 | * @sgt: resulting sg table | |||
610 | * | |||
611 | * Allocate and fill a sg table from a VRAM allocation. | |||
612 | */ | |||
613 | int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, | |||
614 | struct ttm_resource *res, | |||
615 | u64 offset, u64 length, | |||
616 | struct device *dev, | |||
617 | enum dma_data_direction dir, | |||
618 | struct sg_table **sgt) | |||
619 | { | |||
620 | STUB()do { printf("%s: stub\n", __func__); } while(0); | |||
621 | return -ENOSYS78; | |||
622 | #ifdef notyet | |||
623 | struct amdgpu_res_cursor cursor; | |||
624 | struct scatterlist *sg; | |||
625 | int num_entries = 0; | |||
626 | int i, r; | |||
627 | ||||
628 | *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL(0x0001 | 0x0004)); | |||
629 | if (!*sgt) | |||
630 | return -ENOMEM12; | |||
631 | ||||
632 | /* Determine the number of DRM_BUDDY blocks to export */ | |||
633 | amdgpu_res_first(res, offset, length, &cursor); | |||
634 | while (cursor.remaining) { | |||
635 | num_entries++; | |||
636 | amdgpu_res_next(&cursor, cursor.size); | |||
637 | } | |||
638 | ||||
639 | r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL(0x0001 | 0x0004)); | |||
640 | if (r) | |||
641 | goto error_free; | |||
642 | ||||
643 | /* Initialize scatterlist nodes of sg_table */ | |||
644 | for_each_sgtable_sg((*sgt), sg, i) | |||
645 | sg->length = 0; | |||
646 | ||||
647 | /* | |||
648 | * Walk down DRM_BUDDY blocks to populate scatterlist nodes | |||
649 | * @note: Use iterator api to get first the DRM_BUDDY block | |||
650 | * and the number of bytes from it. Access the following | |||
651 | * DRM_BUDDY block(s) if more buffer needs to exported | |||
652 | */ | |||
653 | amdgpu_res_first(res, offset, length, &cursor); | |||
654 | for_each_sgtable_sg((*sgt), sg, i) { | |||
655 | phys_addr_t phys = cursor.start + adev->gmc.aper_base; | |||
656 | size_t size = cursor.size; | |||
657 | dma_addr_t addr; | |||
658 | ||||
659 | addr = dma_map_resource(dev, phys, size, dir, | |||
660 | DMA_ATTR_SKIP_CPU_SYNC); | |||
661 | r = dma_mapping_error(dev, addr); | |||
662 | if (r) | |||
663 | goto error_unmap; | |||
664 | ||||
665 | sg_set_page(sg, NULL((void *)0), size, 0); | |||
666 | sg_dma_address(sg)((sg)->dma_address) = addr; | |||
667 | sg_dma_len(sg)((sg)->length) = size; | |||
668 | ||||
669 | amdgpu_res_next(&cursor, cursor.size); | |||
670 | } | |||
671 | ||||
672 | return 0; | |||
673 | ||||
674 | error_unmap: | |||
675 | for_each_sgtable_sg((*sgt), sg, i) { | |||
676 | if (!sg->length) | |||
677 | continue; | |||
678 | ||||
679 | dma_unmap_resource(dev, sg->dma_address, | |||
680 | sg->length, dir, | |||
681 | DMA_ATTR_SKIP_CPU_SYNC); | |||
682 | } | |||
683 | sg_free_table(*sgt); | |||
684 | ||||
685 | error_free: | |||
686 | kfree(*sgt); | |||
687 | return r; | |||
688 | #endif | |||
689 | } | |||
690 | ||||
691 | /** | |||
692 | * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table | |||
693 | * | |||
694 | * @dev: device pointer | |||
695 | * @dir: data direction of resource to unmap | |||
696 | * @sgt: sg table to free | |||
697 | * | |||
698 | * Free a previously allocate sg table. | |||
699 | */ | |||
700 | void amdgpu_vram_mgr_free_sgt(struct device *dev, | |||
701 | enum dma_data_direction dir, | |||
702 | struct sg_table *sgt) | |||
703 | { | |||
704 | STUB()do { printf("%s: stub\n", __func__); } while(0); | |||
705 | #ifdef notyet | |||
706 | struct scatterlist *sg; | |||
707 | int i; | |||
708 | ||||
709 | for_each_sgtable_sg(sgt, sg, i) | |||
710 | dma_unmap_resource(dev, sg->dma_address, | |||
711 | sg->length, dir, | |||
712 | DMA_ATTR_SKIP_CPU_SYNC); | |||
713 | sg_free_table(sgt); | |||
714 | kfree(sgt); | |||
715 | #endif | |||
716 | } | |||
717 | ||||
718 | /** | |||
719 | * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part | |||
720 | * | |||
721 | * @mgr: amdgpu_vram_mgr pointer | |||
722 | * | |||
723 | * Returns how many bytes are used in the visible part of VRAM | |||
724 | */ | |||
725 | uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) | |||
726 | { | |||
727 | return atomic64_read(&mgr->vis_usage)({ typeof(*(&mgr->vis_usage)) __tmp = *(volatile typeof (*(&mgr->vis_usage)) *)&(*(&mgr->vis_usage) ); membar_datadep_consumer(); __tmp; }); | |||
728 | } | |||
729 | ||||
730 | /** | |||
731 | * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection | |||
732 | * | |||
733 | * @man: TTM memory type manager | |||
734 | * @res: The resource to test | |||
735 | * @place: The place to test against | |||
736 | * @size: Size of the new allocation | |||
737 | * | |||
738 | * Test each drm buddy block for intersection for eviction decision. | |||
739 | */ | |||
740 | static bool_Bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, | |||
741 | struct ttm_resource *res, | |||
742 | const struct ttm_place *place, | |||
743 | size_t size) | |||
744 | { | |||
745 | struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); | |||
746 | struct drm_buddy_block *block; | |||
747 | ||||
748 | /* Check each drm buddy block individually */ | |||
749 | list_for_each_entry(block, &mgr->blocks, link)for (block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = ((&mgr->blocks)->next); (__typeof(*block ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link ) );}); &block->link != (&mgr->blocks); block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = (block->link.next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );})) { | |||
750 | unsigned long fpfn = | |||
751 | amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT12; | |||
752 | unsigned long lpfn = fpfn + | |||
753 | (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT12); | |||
754 | ||||
755 | if (place->fpfn < lpfn && | |||
756 | (!place->lpfn || place->lpfn > fpfn)) | |||
757 | return true1; | |||
758 | } | |||
759 | ||||
760 | return false0; | |||
761 | } | |||
762 | ||||
763 | /** | |||
764 | * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility | |||
765 | * | |||
766 | * @man: TTM memory type manager | |||
767 | * @res: The resource to test | |||
768 | * @place: The place to test against | |||
769 | * @size: Size of the new allocation | |||
770 | * | |||
771 | * Test each drm buddy block for placement compatibility. | |||
772 | */ | |||
773 | static bool_Bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, | |||
774 | struct ttm_resource *res, | |||
775 | const struct ttm_place *place, | |||
776 | size_t size) | |||
777 | { | |||
778 | struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); | |||
779 | struct drm_buddy_block *block; | |||
780 | ||||
781 | /* Check each drm buddy block individually */ | |||
782 | list_for_each_entry(block, &mgr->blocks, link)for (block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = ((&mgr->blocks)->next); (__typeof(*block ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link ) );}); &block->link != (&mgr->blocks); block = ({ const __typeof( ((__typeof(*block) *)0)->link ) *__mptr = (block->link.next); (__typeof(*block) *)( (char *)__mptr - __builtin_offsetof(__typeof(*block), link) );})) { | |||
783 | unsigned long fpfn = | |||
784 | amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT12; | |||
785 | unsigned long lpfn = fpfn + | |||
786 | (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT12); | |||
787 | ||||
788 | if (fpfn < place->fpfn || | |||
789 | (place->lpfn && lpfn > place->lpfn)) | |||
790 | return false0; | |||
791 | } | |||
792 | ||||
793 | return true1; | |||
794 | } | |||
795 | ||||
796 | /** | |||
797 | * amdgpu_vram_mgr_debug - dump VRAM table | |||
798 | * | |||
799 | * @man: TTM memory type manager | |||
800 | * @printer: DRM printer to use | |||
801 | * | |||
802 | * Dump the table content using printk. | |||
803 | */ | |||
804 | static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, | |||
805 | struct drm_printer *printer) | |||
806 | { | |||
807 | struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); | |||
808 | struct drm_buddy *mm = &mgr->mm; | |||
809 | struct amdgpu_vram_reservation *rsv; | |||
810 | ||||
811 | drm_printf(printer, " vis usage:%llu\n", | |||
812 | amdgpu_vram_mgr_vis_usage(mgr)); | |||
813 | ||||
814 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
815 | drm_printf(printer, "default_page_size: %lluKiB\n", | |||
816 | mgr->default_page_size >> 10); | |||
817 | ||||
818 | drm_buddy_print(mm, printer); | |||
819 | ||||
820 | drm_printf(printer, "reserved:\n"); | |||
821 | list_for_each_entry(rsv, &mgr->reserved_pages, blocks)for (rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = ((&mgr->reserved_pages)->next); (__typeof (*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rsv) , blocks) );}); &rsv->blocks != (&mgr->reserved_pages ); rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = (rsv->blocks.next); (__typeof(*rsv) *)( (char * )__mptr - __builtin_offsetof(__typeof(*rsv), blocks) );})) | |||
822 | drm_printf(printer, "%#018llx-%#018llx: %llu\n", | |||
823 | rsv->start, rsv->start + rsv->size, rsv->size); | |||
824 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
825 | } | |||
826 | ||||
827 | static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { | |||
828 | .alloc = amdgpu_vram_mgr_new, | |||
829 | .free = amdgpu_vram_mgr_del, | |||
830 | .intersects = amdgpu_vram_mgr_intersects, | |||
831 | .compatible = amdgpu_vram_mgr_compatible, | |||
832 | .debug = amdgpu_vram_mgr_debug | |||
833 | }; | |||
834 | ||||
835 | /** | |||
836 | * amdgpu_vram_mgr_init - init VRAM manager and DRM MM | |||
837 | * | |||
838 | * @adev: amdgpu_device pointer | |||
839 | * | |||
840 | * Allocate and initialize the VRAM manager. | |||
841 | */ | |||
842 | int amdgpu_vram_mgr_init(struct amdgpu_device *adev) | |||
843 | { | |||
844 | struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; | |||
845 | struct ttm_resource_manager *man = &mgr->manager; | |||
846 | int err; | |||
847 | ||||
848 | ttm_resource_manager_init(man, &adev->mman.bdev, | |||
849 | adev->gmc.real_vram_size); | |||
850 | ||||
851 | man->func = &amdgpu_vram_mgr_func; | |||
852 | ||||
853 | err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE(1 << 12)); | |||
854 | if (err) | |||
855 | return err; | |||
856 | ||||
857 | rw_init(&mgr->lock, "vrmgr")_rw_init_flags(&mgr->lock, "vrmgr", 0, ((void *)0)); | |||
858 | INIT_LIST_HEAD(&mgr->reservations_pending); | |||
859 | INIT_LIST_HEAD(&mgr->reserved_pages); | |||
860 | mgr->default_page_size = PAGE_SIZE(1 << 12); | |||
861 | ||||
862 | ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM2, &mgr->manager); | |||
863 | ttm_resource_manager_set_used(man, true1); | |||
864 | return 0; | |||
865 | } | |||
866 | ||||
867 | /** | |||
868 | * amdgpu_vram_mgr_fini - free and destroy VRAM manager | |||
869 | * | |||
870 | * @adev: amdgpu_device pointer | |||
871 | * | |||
872 | * Destroy and free the VRAM manager, returns -EBUSY if ranges are still | |||
873 | * allocated inside it. | |||
874 | */ | |||
875 | void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) | |||
876 | { | |||
877 | struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; | |||
878 | struct ttm_resource_manager *man = &mgr->manager; | |||
879 | int ret; | |||
880 | struct amdgpu_vram_reservation *rsv, *temp; | |||
881 | ||||
882 | ttm_resource_manager_set_used(man, false0); | |||
883 | ||||
884 | ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); | |||
885 | if (ret) | |||
886 | return; | |||
887 | ||||
888 | mutex_lock(&mgr->lock)rw_enter_write(&mgr->lock); | |||
889 | list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)for (rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = ((&mgr->reservations_pending)->next); ( __typeof(*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof (*rsv), blocks) );}), temp = ({ const __typeof( ((__typeof(*rsv ) *)0)->blocks ) *__mptr = (rsv->blocks.next); (__typeof (*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rsv) , blocks) );}); &rsv->blocks != (&mgr->reservations_pending ); rsv = temp, temp = ({ const __typeof( ((__typeof(*temp) *) 0)->blocks ) *__mptr = (temp->blocks.next); (__typeof(* temp) *)( (char *)__mptr - __builtin_offsetof(__typeof(*temp) , blocks) );})) | |||
890 | kfree(rsv); | |||
891 | ||||
892 | list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks)for (rsv = ({ const __typeof( ((__typeof(*rsv) *)0)->blocks ) *__mptr = ((&mgr->reserved_pages)->next); (__typeof (*rsv) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rsv) , blocks) );}), temp = ({ const __typeof( ((__typeof(*rsv) *) 0)->blocks ) *__mptr = (rsv->blocks.next); (__typeof(*rsv ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*rsv), blocks ) );}); &rsv->blocks != (&mgr->reserved_pages); rsv = temp, temp = ({ const __typeof( ((__typeof(*temp) *)0) ->blocks ) *__mptr = (temp->blocks.next); (__typeof(*temp ) *)( (char *)__mptr - __builtin_offsetof(__typeof(*temp), blocks ) );})) { | |||
893 | drm_buddy_free_list(&mgr->mm, &rsv->allocated); | |||
894 | kfree(rsv); | |||
895 | } | |||
896 | drm_buddy_fini(&mgr->mm); | |||
897 | mutex_unlock(&mgr->lock)rw_exit_write(&mgr->lock); | |||
898 | ||||
899 | ttm_resource_manager_cleanup(man); | |||
900 | ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM2, NULL((void *)0)); | |||
901 | } |