File: | dev/pci/drm/radeon/radeon_gart.c |
Warning: | line 300, column 7 Array access (via field 'pages') results in a null pointer dereference |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* | |||
2 | * Copyright 2008 Advanced Micro Devices, Inc. | |||
3 | * Copyright 2008 Red Hat Inc. | |||
4 | * Copyright 2009 Jerome Glisse. | |||
5 | * | |||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | |||
7 | * copy of this software and associated documentation files (the "Software"), | |||
8 | * to deal in the Software without restriction, including without limitation | |||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |||
10 | * and/or sell copies of the Software, and to permit persons to whom the | |||
11 | * Software is furnished to do so, subject to the following conditions: | |||
12 | * | |||
13 | * The above copyright notice and this permission notice shall be included in | |||
14 | * all copies or substantial portions of the Software. | |||
15 | * | |||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |||
19 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |||
20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |||
21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |||
22 | * OTHER DEALINGS IN THE SOFTWARE. | |||
23 | * | |||
24 | * Authors: Dave Airlie | |||
25 | * Alex Deucher | |||
26 | * Jerome Glisse | |||
27 | */ | |||
28 | ||||
29 | #include <linux/pci.h> | |||
30 | #include <linux/vmalloc.h> | |||
31 | ||||
32 | #include <drm/radeon_drm.h> | |||
33 | #ifdef CONFIG_X861 | |||
34 | #include <asm/set_memory.h> | |||
35 | #endif | |||
36 | #include "radeon.h" | |||
37 | ||||
38 | /* | |||
39 | * GART | |||
40 | * The GART (Graphics Aperture Remapping Table) is an aperture | |||
41 | * in the GPU's address space. System pages can be mapped into | |||
42 | * the aperture and look like contiguous pages from the GPU's | |||
43 | * perspective. A page table maps the pages in the aperture | |||
44 | * to the actual backing pages in system memory. | |||
45 | * | |||
46 | * Radeon GPUs support both an internal GART, as described above, | |||
47 | * and AGP. AGP works similarly, but the GART table is configured | |||
48 | * and maintained by the northbridge rather than the driver. | |||
49 | * Radeon hw has a separate AGP aperture that is programmed to | |||
50 | * point to the AGP aperture provided by the northbridge and the | |||
51 | * requests are passed through to the northbridge aperture. | |||
52 | * Both AGP and internal GART can be used at the same time, however | |||
53 | * that is not currently supported by the driver. | |||
54 | * | |||
55 | * This file handles the common internal GART management. | |||
56 | */ | |||
57 | ||||
58 | /* | |||
59 | * Common GART table functions. | |||
60 | */ | |||
61 | /** | |||
62 | * radeon_gart_table_ram_alloc - allocate system ram for gart page table | |||
63 | * | |||
64 | * @rdev: radeon_device pointer | |||
65 | * | |||
66 | * Allocate system memory for GART page table | |||
67 | * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the | |||
68 | * gart table to be in system memory. | |||
69 | * Returns 0 for success, -ENOMEM for failure. | |||
70 | */ | |||
71 | #ifdef __linux__ | |||
72 | int radeon_gart_table_ram_alloc(struct radeon_device *rdev) | |||
73 | { | |||
74 | void *ptr; | |||
75 | ||||
76 | ptr = dma_alloc_coherent(&rdev->pdev->dev, rdev->gart.table_size, | |||
77 | &rdev->gart.table_addr, GFP_KERNEL(0x0001 | 0x0004)); | |||
78 | if (ptr == NULL((void *)0)) { | |||
79 | return -ENOMEM12; | |||
80 | } | |||
81 | #ifdef CONFIG_X861 | |||
82 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |||
83 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |||
84 | set_memory_uc((unsigned long)ptr, | |||
85 | rdev->gart.table_size >> PAGE_SHIFT12); | |||
86 | } | |||
87 | #endif | |||
88 | rdev->gart.ptr = ptr; | |||
89 | return 0; | |||
90 | } | |||
91 | #else | |||
92 | int radeon_gart_table_ram_alloc(struct radeon_device *rdev) | |||
93 | { | |||
94 | struct drm_dmamem *dmah; | |||
95 | int flags = 0; | |||
96 | ||||
97 | #ifdef CONFIG_X861 | |||
98 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |||
99 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |||
100 | flags |= BUS_DMA_NOCACHE0x0800; | |||
101 | } | |||
102 | #endif | |||
103 | dmah = drm_dmamem_alloc(rdev->dmat, rdev->gart.table_size, | |||
104 | rdev->gart.table_size, 1, rdev->gart.table_size, flags, 0); | |||
105 | if (dmah == NULL((void *)0)) { | |||
106 | return -ENOMEM12; | |||
107 | } | |||
108 | rdev->gart.dmah = dmah; | |||
109 | rdev->gart.table_addr = dmah->map->dm_segs[0].ds_addr; | |||
110 | rdev->gart.ptr = dmah->kva; | |||
111 | memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size)__builtin_memset(((void *)rdev->gart.ptr), (0), (rdev-> gart.table_size)); | |||
112 | return 0; | |||
113 | } | |||
114 | #endif | |||
115 | ||||
116 | /** | |||
117 | * radeon_gart_table_ram_free - free system ram for gart page table | |||
118 | * | |||
119 | * @rdev: radeon_device pointer | |||
120 | * | |||
121 | * Free system memory for GART page table | |||
122 | * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the | |||
123 | * gart table to be in system memory. | |||
124 | */ | |||
125 | #ifdef __linux__ | |||
126 | void radeon_gart_table_ram_free(struct radeon_device *rdev) | |||
127 | { | |||
128 | if (rdev->gart.ptr == NULL((void *)0)) { | |||
129 | return; | |||
130 | } | |||
131 | #ifdef CONFIG_X861 | |||
132 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |||
133 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |||
134 | set_memory_wb((unsigned long)rdev->gart.ptr, | |||
135 | rdev->gart.table_size >> PAGE_SHIFT12); | |||
136 | } | |||
137 | #endif | |||
138 | dma_free_coherent(&rdev->pdev->dev, rdev->gart.table_size, | |||
139 | (void *)rdev->gart.ptr, rdev->gart.table_addr); | |||
140 | rdev->gart.ptr = NULL((void *)0); | |||
141 | rdev->gart.table_addr = 0; | |||
142 | } | |||
143 | #else | |||
144 | void radeon_gart_table_ram_free(struct radeon_device *rdev) | |||
145 | { | |||
146 | if (rdev->gart.ptr == NULL((void *)0)) { | |||
147 | return; | |||
148 | } | |||
149 | #if defined (CONFIG_X861) && defined(__linux__) | |||
150 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |||
151 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |||
152 | set_memory_wb((unsigned long)rdev->gart.ptr, | |||
153 | rdev->gart.table_size >> PAGE_SHIFT12); | |||
154 | } | |||
155 | #endif | |||
156 | drm_dmamem_free(rdev->dmat, rdev->gart.dmah); | |||
157 | rdev->gart.ptr = NULL((void *)0); | |||
158 | rdev->gart.table_addr = 0; | |||
159 | } | |||
160 | #endif | |||
161 | ||||
162 | /** | |||
163 | * radeon_gart_table_vram_alloc - allocate vram for gart page table | |||
164 | * | |||
165 | * @rdev: radeon_device pointer | |||
166 | * | |||
167 | * Allocate video memory for GART page table | |||
168 | * (pcie r4xx, r5xx+). These asics require the | |||
169 | * gart table to be in video memory. | |||
170 | * Returns 0 for success, error for failure. | |||
171 | */ | |||
172 | int radeon_gart_table_vram_alloc(struct radeon_device *rdev) | |||
173 | { | |||
174 | int r; | |||
175 | ||||
176 | if (rdev->gart.robj == NULL((void *)0)) { | |||
177 | r = radeon_bo_create(rdev, rdev->gart.table_size, | |||
178 | PAGE_SIZE(1 << 12), true1, RADEON_GEM_DOMAIN_VRAM0x4, | |||
179 | 0, NULL((void *)0), NULL((void *)0), &rdev->gart.robj); | |||
180 | if (r) { | |||
181 | return r; | |||
182 | } | |||
183 | } | |||
184 | return 0; | |||
185 | } | |||
186 | ||||
187 | /** | |||
188 | * radeon_gart_table_vram_pin - pin gart page table in vram | |||
189 | * | |||
190 | * @rdev: radeon_device pointer | |||
191 | * | |||
192 | * Pin the GART page table in vram so it will not be moved | |||
193 | * by the memory manager (pcie r4xx, r5xx+). These asics require the | |||
194 | * gart table to be in video memory. | |||
195 | * Returns 0 for success, error for failure. | |||
196 | */ | |||
197 | int radeon_gart_table_vram_pin(struct radeon_device *rdev) | |||
198 | { | |||
199 | uint64_t gpu_addr; | |||
200 | int r; | |||
201 | ||||
202 | r = radeon_bo_reserve(rdev->gart.robj, false0); | |||
203 | if (unlikely(r != 0)__builtin_expect(!!(r != 0), 0)) | |||
204 | return r; | |||
205 | r = radeon_bo_pin(rdev->gart.robj, | |||
206 | RADEON_GEM_DOMAIN_VRAM0x4, &gpu_addr); | |||
207 | if (r) { | |||
208 | radeon_bo_unreserve(rdev->gart.robj); | |||
209 | return r; | |||
210 | } | |||
211 | r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); | |||
212 | if (r) | |||
213 | radeon_bo_unpin(rdev->gart.robj); | |||
214 | radeon_bo_unreserve(rdev->gart.robj); | |||
215 | rdev->gart.table_addr = gpu_addr; | |||
216 | ||||
217 | if (!r) { | |||
218 | int i; | |||
219 | ||||
220 | /* We might have dropped some GART table updates while it wasn't | |||
221 | * mapped, restore all entries | |||
222 | */ | |||
223 | for (i = 0; i < rdev->gart.num_gpu_pages; i++) | |||
224 | radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i])(rdev)->asic->gart.set_page((rdev), (i), (rdev->gart .pages_entry[i])); | |||
225 | mb()do { __asm volatile("mfence" ::: "memory"); } while (0); | |||
226 | radeon_gart_tlb_flush(rdev)(rdev)->asic->gart.tlb_flush((rdev)); | |||
227 | } | |||
228 | ||||
229 | return r; | |||
230 | } | |||
231 | ||||
232 | /** | |||
233 | * radeon_gart_table_vram_unpin - unpin gart page table in vram | |||
234 | * | |||
235 | * @rdev: radeon_device pointer | |||
236 | * | |||
237 | * Unpin the GART page table in vram (pcie r4xx, r5xx+). | |||
238 | * These asics require the gart table to be in video memory. | |||
239 | */ | |||
240 | void radeon_gart_table_vram_unpin(struct radeon_device *rdev) | |||
241 | { | |||
242 | int r; | |||
243 | ||||
244 | if (rdev->gart.robj == NULL((void *)0)) { | |||
245 | return; | |||
246 | } | |||
247 | r = radeon_bo_reserve(rdev->gart.robj, false0); | |||
248 | if (likely(r == 0)__builtin_expect(!!(r == 0), 1)) { | |||
249 | radeon_bo_kunmap(rdev->gart.robj); | |||
250 | radeon_bo_unpin(rdev->gart.robj); | |||
251 | radeon_bo_unreserve(rdev->gart.robj); | |||
252 | rdev->gart.ptr = NULL((void *)0); | |||
253 | } | |||
254 | } | |||
255 | ||||
256 | /** | |||
257 | * radeon_gart_table_vram_free - free gart page table vram | |||
258 | * | |||
259 | * @rdev: radeon_device pointer | |||
260 | * | |||
261 | * Free the video memory used for the GART page table | |||
262 | * (pcie r4xx, r5xx+). These asics require the gart table to | |||
263 | * be in video memory. | |||
264 | */ | |||
265 | void radeon_gart_table_vram_free(struct radeon_device *rdev) | |||
266 | { | |||
267 | if (rdev->gart.robj == NULL((void *)0)) { | |||
268 | return; | |||
269 | } | |||
270 | radeon_bo_unref(&rdev->gart.robj); | |||
271 | } | |||
272 | ||||
273 | /* | |||
274 | * Common gart functions. | |||
275 | */ | |||
276 | /** | |||
277 | * radeon_gart_unbind - unbind pages from the gart page table | |||
278 | * | |||
279 | * @rdev: radeon_device pointer | |||
280 | * @offset: offset into the GPU's gart aperture | |||
281 | * @pages: number of pages to unbind | |||
282 | * | |||
283 | * Unbinds the requested pages from the gart page table and | |||
284 | * replaces them with the dummy page (all asics). | |||
285 | */ | |||
286 | void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, | |||
287 | int pages) | |||
288 | { | |||
289 | unsigned t; | |||
290 | unsigned p; | |||
291 | int i, j; | |||
292 | ||||
293 | if (!rdev->gart.ready
| |||
294 | WARN(1, "trying to unbind memory from uninitialized GART !\n")({ int __ret = !!(1); if (__ret) printf("trying to unbind memory from uninitialized GART !\n" ); __builtin_expect(!!(__ret), 0); }); | |||
295 | return; | |||
296 | } | |||
297 | t = offset / RADEON_GPU_PAGE_SIZE4096; | |||
298 | p = t / (PAGE_SIZE(1 << 12) / RADEON_GPU_PAGE_SIZE4096); | |||
299 | for (i = 0; i < pages; i++, p++) { | |||
300 | if (rdev->gart.pages[p]) { | |||
| ||||
301 | rdev->gart.pages[p] = NULL((void *)0); | |||
302 | for (j = 0; j < (PAGE_SIZE(1 << 12) / RADEON_GPU_PAGE_SIZE4096); j++, t++) { | |||
303 | rdev->gart.pages_entry[t] = rdev->dummy_page.entry; | |||
304 | if (rdev->gart.ptr) { | |||
305 | radeon_gart_set_page(rdev, t,(rdev)->asic->gart.set_page((rdev), (t), (rdev->dummy_page .entry)) | |||
306 | rdev->dummy_page.entry)(rdev)->asic->gart.set_page((rdev), (t), (rdev->dummy_page .entry)); | |||
307 | } | |||
308 | } | |||
309 | } | |||
310 | } | |||
311 | if (rdev->gart.ptr) { | |||
312 | mb()do { __asm volatile("mfence" ::: "memory"); } while (0); | |||
313 | radeon_gart_tlb_flush(rdev)(rdev)->asic->gart.tlb_flush((rdev)); | |||
314 | } | |||
315 | } | |||
316 | ||||
317 | /** | |||
318 | * radeon_gart_bind - bind pages into the gart page table | |||
319 | * | |||
320 | * @rdev: radeon_device pointer | |||
321 | * @offset: offset into the GPU's gart aperture | |||
322 | * @pages: number of pages to bind | |||
323 | * @pagelist: pages to bind | |||
324 | * @dma_addr: DMA addresses of pages | |||
325 | * @flags: RADEON_GART_PAGE_* flags | |||
326 | * | |||
327 | * Binds the requested pages to the gart page table | |||
328 | * (all asics). | |||
329 | * Returns 0 for success, -EINVAL for failure. | |||
330 | */ | |||
331 | int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, | |||
332 | int pages, struct vm_page **pagelist, dma_addr_t *dma_addr, | |||
333 | uint32_t flags) | |||
334 | { | |||
335 | unsigned t; | |||
336 | unsigned p; | |||
337 | uint64_t page_base, page_entry; | |||
338 | int i, j; | |||
339 | ||||
340 | if (!rdev->gart.ready) { | |||
341 | WARN(1, "trying to bind memory to uninitialized GART !\n")({ int __ret = !!(1); if (__ret) printf("trying to bind memory to uninitialized GART !\n" ); __builtin_expect(!!(__ret), 0); }); | |||
342 | return -EINVAL22; | |||
343 | } | |||
344 | t = offset / RADEON_GPU_PAGE_SIZE4096; | |||
345 | p = t / (PAGE_SIZE(1 << 12) / RADEON_GPU_PAGE_SIZE4096); | |||
346 | ||||
347 | for (i = 0; i < pages; i++, p++) { | |||
348 | rdev->gart.pages[p] = pagelist[i]; | |||
349 | page_base = dma_addr[i]; | |||
350 | for (j = 0; j < (PAGE_SIZE(1 << 12) / RADEON_GPU_PAGE_SIZE4096); j++, t++) { | |||
351 | page_entry = radeon_gart_get_page_entry(page_base, flags)(rdev)->asic->gart.get_page_entry((page_base), (flags)); | |||
352 | rdev->gart.pages_entry[t] = page_entry; | |||
353 | if (rdev->gart.ptr) { | |||
354 | radeon_gart_set_page(rdev, t, page_entry)(rdev)->asic->gart.set_page((rdev), (t), (page_entry)); | |||
355 | } | |||
356 | page_base += RADEON_GPU_PAGE_SIZE4096; | |||
357 | } | |||
358 | } | |||
359 | if (rdev->gart.ptr) { | |||
360 | mb()do { __asm volatile("mfence" ::: "memory"); } while (0); | |||
361 | radeon_gart_tlb_flush(rdev)(rdev)->asic->gart.tlb_flush((rdev)); | |||
362 | } | |||
363 | return 0; | |||
364 | } | |||
365 | ||||
366 | /** | |||
367 | * radeon_gart_init - init the driver info for managing the gart | |||
368 | * | |||
369 | * @rdev: radeon_device pointer | |||
370 | * | |||
371 | * Allocate the dummy page and init the gart driver info (all asics). | |||
372 | * Returns 0 for success, error for failure. | |||
373 | */ | |||
374 | int radeon_gart_init(struct radeon_device *rdev) | |||
375 | { | |||
376 | int r, i; | |||
377 | ||||
378 | if (rdev->gart.pages) { | |||
| ||||
379 | return 0; | |||
380 | } | |||
381 | /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ | |||
382 | if (PAGE_SIZE(1 << 12) < RADEON_GPU_PAGE_SIZE4096) { | |||
383 | DRM_ERROR("Page size is smaller than GPU page size!\n")__drm_err("Page size is smaller than GPU page size!\n"); | |||
384 | return -EINVAL22; | |||
385 | } | |||
386 | r = radeon_dummy_page_init(rdev); | |||
387 | if (r) | |||
388 | return r; | |||
389 | /* Compute table size */ | |||
390 | rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE(1 << 12); | |||
391 | rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE4096; | |||
392 | DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",printk("\0016" "[" "drm" "] " "GART: num cpu pages %u, num gpu pages %u\n" , rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages) | |||
393 | rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages)printk("\0016" "[" "drm" "] " "GART: num cpu pages %u, num gpu pages %u\n" , rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); | |||
394 | /* Allocate pages table */ | |||
395 | rdev->gart.pages = vzalloc(array_size(sizeof(void *),((sizeof(void *)) * (rdev->gart.num_cpu_pages)) | |||
396 | rdev->gart.num_cpu_pages)((sizeof(void *)) * (rdev->gart.num_cpu_pages))); | |||
397 | if (rdev->gart.pages == NULL((void *)0)) { | |||
398 | radeon_gart_fini(rdev); | |||
399 | return -ENOMEM12; | |||
400 | } | |||
401 | rdev->gart.pages_entry = vmalloc(array_size(sizeof(uint64_t),((sizeof(uint64_t)) * (rdev->gart.num_gpu_pages)) | |||
402 | rdev->gart.num_gpu_pages)((sizeof(uint64_t)) * (rdev->gart.num_gpu_pages))); | |||
403 | if (rdev->gart.pages_entry == NULL((void *)0)) { | |||
404 | radeon_gart_fini(rdev); | |||
405 | return -ENOMEM12; | |||
406 | } | |||
407 | /* set GART entry to point to the dummy page by default */ | |||
408 | for (i = 0; i < rdev->gart.num_gpu_pages; i++) | |||
409 | rdev->gart.pages_entry[i] = rdev->dummy_page.entry; | |||
410 | return 0; | |||
411 | } | |||
412 | ||||
413 | /** | |||
414 | * radeon_gart_fini - tear down the driver info for managing the gart | |||
415 | * | |||
416 | * @rdev: radeon_device pointer | |||
417 | * | |||
418 | * Tear down the gart driver info and free the dummy page (all asics). | |||
419 | */ | |||
420 | void radeon_gart_fini(struct radeon_device *rdev) | |||
421 | { | |||
422 | if (rdev->gart.ready) { | |||
423 | /* unbind pages */ | |||
424 | radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); | |||
425 | } | |||
426 | rdev->gart.ready = false0; | |||
427 | vfree(rdev->gart.pages); | |||
428 | vfree(rdev->gart.pages_entry); | |||
429 | rdev->gart.pages = NULL((void *)0); | |||
430 | rdev->gart.pages_entry = NULL((void *)0); | |||
431 | ||||
432 | radeon_dummy_page_fini(rdev); | |||
433 | } |