raspi/linux
1
0
Fork 1
mirror of https://github.com/raspberrypi/linux.git synced 2025-12-06 10:00:17 +00:00
Code Issues Projects Releases Wiki Activity

drm/amdgpu: rework how PTE flags are generated v3

Browse Source 
Previously we tried to keep the HW specific PTE flags in each mapping,
but for CRIU that isn't sufficient any more since the original value is
needed for the checkpoint procedure.

So rework the whole handling, nuke the early mapping function, keep the
UAPI flags in each mapping instead of the HW flags and translate them to
the HW flags while filling in the PTEs.

Only tested on Navi 23 for now, so probably needs quite a bit of more
work.

v2: fix KFD and SVN handling
v3: one more SVN fix pointed out by Felix
v4: squash in gfx12 fix from David

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
This commit is contained in:
Christian König
2025-06-06 14:13:37 +02:00
committed by Alex Deucher
parent 9f1f7cd467
commit 6716a823d1
16 changed files with 158 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
View File

@@ -494,7 +494,8 @@ static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL); return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL);
} }
static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) static uint64_t get_pte_flags(struct amdgpu_device *adev, struct amdgpu_vm *vm,
struct kgd_mem *mem)
{ {
uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE |
AMDGPU_VM_MTYPE_DEFAULT; AMDGPU_VM_MTYPE_DEFAULT;
@@ -504,7 +505,7 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE)
mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
return amdgpu_gem_va_map_flags(adev, mapping_flags); return mapping_flags;
} }
/** /**
@@ -961,7 +962,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
goto unwind; goto unwind;
} }
attachment[i]->va = va; attachment[i]->va = va;
attachment[i]->pte_flags = get_pte_flags(adev, mem); attachment[i]->pte_flags = get_pte_flags(adev, vm, mem);
attachment[i]->adev = adev; attachment[i]->adev = adev;
list_add(&attachment[i]->list, &mem->attachments); list_add(&attachment[i]->list, &mem->attachments);

37
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
View File

@@ -790,36 +790,6 @@ error:
return fence; return fence;
} }
/**
* amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags
*
* @adev: amdgpu_device pointer
* @flags: GEM UAPI flags
*
* Returns the GEM UAPI flags mapped into hardware for the ASIC.
*/
uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)
{
uint64_t pte_flag = 0;
if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
pte_flag |= AMDGPU_PTE_EXECUTABLE;
if (flags & AMDGPU_VM_PAGE_READABLE)
pte_flag |= AMDGPU_PTE_READABLE;
if (flags & AMDGPU_VM_PAGE_WRITEABLE)
pte_flag |= AMDGPU_PTE_WRITEABLE;
if (flags & AMDGPU_VM_PAGE_PRT)
pte_flag |= AMDGPU_PTE_PRT_FLAG(adev);
if (flags & AMDGPU_VM_PAGE_NOALLOC)
pte_flag |= AMDGPU_PTE_NOALLOC;
if (adev->gmc.gmc_funcs->map_mtype)
pte_flag |= amdgpu_gmc_map_mtype(adev,
flags & AMDGPU_VM_MTYPE_MASK);
return pte_flag;
}
int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp) struct drm_file *filp)
{ {
@@ -840,7 +810,6 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
struct dma_fence_chain *timeline_chain = NULL; struct dma_fence_chain *timeline_chain = NULL;
struct dma_fence *fence; struct dma_fence *fence;
struct drm_exec exec; struct drm_exec exec;
uint64_t va_flags;
uint64_t vm_size; uint64_t vm_size;
int r = 0; int r = 0;
@@ -944,10 +913,9 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
switch (args->operation) { switch (args->operation) {
case AMDGPU_VA_OP_MAP: case AMDGPU_VA_OP_MAP:
va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
r = amdgpu_vm_bo_map(adev, bo_va, args->va_address, r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
args->offset_in_bo, args->map_size, args->offset_in_bo, args->map_size,
va_flags); args->flags);
break; break;
case AMDGPU_VA_OP_UNMAP: case AMDGPU_VA_OP_UNMAP:
r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address); r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
@@ -959,10 +927,9 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
args->map_size); args->map_size);
break; break;
case AMDGPU_VA_OP_REPLACE: case AMDGPU_VA_OP_REPLACE:
va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address, r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
args->offset_in_bo, args->map_size, args->offset_in_bo, args->map_size,
va_flags); args->flags);
break; break;
default: default:
break; break;

1
drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
View File

@@ -63,7 +63,6 @@ int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp); struct drm_file *filp);
int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data, int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp); struct drm_file *filp);
uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags);
int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp); struct drm_file *filp);
int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,

15
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
View File

@@ -154,15 +154,15 @@ struct amdgpu_gmc_funcs {
unsigned pasid); unsigned pasid);
/* enable/disable PRT support */ /* enable/disable PRT support */
void (*set_prt)(struct amdgpu_device *adev, bool enable); void (*set_prt)(struct amdgpu_device *adev, bool enable);
/* map mtype to hardware flags */
uint64_t (*map_mtype)(struct amdgpu_device *adev, uint32_t flags);
/* get the pde for a given mc addr */ /* get the pde for a given mc addr */
void (*get_vm_pde)(struct amdgpu_device *adev, int level, void (*get_vm_pde)(struct amdgpu_device *adev, int level,
u64 *dst, u64 *flags); u64 *dst, u64 *flags);
/* get the pte flags to use for a BO VA mapping */ /* get the pte flags to use for PTEs */
void (*get_vm_pte)(struct amdgpu_device *adev, void (*get_vm_pte)(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
uint64_t *flags); struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *pte_flags);
/* override per-page pte flags */ /* override per-page pte flags */
void (*override_vm_pte_flags)(struct amdgpu_device *dev, void (*override_vm_pte_flags)(struct amdgpu_device *dev,
struct amdgpu_vm *vm, struct amdgpu_vm *vm,
@@ -356,9 +356,10 @@ struct amdgpu_gmc {
#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr)) #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid)) #define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
#define amdgpu_gmc_map_mtype(adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags))
#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags)) #define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
#define amdgpu_gmc_get_vm_pte(adev, mapping, flags) (adev)->gmc.gmc_funcs->get_vm_pte((adev), (mapping), (flags)) #define amdgpu_gmc_get_vm_pte(adev, vm, bo, vm_flags, pte_flags) \
((adev)->gmc.gmc_funcs->get_vm_pte((adev), (vm), (bo), (vm_flags), \
(pte_flags)))
#define amdgpu_gmc_override_vm_pte_flags(adev, vm, addr, pte_flags) \ #define amdgpu_gmc_override_vm_pte_flags(adev, vm, addr, pte_flags) \
(adev)->gmc.gmc_funcs->override_vm_pte_flags \ (adev)->gmc.gmc_funcs->override_vm_pte_flags \
((adev), (vm), (addr), (pte_flags)) ((adev), (vm), (addr), (pte_flags))

2
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
View File

@@ -69,7 +69,7 @@ struct amdgpu_bo_va_mapping {
uint64_t last; uint64_t last;
uint64_t __subtree_last; uint64_t __subtree_last;
uint64_t offset; uint64_t offset;
uint64_t flags; uint32_t flags;
}; };
/* User space allocated BO in a VM */ /* User space allocated BO in a VM */

8
drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
View File

@@ -67,9 +67,9 @@ static inline u64 amdgpu_seq64_get_va_base(struct amdgpu_device *adev)
int amdgpu_seq64_map(struct amdgpu_device *adev, struct amdgpu_vm *vm, int amdgpu_seq64_map(struct amdgpu_device *adev, struct amdgpu_vm *vm,
struct amdgpu_bo_va **bo_va) struct amdgpu_bo_va **bo_va)
{ {
u64 seq64_addr, va_flags;
struct amdgpu_bo *bo; struct amdgpu_bo *bo;
struct drm_exec exec; struct drm_exec exec;
u64 seq64_addr;
int r; int r;
bo = adev->seq64.sbo; bo = adev->seq64.sbo;
@@ -94,9 +94,9 @@ int amdgpu_seq64_map(struct amdgpu_device *adev, struct amdgpu_vm *vm,
seq64_addr = amdgpu_seq64_get_va_base(adev) & AMDGPU_GMC_HOLE_MASK; seq64_addr = amdgpu_seq64_get_va_base(adev) & AMDGPU_GMC_HOLE_MASK;
va_flags = amdgpu_gem_va_map_flags(adev, AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_MTYPE_UC); r = amdgpu_vm_bo_map(adev, *bo_va, seq64_addr, 0,
r = amdgpu_vm_bo_map(adev, *bo_va, seq64_addr, 0, AMDGPU_VA_RESERVED_SEQ64_SIZE, AMDGPU_VA_RESERVED_SEQ64_SIZE,
va_flags); AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_MTYPE_UC);
if (r) { if (r) {
DRM_ERROR("failed to do bo_map on userq sem, err=%d\n", r); DRM_ERROR("failed to do bo_map on userq sem, err=%d\n", r);
amdgpu_vm_bo_del(adev, *bo_va); amdgpu_vm_bo_del(adev, *bo_va);

17
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
View File

@@ -1332,13 +1332,14 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
* but in case of something, we filter the flags in first place * but in case of something, we filter the flags in first place
*/ */
if (!(mapping->flags & AMDGPU_PTE_READABLE)) if (!(mapping->flags & AMDGPU_VM_PAGE_READABLE))
update_flags &= ~AMDGPU_PTE_READABLE; update_flags &= ~AMDGPU_PTE_READABLE;
if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) if (!(mapping->flags & AMDGPU_VM_PAGE_WRITEABLE))
update_flags &= ~AMDGPU_PTE_WRITEABLE; update_flags &= ~AMDGPU_PTE_WRITEABLE;
/* Apply ASIC specific mapping flags */ /* Apply ASIC specific mapping flags */
amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags); amdgpu_gmc_get_vm_pte(adev, vm, bo, mapping->flags,
&update_flags);
trace_amdgpu_vm_bo_update(mapping); trace_amdgpu_vm_bo_update(mapping);
@@ -1479,7 +1480,7 @@ static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_bo_va_mapping *mapping,
struct dma_fence *fence) struct dma_fence *fence)
{ {
if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev)) if (mapping->flags & AMDGPU_VM_PAGE_PRT)
amdgpu_vm_add_prt_cb(adev, fence); amdgpu_vm_add_prt_cb(adev, fence);
kfree(mapping); kfree(mapping);
} }
@@ -1758,7 +1759,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
list_add(&mapping->list, &bo_va->invalids); list_add(&mapping->list, &bo_va->invalids);
amdgpu_vm_it_insert(mapping, &vm->va); amdgpu_vm_it_insert(mapping, &vm->va);
if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev)) if (mapping->flags & AMDGPU_VM_PAGE_PRT)
amdgpu_vm_prt_get(adev); amdgpu_vm_prt_get(adev);
if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved) if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved)
@@ -1818,7 +1819,7 @@ static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,
int amdgpu_vm_bo_map(struct amdgpu_device *adev, int amdgpu_vm_bo_map(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va, struct amdgpu_bo_va *bo_va,
uint64_t saddr, uint64_t offset, uint64_t saddr, uint64_t offset,
uint64_t size, uint64_t flags) uint64_t size, uint32_t flags)
{ {
struct amdgpu_bo_va_mapping *mapping, *tmp; struct amdgpu_bo_va_mapping *mapping, *tmp;
struct amdgpu_bo *bo = bo_va->base.bo; struct amdgpu_bo *bo = bo_va->base.bo;
@@ -1877,7 +1878,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va, struct amdgpu_bo_va *bo_va,
uint64_t saddr, uint64_t offset, uint64_t saddr, uint64_t offset,
uint64_t size, uint64_t flags) uint64_t size, uint32_t flags)
{ {
struct amdgpu_bo_va_mapping *mapping; struct amdgpu_bo_va_mapping *mapping;
struct amdgpu_bo *bo = bo_va->base.bo; struct amdgpu_bo *bo = bo_va->base.bo;
@@ -2734,7 +2735,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
dma_fence_put(vm->last_tlb_flush); dma_fence_put(vm->last_tlb_flush);
list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev) && prt_fini_needed) { if (mapping->flags & AMDGPU_VM_PAGE_PRT && prt_fini_needed) {
amdgpu_vm_prt_fini(adev, vm); amdgpu_vm_prt_fini(adev, vm);
prt_fini_needed = false; prt_fini_needed = false;
} }

4
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
View File

@@ -538,11 +538,11 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
int amdgpu_vm_bo_map(struct amdgpu_device *adev, int amdgpu_vm_bo_map(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va, struct amdgpu_bo_va *bo_va,
uint64_t addr, uint64_t offset, uint64_t addr, uint64_t offset,
uint64_t size, uint64_t flags); uint64_t size, uint32_t flags);
int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va, struct amdgpu_bo_va *bo_va,
uint64_t addr, uint64_t offset, uint64_t addr, uint64_t offset,
uint64_t size, uint64_t flags); uint64_t size, uint32_t flags);
int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
struct amdgpu_bo_va *bo_va, struct amdgpu_bo_va *bo_va,
uint64_t addr); uint64_t addr);

57
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
View File

@@ -466,24 +466,6 @@ static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int
* 0 valid * 0 valid
*/ */
static uint64_t gmc_v10_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
{
switch (flags) {
case AMDGPU_VM_MTYPE_DEFAULT:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_NC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_WC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_WC);
case AMDGPU_VM_MTYPE_CC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_CC);
case AMDGPU_VM_MTYPE_UC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC);
default:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);
}
}
static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level, static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,
uint64_t *addr, uint64_t *flags) uint64_t *addr, uint64_t *flags)
{ {
@@ -508,21 +490,39 @@ static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,
} }
static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
struct amdgpu_bo *bo = mapping->bo_va->base.bo; if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
*flags |= AMDGPU_PTE_EXECUTABLE;
else
*flags &= ~AMDGPU_PTE_EXECUTABLE;
*flags &= ~AMDGPU_PTE_EXECUTABLE; switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; case AMDGPU_VM_MTYPE_DEFAULT:
case AMDGPU_VM_MTYPE_NC:
default:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_WC:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_WC);
break;
case AMDGPU_VM_MTYPE_CC:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_CC);
break;
case AMDGPU_VM_MTYPE_UC:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
break;
}
*flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK; if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
*flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK); *flags |= AMDGPU_PTE_NOALLOC;
else
*flags &= ~AMDGPU_PTE_NOALLOC;
*flags &= ~AMDGPU_PTE_NOALLOC; if (vm_flags & AMDGPU_VM_PAGE_PRT) {
*flags |= (mapping->flags & AMDGPU_PTE_NOALLOC);
if (mapping->flags & AMDGPU_PTE_PRT) {
*flags |= AMDGPU_PTE_PRT; *flags |= AMDGPU_PTE_PRT;
*flags |= AMDGPU_PTE_SNOOPED; *flags |= AMDGPU_PTE_SNOOPED;
*flags |= AMDGPU_PTE_LOG; *flags |= AMDGPU_PTE_LOG;
@@ -563,7 +563,6 @@ static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = {
.flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid, .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid,
.emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb, .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb,
.emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping, .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping,
.map_mtype = gmc_v10_0_map_mtype,
.get_vm_pde = gmc_v10_0_get_vm_pde, .get_vm_pde = gmc_v10_0_get_vm_pde,
.get_vm_pte = gmc_v10_0_get_vm_pte, .get_vm_pte = gmc_v10_0_get_vm_pte,
.get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size, .get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size,

57
drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
View File

@@ -430,24 +430,6 @@ static void gmc_v11_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int
* 0 valid * 0 valid
*/ */
static uint64_t gmc_v11_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
{
switch (flags) {
case AMDGPU_VM_MTYPE_DEFAULT:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_NC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_WC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_WC);
case AMDGPU_VM_MTYPE_CC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_CC);
case AMDGPU_VM_MTYPE_UC:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC);
default:
return AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_NC);
}
}
static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level, static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level,
uint64_t *addr, uint64_t *flags) uint64_t *addr, uint64_t *flags)
{ {
@@ -472,21 +454,39 @@ static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level,
} }
static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
struct amdgpu_bo *bo = mapping->bo_va->base.bo; if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
*flags |= AMDGPU_PTE_EXECUTABLE;
else
*flags &= ~AMDGPU_PTE_EXECUTABLE;
*flags &= ~AMDGPU_PTE_EXECUTABLE; switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; case AMDGPU_VM_MTYPE_DEFAULT:
case AMDGPU_VM_MTYPE_NC:
default:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_WC:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_WC);
break;
case AMDGPU_VM_MTYPE_CC:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_CC);
break;
case AMDGPU_VM_MTYPE_UC:
*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
break;
}
*flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK; if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
*flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK); *flags |= AMDGPU_PTE_NOALLOC;
else
*flags &= ~AMDGPU_PTE_NOALLOC;
*flags &= ~AMDGPU_PTE_NOALLOC; if (vm_flags & AMDGPU_VM_PAGE_PRT) {
*flags |= (mapping->flags & AMDGPU_PTE_NOALLOC);
if (mapping->flags & AMDGPU_PTE_PRT) {
*flags |= AMDGPU_PTE_PRT; *flags |= AMDGPU_PTE_PRT;
*flags |= AMDGPU_PTE_SNOOPED; *flags |= AMDGPU_PTE_SNOOPED;
*flags |= AMDGPU_PTE_LOG; *flags |= AMDGPU_PTE_LOG;
@@ -527,7 +527,6 @@ static const struct amdgpu_gmc_funcs gmc_v11_0_gmc_funcs = {
.flush_gpu_tlb_pasid = gmc_v11_0_flush_gpu_tlb_pasid, .flush_gpu_tlb_pasid = gmc_v11_0_flush_gpu_tlb_pasid,
.emit_flush_gpu_tlb = gmc_v11_0_emit_flush_gpu_tlb, .emit_flush_gpu_tlb = gmc_v11_0_emit_flush_gpu_tlb,
.emit_pasid_mapping = gmc_v11_0_emit_pasid_mapping, .emit_pasid_mapping = gmc_v11_0_emit_pasid_mapping,
.map_mtype = gmc_v11_0_map_mtype,
.get_vm_pde = gmc_v11_0_get_vm_pde, .get_vm_pde = gmc_v11_0_get_vm_pde,
.get_vm_pte = gmc_v11_0_get_vm_pte, .get_vm_pte = gmc_v11_0_get_vm_pte,
.get_vbios_fb_size = gmc_v11_0_get_vbios_fb_size, .get_vbios_fb_size = gmc_v11_0_get_vbios_fb_size,

47
drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
View File

@@ -453,20 +453,6 @@ static void gmc_v12_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid
* 0 valid * 0 valid
*/ */
static uint64_t gmc_v12_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
{
switch (flags) {
case AMDGPU_VM_MTYPE_DEFAULT:
return AMDGPU_PTE_MTYPE_GFX12(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_NC:
return AMDGPU_PTE_MTYPE_GFX12(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_UC:
return AMDGPU_PTE_MTYPE_GFX12(0ULL, MTYPE_UC);
default:
return AMDGPU_PTE_MTYPE_GFX12(0ULL, MTYPE_NC);
}
}
static void gmc_v12_0_get_vm_pde(struct amdgpu_device *adev, int level, static void gmc_v12_0_get_vm_pde(struct amdgpu_device *adev, int level,
uint64_t *addr, uint64_t *flags) uint64_t *addr, uint64_t *flags)
{ {
@@ -490,19 +476,35 @@ static void gmc_v12_0_get_vm_pde(struct amdgpu_device *adev, int level,
} }
static void gmc_v12_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v12_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
struct amdgpu_bo *bo = mapping->bo_va->base.bo; if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
*flags |= AMDGPU_PTE_EXECUTABLE;
else
*flags &= ~AMDGPU_PTE_EXECUTABLE;
*flags &= ~AMDGPU_PTE_EXECUTABLE; switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; case AMDGPU_VM_MTYPE_DEFAULT:
*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_NC:
default:
*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_UC:
*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
break;
}
*flags &= ~AMDGPU_PTE_MTYPE_GFX12_MASK; if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
*flags |= (mapping->flags & AMDGPU_PTE_MTYPE_GFX12_MASK); *flags |= AMDGPU_PTE_NOALLOC;
else
*flags &= ~AMDGPU_PTE_NOALLOC;
if (mapping->flags & AMDGPU_PTE_PRT_GFX12) { if (vm_flags & AMDGPU_VM_PAGE_PRT) {
*flags |= AMDGPU_PTE_PRT_GFX12;
*flags |= AMDGPU_PTE_SNOOPED; *flags |= AMDGPU_PTE_SNOOPED;
*flags |= AMDGPU_PTE_SYSTEM; *flags |= AMDGPU_PTE_SYSTEM;
*flags |= AMDGPU_PTE_IS_PTE; *flags |= AMDGPU_PTE_IS_PTE;
@@ -543,7 +545,6 @@ static const struct amdgpu_gmc_funcs gmc_v12_0_gmc_funcs = {
.flush_gpu_tlb_pasid = gmc_v12_0_flush_gpu_tlb_pasid, .flush_gpu_tlb_pasid = gmc_v12_0_flush_gpu_tlb_pasid,
.emit_flush_gpu_tlb = gmc_v12_0_emit_flush_gpu_tlb, .emit_flush_gpu_tlb = gmc_v12_0_emit_flush_gpu_tlb,
.emit_pasid_mapping = gmc_v12_0_emit_pasid_mapping, .emit_pasid_mapping = gmc_v12_0_emit_pasid_mapping,
.map_mtype = gmc_v12_0_map_mtype,
.get_vm_pde = gmc_v12_0_get_vm_pde, .get_vm_pde = gmc_v12_0_get_vm_pde,
.get_vm_pte = gmc_v12_0_get_vm_pte, .get_vm_pte = gmc_v12_0_get_vm_pte,
.get_vbios_fb_size = gmc_v12_0_get_vbios_fb_size, .get_vbios_fb_size = gmc_v12_0_get_vbios_fb_size,

4
drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
View File

@@ -382,7 +382,9 @@ static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level,
} }
static void gmc_v6_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v6_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
*flags &= ~AMDGPU_PTE_EXECUTABLE; *flags &= ~AMDGPU_PTE_EXECUTABLE;

4
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
View File

@@ -504,7 +504,9 @@ static void gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, int level,
} }
static void gmc_v7_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v7_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
*flags &= ~AMDGPU_PTE_EXECUTABLE; *flags &= ~AMDGPU_PTE_EXECUTABLE;

10
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
View File

@@ -716,11 +716,15 @@ static void gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, int level,
} }
static void gmc_v8_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v8_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
*flags &= ~AMDGPU_PTE_EXECUTABLE; if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; *flags |= AMDGPU_PTE_EXECUTABLE;
else
*flags &= ~AMDGPU_PTE_EXECUTABLE;
*flags &= ~AMDGPU_PTE_PRT; *flags &= ~AMDGPU_PTE_PRT;
} }

61
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
View File

@@ -1073,27 +1073,6 @@ static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int v
* 0 valid * 0 valid
*/ */
static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
{
switch (flags) {
case AMDGPU_VM_MTYPE_DEFAULT:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_NC:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC);
case AMDGPU_VM_MTYPE_WC:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_WC);
case AMDGPU_VM_MTYPE_RW:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_RW);
case AMDGPU_VM_MTYPE_CC:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_CC);
case AMDGPU_VM_MTYPE_UC:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_UC);
default:
return AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC);
}
}
static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
uint64_t *addr, uint64_t *flags) uint64_t *addr, uint64_t *flags)
{ {
@@ -1123,6 +1102,7 @@ static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev, static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
struct amdgpu_vm *vm, struct amdgpu_vm *vm,
struct amdgpu_bo *bo, struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
@@ -1236,25 +1216,43 @@ static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
} }
static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
struct amdgpu_bo_va_mapping *mapping, struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
uint32_t vm_flags,
uint64_t *flags) uint64_t *flags)
{ {
struct amdgpu_bo *bo = mapping->bo_va->base.bo; if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
*flags |= AMDGPU_PTE_EXECUTABLE;
else
*flags &= ~AMDGPU_PTE_EXECUTABLE;
*flags &= ~AMDGPU_PTE_EXECUTABLE; switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; case AMDGPU_VM_MTYPE_DEFAULT:
case AMDGPU_VM_MTYPE_NC:
default:
*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_NC);
break;
case AMDGPU_VM_MTYPE_WC:
*flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_WC);
break;
case AMDGPU_VM_MTYPE_RW:
*flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_RW);
break;
case AMDGPU_VM_MTYPE_CC:
*flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC);
break;
case AMDGPU_VM_MTYPE_UC:
*flags |= AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_UC);
break;
}
*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; if (vm_flags & AMDGPU_VM_PAGE_PRT) {
*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
if (mapping->flags & AMDGPU_PTE_PRT) {
*flags |= AMDGPU_PTE_PRT; *flags |= AMDGPU_PTE_PRT;
*flags &= ~AMDGPU_PTE_VALID; *flags &= ~AMDGPU_PTE_VALID;
} }
if ((*flags & AMDGPU_PTE_VALID) && bo) if ((*flags & AMDGPU_PTE_VALID) && bo)
gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.vm, bo, gmc_v9_0_get_coherence_flags(adev, vm, bo, vm_flags, flags);
flags);
} }
static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev, static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
@@ -1391,7 +1389,6 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
.map_mtype = gmc_v9_0_map_mtype,
.get_vm_pde = gmc_v9_0_get_vm_pde, .get_vm_pde = gmc_v9_0_get_vm_pde,
.get_vm_pte = gmc_v9_0_get_vm_pte, .get_vm_pte = gmc_v9_0_get_vm_pte,
.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags, .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,

13
drivers/gpu/drm/amd/amdkfd/kfd_svm.c
View File

@@ -1189,7 +1189,7 @@ svm_nodes_in_same_hive(struct kfd_node *node_a, struct kfd_node *node_b)
} }
static uint64_t static uint64_t
svm_range_get_pte_flags(struct kfd_node *node, svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
struct svm_range *prange, int domain) struct svm_range *prange, int domain)
{ {
struct kfd_node *bo_node; struct kfd_node *bo_node;
@@ -1292,10 +1292,6 @@ svm_range_get_pte_flags(struct kfd_node *node,
AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
} }
mapping_flags |= AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;
if (flags & KFD_IOCTL_SVM_FLAG_GPU_RO)
mapping_flags &= ~AMDGPU_VM_PAGE_WRITEABLE;
if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC) if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC)
mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
@@ -1305,7 +1301,10 @@ svm_range_get_pte_flags(struct kfd_node *node,
if (gc_ip_version >= IP_VERSION(12, 0, 0)) if (gc_ip_version >= IP_VERSION(12, 0, 0))
pte_flags |= AMDGPU_PTE_IS_PTE; pte_flags |= AMDGPU_PTE_IS_PTE;
pte_flags |= amdgpu_gem_va_map_flags(node->adev, mapping_flags); amdgpu_gmc_get_vm_pte(node->adev, vm, NULL, mapping_flags, &pte_flags);
pte_flags |= AMDGPU_PTE_READABLE;
if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
pte_flags |= AMDGPU_PTE_WRITEABLE;
return pte_flags; return pte_flags;
} }
@@ -1412,7 +1411,7 @@ svm_range_map_to_gpu(struct kfd_process_device *pdd, struct svm_range *prange,
pr_debug("Mapping range [0x%lx 0x%llx] on domain: %s\n", pr_debug("Mapping range [0x%lx 0x%llx] on domain: %s\n",
last_start, prange->start + i, last_domain ? "GPU" : "CPU"); last_start, prange->start + i, last_domain ? "GPU" : "CPU");
pte_flags = svm_range_get_pte_flags(pdd->dev, prange, last_domain); pte_flags = svm_range_get_pte_flags(pdd->dev, vm, prange, last_domain);
if (readonly) if (readonly)
pte_flags &= ~AMDGPU_PTE_WRITEABLE; pte_flags &= ~AMDGPU_PTE_WRITEABLE;