Commit f9af3c16 authored by Alex Sierra's avatar Alex Sierra Committed by Alex Deucher
Browse files

drm/amdkfd: track unified memory reservation with xnack off 



[WHY]
Unified memory with xnack off should be tracked, as userptr mappings
and legacy allocations do. To avoid oversuscribe system memory when
xnack off.
[How]
Exposing functions reserve_mem_limit and unreserve_mem_limit to SVM
API and call them on every prange creation and free.

Signed-off-by: default avatarAlex Sierra <alex.sierra@amd.com>
Reviewed-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent 06ac561f

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h

+4 −0
Original line number Diff line number Diff line
@@ -305,6 +305,10 @@ bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem * 
void amdgpu_amdkfd_block_mmu_notifications(void *p);

int amdgpu_amdkfd_criu_resume(void *p);

bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev);

int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,

		uint64_t size, u32 alloc_flag);

void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,

		uint64_t size, u32 alloc_flag);



#if IS_ENABLED(CONFIG_HSA_AMD)

void amdgpu_amdkfd_gpuvm_init_mem_limits(void);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c

+14 −9
Original line number Diff line number Diff line
@@ -129,7 +129,7 @@ void amdgpu_amdkfd_reserve_system_mem(uint64_t size) 
 *

 * Return: returns -ENOMEM in case of error, ZERO otherwise

 */

static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,

int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,

		uint64_t size, u32 alloc_flag)

{

	uint64_t reserved_for_pt =
@@ -169,7 +169,7 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 
	     kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) ||

	    (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >

	     kfd_mem_limit.max_ttm_mem_limit) ||

	    (adev->kfd.vram_used + vram_needed >

	    (adev && adev->kfd.vram_used + vram_needed >

	     adev->gmc.real_vram_size -

	     atomic64_read(&adev->vram_pin_size) -

	     reserved_for_pt)) {
@@ -180,6 +180,9 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 
	/* Update memory accounting by decreasing available system

	 * memory, TTM memory and GPU memory as computed above

	 */

	WARN_ONCE(vram_needed && !adev,

		  "adev reference can't be null when vram is used");

	if (adev)

		adev->kfd.vram_used += vram_needed;

	kfd_mem_limit.system_mem_used += system_mem_needed;

	kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
@@ -189,7 +192,7 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 
	return ret;

}



static void unreserve_mem_limit(struct amdgpu_device *adev,

void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,

		uint64_t size, u32 alloc_flag)

{

	spin_lock(&kfd_mem_limit.mem_limit_lock);
@@ -198,6 +201,9 @@ static void unreserve_mem_limit(struct amdgpu_device *adev, 
		kfd_mem_limit.system_mem_used -= size;

		kfd_mem_limit.ttm_mem_used -= size;

	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {

		WARN_ONCE(!adev,

			  "adev reference can't be null when alloc mem flags vram is set");

		if (adev)

			adev->kfd.vram_used -= ALIGN(size, VRAM_ALLOCATION_ALIGN);

	} else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {

		kfd_mem_limit.system_mem_used -= size;
@@ -207,8 +213,7 @@ static void unreserve_mem_limit(struct amdgpu_device *adev, 
		pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag);

		goto release;

	}



	WARN_ONCE(adev->kfd.vram_used < 0,

	WARN_ONCE(adev && adev->kfd.vram_used < 0,

		  "KFD VRAM memory accounting unbalanced");

	WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,

		  "KFD TTM memory accounting unbalanced");
@@ -225,7 +230,7 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) 
	u32 alloc_flags = bo->kfd_bo->alloc_flags;

	u64 size = amdgpu_bo_size(bo);



	unreserve_mem_limit(adev, size, alloc_flags);

	amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags);



	kfree(bo->kfd_bo);

}
@@ -1749,7 +1754,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( 
	/* Don't unreserve system mem limit twice */

	goto err_reserve_limit;

err_bo_create:

	unreserve_mem_limit(adev, size, flags);

	amdgpu_amdkfd_unreserve_mem_limit(adev, size, flags);

err_reserve_limit:

	mutex_destroy(&(*mem)->lock);

	if (gobj)

drivers/gpu/drm/amd/amdkfd/kfd_svm.c

+41 −17
Original line number Diff line number Diff line
@@ -266,13 +266,22 @@ void svm_range_free_dma_mappings(struct svm_range *prange) 
	}

}



static void svm_range_free(struct svm_range *prange)

static void svm_range_free(struct svm_range *prange, bool update_mem_usage)

{

	uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT;

	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);



	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,

		 prange->start, prange->last);



	svm_range_vram_node_free(prange);

	svm_range_free_dma_mappings(prange);



	if (update_mem_usage && !p->xnack_enabled) {

		pr_debug("unreserve mem limit: %lld\n", size);

		amdgpu_amdkfd_unreserve_mem_limit(NULL, size,

					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);

	}

	mutex_destroy(&prange->lock);

	mutex_destroy(&prange->migrate_mutex);

	kfree(prange);
@@ -291,7 +300,7 @@ svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc, 


static struct

svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,

			 uint64_t last)

			 uint64_t last, bool update_mem_usage)

{

	uint64_t size = last - start + 1;

	struct svm_range *prange;
@@ -300,6 +309,15 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, 
	prange = kzalloc(sizeof(*prange), GFP_KERNEL);

	if (!prange)

		return NULL;



	p = container_of(svms, struct kfd_process, svms);

	if (!p->xnack_enabled && update_mem_usage &&

	    amdgpu_amdkfd_reserve_mem_limit(NULL, size << PAGE_SHIFT,

					    KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) {

		pr_info("SVM mapping failed, exceeds resident system memory limit\n");

		kfree(prange);

		return NULL;

	}

	prange->npages = size;

	prange->svms = svms;

	prange->start = start;
@@ -314,7 +332,6 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, 
	mutex_init(&prange->migrate_mutex);

	mutex_init(&prange->lock);



	p = container_of(svms, struct kfd_process, svms);

	if (p->xnack_enabled)

		bitmap_copy(prange->bitmap_access, svms->bitmap_supported,

			    MAX_GPU_INSTANCE);
@@ -1007,9 +1024,9 @@ svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last, 


	svms = prange->svms;

	if (old_start == start)

		*new = svm_range_new(svms, last + 1, old_last);

		*new = svm_range_new(svms, last + 1, old_last, false);

	else

		*new = svm_range_new(svms, old_start, start - 1);

		*new = svm_range_new(svms, old_start, start - 1, false);

	if (!*new)

		return -ENOMEM;
@@ -1017,7 +1034,7 @@ svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last, 
	if (r) {

		pr_debug("failed %d split [0x%llx 0x%llx] to [0x%llx 0x%llx]\n",

			 r, old_start, old_last, start, last);

		svm_range_free(*new);

		svm_range_free(*new, false);

		*new = NULL;

	}
@@ -1852,7 +1869,7 @@ static struct svm_range *svm_range_clone(struct svm_range *old) 
{

	struct svm_range *new;



	new = svm_range_new(old->svms, old->start, old->last);

	new = svm_range_new(old->svms, old->start, old->last, false);

	if (!new)

		return NULL;
@@ -1905,7 +1922,7 @@ svm_range_split_new(struct svm_range_list *svms, uint64_t start, uint64_t last, 
	while (last >= start) {

		l = min(last, ALIGN_DOWN(start + max_pages, max_pages) - 1);



		prange = svm_range_new(svms, start, l);

		prange = svm_range_new(svms, start, l, true);

		if (!prange)

			return -ENOMEM;

		list_add(&prange->list, insert_list);
@@ -1956,6 +1973,7 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, 
	struct interval_tree_node *node;

	struct svm_range *prange;

	struct svm_range *tmp;

	struct list_head new_list;

	int r = 0;



	pr_debug("svms 0x%p [0x%llx 0x%lx]\n", &p->svms, start, last);
@@ -1963,6 +1981,7 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, 
	INIT_LIST_HEAD(update_list);

	INIT_LIST_HEAD(insert_list);

	INIT_LIST_HEAD(remove_list);

	INIT_LIST_HEAD(&new_list);



	node = interval_tree_iter_first(&svms->objects, start, last);

	while (node) {
@@ -2020,7 +2039,7 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, 
		if (node->start > start) {

			r = svm_range_split_new(svms, start, node->start - 1,

						READ_ONCE(max_svm_range_pages),

						insert_list, update_list);

						&new_list, update_list);

			if (r)

				goto out;

		}
@@ -2033,12 +2052,17 @@ svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, 
	if (start <= last)

		r = svm_range_split_new(svms, start, last,

					READ_ONCE(max_svm_range_pages),

					insert_list, update_list);

					&new_list, update_list);



out:

	if (r)

	if (r) {

		list_for_each_entry_safe(prange, tmp, insert_list, list)

			svm_range_free(prange);

			svm_range_free(prange, false);

		list_for_each_entry_safe(prange, tmp, &new_list, list)

			svm_range_free(prange, true);

	} else {

		list_splice(&new_list, insert_list);

	}



	return r;

}
@@ -2085,7 +2109,7 @@ svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange, 
			 svms, prange, prange->start, prange->last);

		svm_range_unlink(prange);

		svm_range_remove_notifier(prange);

		svm_range_free(prange);

		svm_range_free(prange, true);

		break;

	case SVM_OP_UPDATE_RANGE_NOTIFIER:

		pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n",
@@ -2648,14 +2672,14 @@ svm_range *svm_range_create_unregistered_range(struct amdgpu_device *adev, 
		last = addr;

	}



	prange = svm_range_new(&p->svms, start, last);

	prange = svm_range_new(&p->svms, start, last, true);

	if (!prange) {

		pr_debug("Failed to create prange in address [0x%llx]\n", addr);

		return NULL;

	}

	if (kfd_process_gpuid_from_adev(p, adev, &gpuid, &gpuidx)) {

		pr_debug("failed to get gpuid from kgd\n");

		svm_range_free(prange);

		svm_range_free(prange, true);

		return NULL;

	}
@@ -2955,7 +2979,7 @@ void svm_range_list_fini(struct kfd_process *p) 
	list_for_each_entry_safe(prange, next, &p->svms.list, list) {

		svm_range_unlink(prange);

		svm_range_remove_notifier(prange);

		svm_range_free(prange);

		svm_range_free(prange, true);

	}



	mutex_destroy(&p->svms.lock);
@@ -3371,7 +3395,7 @@ svm_range_set_attr(struct kfd_process *p, struct mm_struct *mm, 
			 prange->last);

		svm_range_unlink(prange);

		svm_range_remove_notifier(prange);

		svm_range_free(prange);

		svm_range_free(prange, false);

	}



	mmap_write_downgrade(mm);