Commit f0748674 authored by Tomer Tayar's avatar Tomer Tayar Committed by Oded Gabbay
Browse files

habanalabs: Modify the cs_cnt of a CB to be atomic 



Modify the CS counter of a CB to be atomic, so no locking is required
when it is being modified or read.

Signed-off-by: default avatarTomer Tayar <ttayar@habana.ai>
Reviewed-by: default avatarOded Gabbay <ogabbay@kernel.org>
Signed-off-by: default avatarOded Gabbay <ogabbay@kernel.org>
parent 3e438b42

drivers/misc/habanalabs/common/command_submission.c

+7 −22
Original line number Diff line number Diff line
@@ -233,10 +233,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job) 
			job->patched_cb = parser.patched_cb;

			job->job_cb_size = parser.patched_cb_size;

			job->contains_dma_pkt = parser.contains_dma_pkt;



			spin_lock(&job->patched_cb->lock);

			job->patched_cb->cs_cnt++;

			spin_unlock(&job->patched_cb->lock);

			atomic_inc(&job->patched_cb->cs_cnt);

		}



		/*
@@ -244,9 +241,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job) 
		 * original CB anymore because it was already parsed and

		 * won't be accessed again for this CS

		 */

		spin_lock(&job->user_cb->lock);

		job->user_cb->cs_cnt--;

		spin_unlock(&job->user_cb->lock);

		atomic_dec(&job->user_cb->cs_cnt);

		hl_cb_put(job->user_cb);

		job->user_cb = NULL;

	} else if (!rc) {
@@ -268,10 +263,7 @@ static void complete_job(struct hl_device *hdev, struct hl_cs_job *job) 
		 * created, so we need to check it's not NULL

		 */

		if (job->patched_cb) {

			spin_lock(&job->patched_cb->lock);

			job->patched_cb->cs_cnt--;

			spin_unlock(&job->patched_cb->lock);



			atomic_dec(&job->patched_cb->cs_cnt);

			hl_cb_put(job->patched_cb);

		}

	}
@@ -284,10 +276,7 @@ static void complete_job(struct hl_device *hdev, struct hl_cs_job *job) 
	if (job->is_kernel_allocated_cb &&

		((job->queue_type == QUEUE_TYPE_HW && hdev->mmu_enable) ||

				job->queue_type == QUEUE_TYPE_INT)) {

		spin_lock(&job->user_cb->lock);

		job->user_cb->cs_cnt--;

		spin_unlock(&job->user_cb->lock);



		atomic_dec(&job->user_cb->cs_cnt);

		hl_cb_put(job->user_cb);

	}
@@ -680,9 +669,7 @@ static struct hl_cb *get_cb_from_cs_chunk(struct hl_device *hdev, 
		goto release_cb;

	}



	spin_lock(&cb->lock);

	cb->cs_cnt++;

	spin_unlock(&cb->lock);

	atomic_inc(&cb->cs_cnt);



	return cb;
@@ -936,9 +923,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, 
	goto put_cs;



release_cb:

	spin_lock(&cb->lock);

	cb->cs_cnt--;

	spin_unlock(&cb->lock);

	atomic_dec(&cb->cs_cnt);

	hl_cb_put(cb);

free_cs_object:

	cs_rollback(hdev, cs);
@@ -1133,7 +1118,7 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, 
	job->id = 0;

	job->cs = cs;

	job->user_cb = cb;

	job->user_cb->cs_cnt++;

	atomic_inc(&job->user_cb->cs_cnt);

	job->user_cb_size = cb_size;

	job->hw_queue_id = q_idx;

drivers/misc/habanalabs/common/debugfs.c

+1 −1
Original line number Diff line number Diff line
@@ -116,7 +116,7 @@ static int command_buffers_show(struct seq_file *s, void *data) 
			"   %03llu        %d    0x%08x      %d          %d          %d\n",

			cb->id, cb->ctx->asid, cb->size,

			kref_read(&cb->refcount),

			cb->mmap, cb->cs_cnt);

			cb->mmap, atomic_read(&cb->cs_cnt));

	}



	spin_unlock(&dev_entry->cb_spinlock);

drivers/misc/habanalabs/common/habanalabs.h

+2 −2
Original line number Diff line number Diff line
@@ -532,7 +532,7 @@ struct hl_cb_mgr { 
 * @refcount: reference counter for usage of the CB.

 * @hdev: pointer to device this CB belongs to.

 * @ctx: pointer to the CB owner's context.

 * @lock: spinlock to protect mmap/cs flows.

 * @lock: spinlock to protect mmap flows.

 * @debugfs_list: node in debugfs list of command buffers.

 * @pool_list: node in pool list of command buffers.

 * @va_block_list: list of virtual addresses blocks of the CB if it is mapped to
@@ -561,7 +561,7 @@ struct hl_cb { 
	dma_addr_t		bus_address;

	u32			mmap_size;

	u32			size;

	u32			cs_cnt;

	atomic_t		cs_cnt;

	u8			mmap;

	u8			is_pool;

	u8			is_internal;

drivers/misc/habanalabs/gaudi/gaudi.c

+5 −5
Original line number Diff line number Diff line
@@ -838,7 +838,7 @@ static int _gaudi_init_tpc_mem(struct hl_device *hdev, 


	job->id = 0;

	job->user_cb = cb;

	job->user_cb->cs_cnt++;

	atomic_inc(&job->user_cb->cs_cnt);

	job->user_cb_size = cb_size;

	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;

	job->patched_cb = job->user_cb;
@@ -861,7 +861,7 @@ static int _gaudi_init_tpc_mem(struct hl_device *hdev, 
	hl_userptr_delete_list(hdev, &job->userptr_list);

	hl_debugfs_remove_job(hdev, job);

	kfree(job);

	cb->cs_cnt--;

	atomic_dec(&cb->cs_cnt);



release_cb:

	hl_cb_put(cb);
@@ -1248,7 +1248,7 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, 
	job->id = 0;

	job->cs = cs;

	job->user_cb = cb;

	job->user_cb->cs_cnt++;

	atomic_inc(&job->user_cb->cs_cnt);

	job->user_cb_size = cb_size;

	job->hw_queue_id = queue_id;
@@ -5570,7 +5570,7 @@ static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, 


	job->id = 0;

	job->user_cb = cb;

	job->user_cb->cs_cnt++;

	atomic_inc(&job->user_cb->cs_cnt);

	job->user_cb_size = cb_size;

	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;

	job->patched_cb = job->user_cb;
@@ -5581,7 +5581,7 @@ static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr, 
	rc = gaudi_send_job_on_qman0(hdev, job);

	hl_debugfs_remove_job(hdev, job);

	kfree(job);

	cb->cs_cnt--;

	atomic_dec(&cb->cs_cnt);



	/* Verify DMA is OK */

	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE);

drivers/misc/habanalabs/goya/goya.c

+2 −2
Original line number Diff line number Diff line
@@ -4811,7 +4811,7 @@ static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, 


	job->id = 0;

	job->user_cb = cb;

	job->user_cb->cs_cnt++;

	atomic_inc(&job->user_cb->cs_cnt);

	job->user_cb_size = cb_size;

	job->hw_queue_id = GOYA_QUEUE_ID_DMA_0;

	job->patched_cb = job->user_cb;
@@ -4823,7 +4823,7 @@ static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, 


	hl_debugfs_remove_job(hdev, job);

	kfree(job);

	cb->cs_cnt--;

	atomic_dec(&cb->cs_cnt);



release_cb:

	hl_cb_put(cb);