Commit 66a76401 authored by Ofir Bitton's avatar Ofir Bitton Committed by Oded Gabbay
Browse files

habanalabs: add 'needs reset' state in driver 



The new state indicates that device should be reset in order
to re-gain funcionality.
This unique state can occur if reset_on_lockup is disabled
and an actual lockup has occurred.

Signed-off-by: default avatarOfir Bitton <obitton@habana.ai>
Reviewed-by: default avatarOded Gabbay <ogabbay@kernel.org>
Signed-off-by: default avatarOded Gabbay <ogabbay@kernel.org>
parent f2d032ee

drivers/misc/habanalabs/common/command_buffer.c

+3 −2
Original line number Diff line number Diff line
@@ -379,13 +379,14 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) 
{

	union hl_cb_args *args = data;

	struct hl_device *hdev = hpriv->hdev;

	enum hl_device_status status;

	u64 handle = 0;

	int rc;



	if (hl_device_disabled_or_in_reset(hdev)) {

	if (!hl_device_operational(hdev, &status)) {

		dev_warn_ratelimited(hdev->dev,

			"Device is %s. Can't execute CB IOCTL\n",

			atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");

			hdev->status[status]);

		return -EBUSY;

	}

drivers/misc/habanalabs/common/command_submission.c

+5 −2
Original line number Diff line number Diff line
@@ -427,6 +427,8 @@ static void cs_timedout(struct work_struct *work) 


	if (hdev->reset_on_lockup)

		hl_device_reset(hdev, false, false);

	else

		hdev->needs_reset = true;

}



static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
@@ -689,12 +691,13 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) 
	struct hl_device *hdev = hpriv->hdev;

	struct hl_ctx *ctx = hpriv->ctx;

	u32 cs_type_flags, num_chunks;

	enum hl_device_status status;

	enum hl_cs_type cs_type;



	if (hl_device_disabled_or_in_reset(hdev)) {

	if (!hl_device_operational(hdev, &status)) {

		dev_warn_ratelimited(hdev->dev,

			"Device is %s. Can't submit new CS\n",

			atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");

			hdev->status[status]);

		return -EBUSY;

	}

drivers/misc/habanalabs/common/debugfs.c

+3 −3
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr, 
	struct cpucp_packet pkt;

	int rc;



	if (hl_device_disabled_or_in_reset(hdev))

	if (!hl_device_operational(hdev, NULL))

		return -EBUSY;



	memset(&pkt, 0, sizeof(pkt));
@@ -50,7 +50,7 @@ static int hl_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr, 
	struct cpucp_packet pkt;

	int rc;



	if (hl_device_disabled_or_in_reset(hdev))

	if (!hl_device_operational(hdev, NULL))

		return -EBUSY;



	memset(&pkt, 0, sizeof(pkt));
@@ -76,7 +76,7 @@ static void hl_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state) 
	struct cpucp_packet pkt;

	int rc;



	if (hl_device_disabled_or_in_reset(hdev))

	if (!hl_device_operational(hdev, NULL))

		return;



	memset(&pkt, 0, sizeof(pkt));

drivers/misc/habanalabs/common/device.c

+24 −9
Original line number Diff line number Diff line
@@ -15,14 +15,6 @@ 


#define HL_PLDM_PENDING_RESET_PER_SEC	(HL_PENDING_RESET_PER_SEC * 10)



bool hl_device_disabled_or_in_reset(struct hl_device *hdev)

{

	if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))

		return true;

	else

		return false;

}



enum hl_device_status hl_device_status(struct hl_device *hdev)

{

	enum hl_device_status status;
@@ -31,12 +23,34 @@ enum hl_device_status hl_device_status(struct hl_device *hdev) 
		status = HL_DEVICE_STATUS_MALFUNCTION;

	else if (atomic_read(&hdev->in_reset))

		status = HL_DEVICE_STATUS_IN_RESET;

	else if (hdev->needs_reset)

		status = HL_DEVICE_STATUS_NEEDS_RESET;

	else

		status = HL_DEVICE_STATUS_OPERATIONAL;



	return status;

}



bool hl_device_operational(struct hl_device *hdev,

		enum hl_device_status *status)

{

	enum hl_device_status current_status;



	current_status = hl_device_status(hdev);

	if (status)

		*status = current_status;



	switch (current_status) {

	case HL_DEVICE_STATUS_IN_RESET:

	case HL_DEVICE_STATUS_MALFUNCTION:

	case HL_DEVICE_STATUS_NEEDS_RESET:

		return false;

	case HL_DEVICE_STATUS_OPERATIONAL:

	default:

		return true;

	}

}



static void hpriv_release(struct kref *ref)

{

	struct hl_fpriv *hpriv;
@@ -411,7 +425,7 @@ static void hl_device_heartbeat(struct work_struct *work) 
	struct hl_device *hdev = container_of(work, struct hl_device,

						work_heartbeat.work);



	if (hl_device_disabled_or_in_reset(hdev))

	if (!hl_device_operational(hdev, NULL))

		goto reschedule;



	if (!hdev->asic_funcs->send_heartbeat(hdev))
@@ -1091,6 +1105,7 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset, 
	}



	atomic_set(&hdev->in_reset, 0);

	hdev->needs_reset = false;



	if (hard_reset)

		hdev->hard_reset_cnt++;

drivers/misc/habanalabs/common/habanalabs.h

+12 −2
Original line number Diff line number Diff line
@@ -1432,6 +1432,10 @@ struct hl_dbg_device_entry { 
 * DEVICES

 */



#define HL_STR_MAX	32



#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_NEEDS_RESET + 1)



/* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe

 * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.

 */
@@ -1706,6 +1710,7 @@ struct hl_mmu_funcs { 
 * @hwmon_dev: H/W monitor device.

 * @pm_mng_profile: current power management profile.

 * @hl_chip_info: ASIC's sensors information.

 * @device_status_description: device status description.

 * @hl_debugfs: device's debugfs manager.

 * @cb_pool: list of preallocated CBs.

 * @cb_pool_lock: protects the CB pool.
@@ -1774,6 +1779,8 @@ struct hl_mmu_funcs { 
 * @supports_coresight: is CoreSight supported.

 * @supports_soft_reset: is soft reset supported.

 * @supports_cb_mapping: is mapping a CB to the device's MMU supported.

 * @needs_reset: true if reset_on_lockup is false and device should be reset

 *               due to lockup.

 */

struct hl_device {

	struct pci_dev			*pdev;
@@ -1786,7 +1793,8 @@ struct hl_device { 
	struct device			*dev_ctrl;

	struct delayed_work		work_freq;

	struct delayed_work		work_heartbeat;

	char				asic_name[32];

	char				asic_name[HL_STR_MAX];

	char				status[HL_DEV_STS_MAX][HL_STR_MAX];

	enum hl_asic_type		asic_type;

	struct hl_cq			*completion_queue;

	struct workqueue_struct		**cq_wq;
@@ -1876,6 +1884,7 @@ struct hl_device { 
	u8				supports_coresight;

	u8				supports_soft_reset;

	u8				supports_cb_mapping;

	u8				needs_reset;



	/* Parameters for bring-up */

	u64				nic_ports_mask;
@@ -1978,7 +1987,8 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size, 


int hl_device_open(struct inode *inode, struct file *filp);

int hl_device_open_ctrl(struct inode *inode, struct file *filp);

bool hl_device_disabled_or_in_reset(struct hl_device *hdev);

bool hl_device_operational(struct hl_device *hdev,

		enum hl_device_status *status);

enum hl_device_status hl_device_status(struct hl_device *hdev);

int hl_device_set_debug_mode(struct hl_device *hdev, bool enable);

int create_hdev(struct hl_device **dev, struct pci_dev *pdev,