Commit 8613dec0 authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge tag 'nvme-6.2-2022-11-29' of git://git.infradead.org/nvme into for-6.2/block 

Pull NVMe updates from Christoph:

"nvme updates for Linux 6.2

 - support some passthrough commands without CAP_SYS_ADMIN
   (Kanchan Joshi)
 - refactor PCIe probing and reset (Christoph Hellwig)
 - various fabrics authentication fixes and improvements (Sagi Grimberg)
 - avoid fallback to sequential scan due to transient issues
   (Uday Shankar)
 - implement support for the DEAC bit in Write Zeroes (Christoph Hellwig)
 - allow overriding the IEEE OUI and firmware revision in configfs for
   nvmet (Aleksandr Miloserdov)
 - force reconnect when number of queue changes in nvmet (Daniel Wagner)
 - minor fixes and improvements (Uros Bizjak, Joel Granados,
   Sagi Grimberg, Christoph Hellwig, Christophe JAILLET)"

* tag 'nvme-6.2-2022-11-29' of git://git.infradead.org/nvme: (45 commits)
  nvmet: expose firmware revision to configfs
  nvmet: expose IEEE OUI to configfs
  nvme: rename the queue quiescing helpers
  nvmet: fix a memory leak in nvmet_auth_set_key
  nvme: return err on nvme_init_non_mdts_limits fail
  nvme: avoid fallback to sequential scan due to transient issues
  nvme-rdma: stop auth work after tearing down queues in error recovery
  nvme-tcp: stop auth work after tearing down queues in error recovery
  nvme-auth: have dhchap_auth_work wait for queues auth to complete
  nvme-auth: remove redundant auth_work flush
  nvme-auth: convert dhchap_auth_list to an array
  nvme-auth: check chap ctrl_key once constructed
  nvme-auth: no need to reset chap contexts on re-authentication
  nvme-auth: remove redundant deallocations
  nvme-auth: clear sensitive info right after authentication completes
  nvme-auth: guarantee dhchap buffers under memory pressure
  nvme-auth: don't keep long lived 4k dhchap buffer
  nvme-auth: remove redundant if statement
  nvme-auth: don't override ctrl keys before validation
  nvme-auth: don't ignore key generation failures when initializing ctrl keys
  ...
parents 3692fec8 68c5444c

drivers/nvme/host/apple.c

+7 −7
Original line number Diff line number Diff line
@@ -821,7 +821,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) 
	if (!dead && shutdown && freeze)

		nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT);



	nvme_stop_queues(&anv->ctrl);

	nvme_quiesce_io_queues(&anv->ctrl);



	if (!dead) {

		if (READ_ONCE(anv->ioq.enabled)) {
@@ -837,7 +837,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) 
	WRITE_ONCE(anv->ioq.enabled, false);

	WRITE_ONCE(anv->adminq.enabled, false);

	mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */

	nvme_stop_admin_queue(&anv->ctrl);

	nvme_quiesce_admin_queue(&anv->ctrl);



	/* last chance to complete any requests before nvme_cancel_request */

	spin_lock_irqsave(&anv->lock, flags);
@@ -854,8 +854,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) 
	 * deadlocking blk-mq hot-cpu notifier.

	 */

	if (shutdown) {

		nvme_start_queues(&anv->ctrl);

		nvme_start_admin_queue(&anv->ctrl);

		nvme_unquiesce_io_queues(&anv->ctrl);

		nvme_unquiesce_admin_queue(&anv->ctrl);

	}

}
@@ -1093,7 +1093,7 @@ static void apple_nvme_reset_work(struct work_struct *work) 


	dev_dbg(anv->dev, "Starting admin queue");

	apple_nvme_init_queue(&anv->adminq);

	nvme_start_admin_queue(&anv->ctrl);

	nvme_unquiesce_admin_queue(&anv->ctrl);



	if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) {

		dev_warn(anv->ctrl.device,
@@ -1102,7 +1102,7 @@ static void apple_nvme_reset_work(struct work_struct *work) 
		goto out;

	}



	ret = nvme_init_ctrl_finish(&anv->ctrl);

	ret = nvme_init_ctrl_finish(&anv->ctrl, false);

	if (ret)

		goto out;
@@ -1127,7 +1127,7 @@ static void apple_nvme_reset_work(struct work_struct *work) 


	anv->ctrl.queue_count = nr_io_queues + 1;



	nvme_start_queues(&anv->ctrl);

	nvme_unquiesce_io_queues(&anv->ctrl);

	nvme_wait_freeze(&anv->ctrl);

	blk_mq_update_nr_hw_queues(&anv->tagset, 1);

	nvme_unfreeze(&anv->ctrl);

drivers/nvme/host/auth.c

+136 −122
Original line number Diff line number Diff line
@@ -13,6 +13,10 @@ 
#include "fabrics.h"

#include <linux/nvme-auth.h>



#define CHAP_BUF_SIZE 4096

static struct kmem_cache *nvme_chap_buf_cache;

static mempool_t *nvme_chap_buf_pool;



struct nvme_dhchap_queue_context {

	struct list_head entry;

	struct work_struct auth_work;
@@ -20,7 +24,6 @@ struct nvme_dhchap_queue_context { 
	struct crypto_shash *shash_tfm;

	struct crypto_kpp *dh_tfm;

	void *buf;

	size_t buf_size;

	int qid;

	int error;

	u32 s1;
@@ -47,6 +50,12 @@ struct nvme_dhchap_queue_context { 
#define nvme_auth_queue_from_qid(ctrl, qid) \

	(qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q



static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)

{

	return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +

			ctrl->opts->nr_poll_queues + 1;

}



static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,

			    void *data, size_t data_len, bool auth_send)

{
@@ -112,7 +121,7 @@ static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 
	struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;

	size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);



	if (chap->buf_size < size) {

	if (size > CHAP_BUF_SIZE) {

		chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

		return -EINVAL;

	}
@@ -147,7 +156,7 @@ static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 
	const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);

	const char *hmac_name, *kpp_name;



	if (chap->buf_size < size) {

	if (size > CHAP_BUF_SIZE) {

		chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

		return NVME_SC_INVALID_FIELD;

	}
@@ -197,12 +206,6 @@ static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 
		return NVME_SC_AUTH_REQUIRED;

	}



	/* Reset host response if the hash had been changed */

	if (chap->hash_id != data->hashid) {

		kfree(chap->host_response);

		chap->host_response = NULL;

	}



	chap->hash_id = data->hashid;

	chap->hash_len = data->hl;

	dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
@@ -219,14 +222,6 @@ static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 
		return NVME_SC_AUTH_REQUIRED;

	}



	/* Clear host and controller key to avoid accidental reuse */

	kfree_sensitive(chap->host_key);

	chap->host_key = NULL;

	chap->host_key_len = 0;

	kfree_sensitive(chap->ctrl_key);

	chap->ctrl_key = NULL;

	chap->ctrl_key_len = 0;



	if (chap->dhgroup_id == data->dhgid &&

	    (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {

		dev_dbg(ctrl->device,
@@ -302,7 +297,7 @@ static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 
	if (chap->host_key_len)

		size += chap->host_key_len;



	if (chap->buf_size < size) {

	if (size > CHAP_BUF_SIZE) {

		chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

		return -EINVAL;

	}
@@ -344,10 +339,10 @@ static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 
	struct nvmf_auth_dhchap_success1_data *data = chap->buf;

	size_t size = sizeof(*data);



	if (ctrl->ctrl_key)

	if (chap->ctrl_key)

		size += chap->hash_len;



	if (chap->buf_size < size) {

	if (size > CHAP_BUF_SIZE) {

		chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

		return NVME_SC_INVALID_FIELD;

	}
@@ -521,6 +516,7 @@ static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 
		ret = PTR_ERR(ctrl_response);

		return ret;

	}



	ret = crypto_shash_setkey(chap->shash_tfm,

			ctrl_response, ctrl->ctrl_key->len);

	if (ret) {
@@ -621,9 +617,6 @@ static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 
	if (ret) {

		dev_dbg(ctrl->device,

			"failed to generate public key, error %d\n", ret);

		kfree(chap->host_key);

		chap->host_key = NULL;

		chap->host_key_len = 0;

		chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

		return ret;

	}
@@ -643,9 +636,6 @@ static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 
	if (ret) {

		dev_dbg(ctrl->device,

			"failed to generate shared secret, error %d\n", ret);

		kfree_sensitive(chap->sess_key);

		chap->sess_key = NULL;

		chap->sess_key_len = 0;

		chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;

		return ret;

	}
@@ -654,7 +644,7 @@ static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 
	return 0;

}



static void __nvme_auth_reset(struct nvme_dhchap_queue_context *chap)

static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap)

{

	kfree_sensitive(chap->host_response);

	chap->host_response = NULL;
@@ -674,24 +664,20 @@ static void __nvme_auth_reset(struct nvme_dhchap_queue_context *chap) 
	chap->transaction = 0;

	memset(chap->c1, 0, sizeof(chap->c1));

	memset(chap->c2, 0, sizeof(chap->c2));

	mempool_free(chap->buf, nvme_chap_buf_pool);

	chap->buf = NULL;

}



static void __nvme_auth_free(struct nvme_dhchap_queue_context *chap)

static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap)

{

	__nvme_auth_reset(chap);

	nvme_auth_reset_dhchap(chap);

	if (chap->shash_tfm)

		crypto_free_shash(chap->shash_tfm);

	if (chap->dh_tfm)

		crypto_free_kpp(chap->dh_tfm);

	kfree_sensitive(chap->ctrl_key);

	kfree_sensitive(chap->host_key);

	kfree_sensitive(chap->sess_key);

	kfree_sensitive(chap->host_response);

	kfree(chap->buf);

	kfree(chap);

}



static void __nvme_auth_work(struct work_struct *work)

static void nvme_queue_auth_work(struct work_struct *work)

{

	struct nvme_dhchap_queue_context *chap =

		container_of(work, struct nvme_dhchap_queue_context, auth_work);
@@ -699,6 +685,16 @@ static void __nvme_auth_work(struct work_struct *work) 
	size_t tl;

	int ret = 0;



	/*

	 * Allocate a large enough buffer for the entire negotiation:

	 * 4k is enough to ffdhe8192.

	 */

	chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL);

	if (!chap->buf) {

		chap->error = -ENOMEM;

		return;

	}



	chap->transaction = ctrl->transaction++;



	/* DH-HMAC-CHAP Step 1: send negotiate */
@@ -720,8 +716,9 @@ static void __nvme_auth_work(struct work_struct *work) 
	dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",

		__func__, chap->qid);



	memset(chap->buf, 0, chap->buf_size);

	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, chap->buf_size, false);

	memset(chap->buf, 0, CHAP_BUF_SIZE);

	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,

			       false);

	if (ret) {

		dev_warn(ctrl->device,

			 "qid %d failed to receive challenge, %s %d\n",
@@ -757,11 +754,14 @@ static void __nvme_auth_work(struct work_struct *work) 


	dev_dbg(ctrl->device, "%s: qid %d host response\n",

		__func__, chap->qid);

	mutex_lock(&ctrl->dhchap_auth_mutex);

	ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);

	if (ret) {

		mutex_unlock(&ctrl->dhchap_auth_mutex);

		chap->error = ret;

		goto fail2;

	}

	mutex_unlock(&ctrl->dhchap_auth_mutex);



	/* DH-HMAC-CHAP Step 3: send reply */

	dev_dbg(ctrl->device, "%s: qid %d send reply\n",
@@ -783,8 +783,9 @@ static void __nvme_auth_work(struct work_struct *work) 
	dev_dbg(ctrl->device, "%s: qid %d receive success1\n",

		__func__, chap->qid);



	memset(chap->buf, 0, chap->buf_size);

	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, chap->buf_size, false);

	memset(chap->buf, 0, CHAP_BUF_SIZE);

	ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,

			       false);

	if (ret) {

		dev_warn(ctrl->device,

			 "qid %d failed to receive success1, %s %d\n",
@@ -801,16 +802,19 @@ static void __nvme_auth_work(struct work_struct *work) 
		return;

	}



	mutex_lock(&ctrl->dhchap_auth_mutex);

	if (ctrl->ctrl_key) {

		dev_dbg(ctrl->device,

			"%s: qid %d controller response\n",

			__func__, chap->qid);

		ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);

		if (ret) {

			mutex_unlock(&ctrl->dhchap_auth_mutex);

			chap->error = ret;

			goto fail2;

		}

	}

	mutex_unlock(&ctrl->dhchap_auth_mutex);



	ret = nvme_auth_process_dhchap_success1(ctrl, chap);

	if (ret) {
@@ -819,7 +823,7 @@ static void __nvme_auth_work(struct work_struct *work) 
		goto fail2;

	}



	if (ctrl->ctrl_key) {

	if (chap->ctrl_key) {

		/* DH-HMAC-CHAP Step 5: send success2 */

		dev_dbg(ctrl->device, "%s: qid %d send success2\n",

			__func__, chap->qid);
@@ -860,42 +864,8 @@ int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 
		return -ENOKEY;

	}



	mutex_lock(&ctrl->dhchap_auth_mutex);

	/* Check if the context is already queued */

	list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) {

		WARN_ON(!chap->buf);

		if (chap->qid == qid) {

			dev_dbg(ctrl->device, "qid %d: re-using context\n", qid);

			mutex_unlock(&ctrl->dhchap_auth_mutex);

			flush_work(&chap->auth_work);

			__nvme_auth_reset(chap);

			queue_work(nvme_wq, &chap->auth_work);

			return 0;

		}

	}

	chap = kzalloc(sizeof(*chap), GFP_KERNEL);

	if (!chap) {

		mutex_unlock(&ctrl->dhchap_auth_mutex);

		return -ENOMEM;

	}

	chap->qid = (qid == NVME_QID_ANY) ? 0 : qid;

	chap->ctrl = ctrl;



	/*

	 * Allocate a large enough buffer for the entire negotiation:

	 * 4k should be enough to ffdhe8192.

	 */

	chap->buf_size = 4096;

	chap->buf = kzalloc(chap->buf_size, GFP_KERNEL);

	if (!chap->buf) {

		mutex_unlock(&ctrl->dhchap_auth_mutex);

		kfree(chap);

		return -ENOMEM;

	}



	INIT_WORK(&chap->auth_work, __nvme_auth_work);

	list_add(&chap->entry, &ctrl->dhchap_auth_list);

	mutex_unlock(&ctrl->dhchap_auth_mutex);

	chap = &ctrl->dhchap_ctxs[qid];

	cancel_work_sync(&chap->auth_work);

	queue_work(nvme_wq, &chap->auth_work);

	return 0;

}
@@ -906,40 +876,28 @@ int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 
	struct nvme_dhchap_queue_context *chap;

	int ret;



	mutex_lock(&ctrl->dhchap_auth_mutex);

	list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) {

		if (chap->qid != qid)

			continue;

		mutex_unlock(&ctrl->dhchap_auth_mutex);

	chap = &ctrl->dhchap_ctxs[qid];

	flush_work(&chap->auth_work);

	ret = chap->error;

	/* clear sensitive info */

	nvme_auth_reset_dhchap(chap);

	return ret;

}

	mutex_unlock(&ctrl->dhchap_auth_mutex);

	return -ENXIO;

}

EXPORT_SYMBOL_GPL(nvme_auth_wait);



void nvme_auth_reset(struct nvme_ctrl *ctrl)

{

	struct nvme_dhchap_queue_context *chap;



	mutex_lock(&ctrl->dhchap_auth_mutex);

	list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) {

		mutex_unlock(&ctrl->dhchap_auth_mutex);

		flush_work(&chap->auth_work);

		__nvme_auth_reset(chap);

	}

	mutex_unlock(&ctrl->dhchap_auth_mutex);

}

EXPORT_SYMBOL_GPL(nvme_auth_reset);



static void nvme_dhchap_auth_work(struct work_struct *work)

static void nvme_ctrl_auth_work(struct work_struct *work)

{

	struct nvme_ctrl *ctrl =

		container_of(work, struct nvme_ctrl, dhchap_auth_work);

	int ret, q;



	/*

	 * If the ctrl is no connected, bail as reconnect will handle

	 * authentication.

	 */

	if (ctrl->state != NVME_CTRL_LIVE)

		return;



	/* Authenticate admin queue first */

	ret = nvme_auth_negotiate(ctrl, 0);

	if (ret) {
@@ -968,43 +926,75 @@ static void nvme_dhchap_auth_work(struct work_struct *work) 
	 * Failure is a soft-state; credentials remain valid until

	 * the controller terminates the connection.

	 */

	for (q = 1; q < ctrl->queue_count; q++) {

		ret = nvme_auth_wait(ctrl, q);

		if (ret)

			dev_warn(ctrl->device,

				 "qid %d: authentication failed\n", q);

	}

}



void nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)

int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)

{

	INIT_LIST_HEAD(&ctrl->dhchap_auth_list);

	INIT_WORK(&ctrl->dhchap_auth_work, nvme_dhchap_auth_work);

	struct nvme_dhchap_queue_context *chap;

	int i, ret;



	mutex_init(&ctrl->dhchap_auth_mutex);

	INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);

	if (!ctrl->opts)

		return;

	nvme_auth_generate_key(ctrl->opts->dhchap_secret, &ctrl->host_key);

	nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, &ctrl->ctrl_key);

		return 0;

	ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,

			&ctrl->host_key);

	if (ret)

		return ret;

	ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,

			&ctrl->ctrl_key);

	if (ret)

		goto err_free_dhchap_secret;



	if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)

		return ret;



	ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),

				sizeof(*chap), GFP_KERNEL);

	if (!ctrl->dhchap_ctxs) {

		ret = -ENOMEM;

		goto err_free_dhchap_ctrl_secret;

	}



	for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {

		chap = &ctrl->dhchap_ctxs[i];

		chap->qid = i;

		chap->ctrl = ctrl;

		INIT_WORK(&chap->auth_work, nvme_queue_auth_work);

	}



	return 0;

err_free_dhchap_ctrl_secret:

	nvme_auth_free_key(ctrl->ctrl_key);

	ctrl->ctrl_key = NULL;

err_free_dhchap_secret:

	nvme_auth_free_key(ctrl->host_key);

	ctrl->host_key = NULL;

	return ret;

}

EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);



void nvme_auth_stop(struct nvme_ctrl *ctrl)

{

	struct nvme_dhchap_queue_context *chap = NULL, *tmp;



	cancel_work_sync(&ctrl->dhchap_auth_work);

	mutex_lock(&ctrl->dhchap_auth_mutex);

	list_for_each_entry_safe(chap, tmp, &ctrl->dhchap_auth_list, entry)

		cancel_work_sync(&chap->auth_work);

	mutex_unlock(&ctrl->dhchap_auth_mutex);

}

EXPORT_SYMBOL_GPL(nvme_auth_stop);



void nvme_auth_free(struct nvme_ctrl *ctrl)

{

	struct nvme_dhchap_queue_context *chap = NULL, *tmp;

	int i;



	mutex_lock(&ctrl->dhchap_auth_mutex);

	list_for_each_entry_safe(chap, tmp, &ctrl->dhchap_auth_list, entry) {

		list_del_init(&chap->entry);

		flush_work(&chap->auth_work);

		__nvme_auth_free(chap);

	if (ctrl->dhchap_ctxs) {

		for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)

			nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);

		kfree(ctrl->dhchap_ctxs);

	}

	mutex_unlock(&ctrl->dhchap_auth_mutex);

	if (ctrl->host_key) {

		nvme_auth_free_key(ctrl->host_key);

		ctrl->host_key = NULL;
@@ -1015,3 +1005,27 @@ void nvme_auth_free(struct nvme_ctrl *ctrl) 
	}

}

EXPORT_SYMBOL_GPL(nvme_auth_free);



int __init nvme_init_auth(void)

{

	nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache",

				CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL);

	if (!nvme_chap_buf_cache)

		return -ENOMEM;



	nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab,

			mempool_free_slab, nvme_chap_buf_cache);

	if (!nvme_chap_buf_pool)

		goto err_destroy_chap_buf_cache;



	return 0;

err_destroy_chap_buf_cache:

	kmem_cache_destroy(nvme_chap_buf_cache);

	return -ENOMEM;

}



void __exit nvme_exit_auth(void)

{

	mempool_destroy(nvme_chap_buf_pool);

	kmem_cache_destroy(nvme_chap_buf_cache);

}

drivers/nvme/host/core.c

+93 −32
Original line number Diff line number Diff line
@@ -850,8 +850,11 @@ static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns, 
	cmnd->write_zeroes.length =

		cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);



	if (!(req->cmd_flags & REQ_NOUNMAP) && (ns->features & NVME_NS_DEAC))

		cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC);



	if (nvme_ns_has_pi(ns)) {

		cmnd->write_zeroes.control = cpu_to_le16(NVME_RW_PRINFO_PRACT);

		cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT);



		switch (ns->pi_type) {

		case NVME_NS_DPS_PI_TYPE1:
@@ -1120,8 +1123,10 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects, 
		mutex_unlock(&ctrl->subsys->lock);

		mutex_unlock(&ctrl->scan_lock);

	}

	if (effects & NVME_CMD_EFFECTS_CCC)

		nvme_init_ctrl_finish(ctrl);

	if (effects & NVME_CMD_EFFECTS_CCC) {

		dev_info(ctrl->device,

"controller capabilities changed, reset may be required to take effect.\n");

	}

	if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {

		nvme_queue_scan(ctrl);

		flush_work(&ctrl->scan_work);
@@ -2003,6 +2008,14 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, 
		}

	}



	/*

	 * Only set the DEAC bit if the device guarantees that reads from

	 * deallocated data return zeroes.  While the DEAC bit does not

	 * require that, it must be a no-op if reads from deallocated data

	 * do not return zeroes.

	 */

	if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3)))

		ns->features |= NVME_NS_DEAC;

	set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));

	set_bit(NVME_NS_READY, &ns->flags);

	blk_mq_unfreeze_queue(ns->disk->queue);
@@ -2179,7 +2192,7 @@ const struct pr_ops nvme_pr_ops = { 
};



#ifdef CONFIG_BLK_SED_OPAL

int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,

static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,

		bool send)

{

	struct nvme_ctrl *ctrl = data;
@@ -2196,7 +2209,23 @@ int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len, 
	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,

			NVME_QID_ANY, 1, 0);

}

EXPORT_SYMBOL_GPL(nvme_sec_submit);



static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)

{

	if (ctrl->oacs & NVME_CTRL_OACS_SEC_SUPP) {

		if (!ctrl->opal_dev)

			ctrl->opal_dev = init_opal_dev(ctrl, &nvme_sec_submit);

		else if (was_suspended)

			opal_unlock_from_suspend(ctrl->opal_dev);

	} else {

		free_opal_dev(ctrl->opal_dev);

		ctrl->opal_dev = NULL;

	}

}

#else

static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)

{

}

#endif /* CONFIG_BLK_SED_OPAL */



#ifdef CONFIG_BLK_DEV_ZONED
@@ -3049,7 +3078,7 @@ static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl) 


	id = kzalloc(sizeof(*id), GFP_KERNEL);

	if (!id)

		return 0;

		return -ENOMEM;



	c.identify.opcode = nvme_admin_identify;

	c.identify.cns = NVME_ID_CNS_CS_CTRL;
@@ -3229,7 +3258,7 @@ static int nvme_init_identify(struct nvme_ctrl *ctrl) 
 * register in our nvme_ctrl structure.  This should be called as soon as

 * the admin queue is fully up and running.

 */

int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl)

int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)

{

	int ret;
@@ -3260,6 +3289,8 @@ int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl) 
	if (ret < 0)

		return ret;



	nvme_configure_opal(ctrl, was_suspended);



	if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {

		/*

		 * Do not return errors unless we are in a controller reset,
@@ -3745,15 +3776,19 @@ static ssize_t nvme_ctrl_dhchap_secret_store(struct device *dev, 
	memcpy(dhchap_secret, buf, count);

	nvme_auth_stop(ctrl);

	if (strcmp(dhchap_secret, opts->dhchap_secret)) {

		struct nvme_dhchap_key *key, *host_key;

		int ret;



		ret = nvme_auth_generate_key(dhchap_secret, &ctrl->host_key);

		ret = nvme_auth_generate_key(dhchap_secret, &key);

		if (ret)

			return ret;

		kfree(opts->dhchap_secret);

		opts->dhchap_secret = dhchap_secret;

		/* Key has changed; re-authentication with new key */

		nvme_auth_reset(ctrl);

		host_key = ctrl->host_key;

		mutex_lock(&ctrl->dhchap_auth_mutex);

		ctrl->host_key = key;

		mutex_unlock(&ctrl->dhchap_auth_mutex);

		nvme_auth_free_key(host_key);

	}

	/* Start re-authentication */

	dev_info(ctrl->device, "re-authenticating controller\n");
@@ -3795,15 +3830,19 @@ static ssize_t nvme_ctrl_dhchap_ctrl_secret_store(struct device *dev, 
	memcpy(dhchap_secret, buf, count);

	nvme_auth_stop(ctrl);

	if (strcmp(dhchap_secret, opts->dhchap_ctrl_secret)) {

		struct nvme_dhchap_key *key, *ctrl_key;

		int ret;



		ret = nvme_auth_generate_key(dhchap_secret, &ctrl->ctrl_key);

		ret = nvme_auth_generate_key(dhchap_secret, &key);

		if (ret)

			return ret;

		kfree(opts->dhchap_ctrl_secret);

		opts->dhchap_ctrl_secret = dhchap_secret;

		/* Key has changed; re-authentication with new key */

		nvme_auth_reset(ctrl);

		ctrl_key = ctrl->ctrl_key;

		mutex_lock(&ctrl->dhchap_auth_mutex);

		ctrl->ctrl_key = key;

		mutex_unlock(&ctrl->dhchap_auth_mutex);

		nvme_auth_free_key(ctrl_key);

	}

	/* Start re-authentication */

	dev_info(ctrl->device, "re-authenticating controller\n");
@@ -3875,10 +3914,11 @@ static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj, 
	return a->mode;

}



static const struct attribute_group nvme_dev_attrs_group = {

const struct attribute_group nvme_dev_attrs_group = {

	.attrs		= nvme_dev_attrs,

	.is_visible	= nvme_dev_attrs_are_visible,

};

EXPORT_SYMBOL_GPL(nvme_dev_attrs_group);



static const struct attribute_group *nvme_dev_attr_groups[] = {

	&nvme_dev_attrs_group,
@@ -4420,9 +4460,6 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) 
	u32 prev = 0;

	int ret = 0, i;



	if (nvme_ctrl_limited_cns(ctrl))

		return -EOPNOTSUPP;



	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);

	if (!ns_list)

		return -ENOMEM;
@@ -4530,8 +4567,18 @@ static void nvme_scan_work(struct work_struct *work) 
	}



	mutex_lock(&ctrl->scan_lock);

	if (nvme_scan_ns_list(ctrl) != 0)

	if (nvme_ctrl_limited_cns(ctrl)) {

		nvme_scan_ns_sequential(ctrl);

	} else {

		/*

		 * Fall back to sequential scan if DNR is set to handle broken

		 * devices which should support Identify NS List (as per the VS

		 * they report) but don't actually support it.

		 */

		ret = nvme_scan_ns_list(ctrl);

		if (ret > 0 && ret & NVME_SC_DNR)

			nvme_scan_ns_sequential(ctrl);

	}

	mutex_unlock(&ctrl->scan_lock);

}
@@ -4563,7 +4610,7 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl) 
	 */

	if (ctrl->state == NVME_CTRL_DEAD) {

		nvme_mark_namespaces_dead(ctrl);

		nvme_start_queues(ctrl);

		nvme_unquiesce_io_queues(ctrl);

	}



	/* this is a no-op when called from the controller reset handler */
@@ -4690,7 +4737,7 @@ static void nvme_fw_act_work(struct work_struct *work) 
		fw_act_timeout = jiffies +

				msecs_to_jiffies(admin_timeout * 1000);



	nvme_stop_queues(ctrl);

	nvme_quiesce_io_queues(ctrl);

	while (nvme_ctrl_pp_status(ctrl)) {

		if (time_after(jiffies, fw_act_timeout)) {

			dev_warn(ctrl->device,
@@ -4704,7 +4751,7 @@ static void nvme_fw_act_work(struct work_struct *work) 
	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))

		return;



	nvme_start_queues(ctrl);

	nvme_unquiesce_io_queues(ctrl);

	/* read FW slot information to clear the AER */

	nvme_get_fw_slot_info(ctrl);
@@ -4949,7 +4996,7 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl) 


	if (ctrl->queue_count > 1) {

		nvme_queue_scan(ctrl);

		nvme_start_queues(ctrl);

		nvme_unquiesce_io_queues(ctrl);

		nvme_mpath_update(ctrl);

	}
@@ -4994,6 +5041,7 @@ static void nvme_free_ctrl(struct device *dev) 
	nvme_auth_stop(ctrl);

	nvme_auth_free(ctrl);

	__free_page(ctrl->discard_page);

	free_opal_dev(ctrl->opal_dev);



	if (subsys) {

		mutex_lock(&nvme_subsystems_lock);
@@ -5059,6 +5107,9 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 
			ctrl->instance);

	ctrl->device->class = nvme_class;

	ctrl->device->parent = ctrl->dev;

	if (ops->dev_attr_groups)

		ctrl->device->groups = ops->dev_attr_groups;

	else

		ctrl->device->groups = nvme_dev_attr_groups;

	ctrl->device->release = nvme_free_ctrl;

	dev_set_drvdata(ctrl->device, ctrl);
@@ -5083,9 +5134,13 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, 


	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));

	nvme_mpath_init_ctrl(ctrl);

	nvme_auth_init_ctrl(ctrl);

	ret = nvme_auth_init_ctrl(ctrl);

	if (ret)

		goto out_free_cdev;



	return 0;

out_free_cdev:

	cdev_device_del(&ctrl->cdev, ctrl->device);

out_free_name:

	nvme_put_ctrl(ctrl);

	kfree_const(ctrl->device->kobj.name);
@@ -5158,37 +5213,37 @@ void nvme_start_freeze(struct nvme_ctrl *ctrl) 
}

EXPORT_SYMBOL_GPL(nvme_start_freeze);



void nvme_stop_queues(struct nvme_ctrl *ctrl)

void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)

{

	if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))

		blk_mq_quiesce_tagset(ctrl->tagset);

	else

		blk_mq_wait_quiesce_done(ctrl->tagset);

}

EXPORT_SYMBOL_GPL(nvme_stop_queues);

EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);



void nvme_start_queues(struct nvme_ctrl *ctrl)

void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)

{

	if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))

		blk_mq_unquiesce_tagset(ctrl->tagset);

}

EXPORT_SYMBOL_GPL(nvme_start_queues);

EXPORT_SYMBOL_GPL(nvme_unquiesce_io_queues);



void nvme_stop_admin_queue(struct nvme_ctrl *ctrl)

void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl)

{

	if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))

		blk_mq_quiesce_queue(ctrl->admin_q);

	else

		blk_mq_wait_quiesce_done(ctrl->admin_q->tag_set);

}

EXPORT_SYMBOL_GPL(nvme_stop_admin_queue);

EXPORT_SYMBOL_GPL(nvme_quiesce_admin_queue);



void nvme_start_admin_queue(struct nvme_ctrl *ctrl)

void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl)

{

	if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))

		blk_mq_unquiesce_queue(ctrl->admin_q);

}

EXPORT_SYMBOL_GPL(nvme_start_admin_queue);

EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);



void nvme_sync_io_queues(struct nvme_ctrl *ctrl)

{
@@ -5299,8 +5354,13 @@ static int __init nvme_core_init(void) 
		goto unregister_generic_ns;

	}



	result = nvme_init_auth();

	if (result)

		goto destroy_ns_chr;

	return 0;



destroy_ns_chr:

	class_destroy(nvme_ns_chr_class);

unregister_generic_ns:

	unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);

destroy_subsys_class:
@@ -5321,6 +5381,7 @@ static int __init nvme_core_init(void) 


static void __exit nvme_core_exit(void)

{

	nvme_exit_auth();

	class_destroy(nvme_ns_chr_class);

	class_destroy(nvme_subsys_class);

	class_destroy(nvme_class);

drivers/nvme/host/fc.c

+18 −17

File changed.

Preview size limit exceeded, changes collapsed.

drivers/nvme/host/ioctl.c

+83 −33

File changed.

Preview size limit exceeded, changes collapsed.