Commit aefd9d71 authored by Li Xi's avatar Li Xi Committed by Greg Kroah-Hartman
Browse files

staging/lustre/osc: use global osc_rq_pool to reduce memory usage 



The per-osc request pools consume a lot of memory if there are
hundreds of OSCs on one client. This will be a critical problem
if the client doesn't have sufficient memory for both OSCs and
applications.

This patch replaces per-osc request pools with a global pool
osc_rq_pool. The total memory usage is 5MB by default. And it
can be set by a module parameter of OSC:
"options osc osc_reqpool_mem_max=POOL_SIZE". The unit of POOL_SIZE
is MB. If cl_max_rpcs_in_flight is the same for all OSCs, the
memory usage of the OSC pool can be calculated as:
Min(POOL_SIZE * 1M,
    (cl_max_rpcs_in_flight + 2) * OSC number * OST_MAXREQSIZE)

Also, this patch changes the allocation logic of OSC write requests.
The allocation from osc_rq_pool will only be tried after normal
allocation failed.

Signed-off-by: default avatarWu Libin <lwu@ddn.com>
Signed-off-by: default avatarWang Shilong <wshilong@ddn.com>
Signed-off-by: default avatarLi Xi <lixi@ddn.com>
Reviewed-on: http://review.whamcloud.com/15422
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-6770


Reviewed-by: default avatarJinshan Xiong <jinshan.xiong@intel.com>
Reviewed-by: default avatarAndreas Dilger <andreas.dilger@intel.com>
Signed-off-by: default avatarOleg Drokin <oleg.drokin@intel.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent ae9c46d1

drivers/staging/lustre/lustre/include/lustre_import.h

+0 −2
Original line number Diff line number Diff line
@@ -306,8 +306,6 @@ struct obd_import { 
	__u32		     imp_msg_magic;

	__u32		     imp_msghdr_flags;       /* adjusted based on server capability */



	struct ptlrpc_request_pool *imp_rq_pool;	  /* emergency request pool */



	struct imp_at	     imp_at;		 /* adaptive timeout data */

	time_t		    imp_last_reply_time;    /* for health check */

};

drivers/staging/lustre/lustre/include/lustre_net.h

+3 −4
Original line number Diff line number Diff line
@@ -500,7 +500,7 @@ struct ptlrpc_request_pool { 
	/** Maximum message size that would fit into a request from this pool */

	int prp_rq_size;

	/** Function to allocate more requests for this pool */

	void (*prp_populate)(struct ptlrpc_request_pool *, int);

	int (*prp_populate)(struct ptlrpc_request_pool *, int);

};



struct lu_context;
@@ -2381,11 +2381,11 @@ void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc, 
			    struct ptlrpc_request *req);



void ptlrpc_free_rq_pool(struct ptlrpc_request_pool *pool);

void ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);

int ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);



struct ptlrpc_request_pool *

ptlrpc_init_rq_pool(int, int,

		    void (*populate_pool)(struct ptlrpc_request_pool *, int));

		    int (*populate_pool)(struct ptlrpc_request_pool *, int));



void ptlrpc_at_set_req_timeout(struct ptlrpc_request *req);

struct ptlrpc_request *ptlrpc_request_alloc(struct obd_import *imp,
@@ -2957,7 +2957,6 @@ void ptlrpc_lprocfs_brw(struct ptlrpc_request *req, int bytes); 


/* ptlrpc/llog_client.c */

extern struct llog_operations llog_client_ops;



/** @} net */



#endif

drivers/staging/lustre/lustre/include/obd_class.h

+0 −4
Original line number Diff line number Diff line
@@ -626,10 +626,6 @@ static inline void obd_cleanup_client_import(struct obd_device *obd) 
		CDEBUG(D_CONFIG, "%s: client import never connected\n",

		       obd->obd_name);

		ptlrpc_invalidate_import(imp);

		if (imp->imp_rq_pool) {

			ptlrpc_free_rq_pool(imp->imp_rq_pool);

			imp->imp_rq_pool = NULL;

		}

		client_destroy_import(imp);

		obd->u.cli.cl_import = NULL;

	}

drivers/staging/lustre/lustre/obdclass/genops.c

+0 −1
Original line number Diff line number Diff line
@@ -1663,7 +1663,6 @@ static void obd_zombie_export_add(struct obd_export *exp) 
static void obd_zombie_import_add(struct obd_import *imp)

{

	LASSERT(!imp->imp_sec);

	LASSERT(!imp->imp_rq_pool);

	spin_lock(&obd_zombie_impexp_lock);

	LASSERT(list_empty(&imp->imp_zombie_chain));

	zombies_count++;

drivers/staging/lustre/lustre/osc/lproc_osc.c

+14 −3
Original line number Diff line number Diff line
@@ -96,9 +96,9 @@ static ssize_t max_rpcs_in_flight_store(struct kobject *kobj, 
	struct obd_device *dev = container_of(kobj, struct obd_device,

					      obd_kobj);

	struct client_obd *cli = &dev->u.cli;

	struct ptlrpc_request_pool *pool = cli->cl_import->imp_rq_pool;

	int rc;

	unsigned long val;

	int adding, added, req_count;



	rc = kstrtoul(buffer, 10, &val);

	if (rc)
@@ -107,8 +107,19 @@ static ssize_t max_rpcs_in_flight_store(struct kobject *kobj, 
	if (val < 1 || val > OSC_MAX_RIF_MAX)

		return -ERANGE;



	if (pool && val > cli->cl_max_rpcs_in_flight)

		pool->prp_populate(pool, val-cli->cl_max_rpcs_in_flight);

	adding = val - cli->cl_max_rpcs_in_flight;

	req_count = atomic_read(&osc_pool_req_count);

	if (adding > 0 && req_count < osc_reqpool_maxreqcount) {

		/*

		 * There might be some race which will cause over-limit

		 * allocation, but it is fine.

		 */

		if (req_count + adding > osc_reqpool_maxreqcount)

			adding = osc_reqpool_maxreqcount - req_count;



		added = osc_rq_pool->prp_populate(osc_rq_pool, adding);

		atomic_add(added, &osc_pool_req_count);

	}



	client_obd_list_lock(&cli->cl_loi_list_lock);

	cli->cl_max_rpcs_in_flight = val;