Commit 07e10515 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

rcu-tasks: Create struct to hold state information 



This commit creates an rcu_tasks struct to hold state information for
RCU Tasks.  This is a preparation commit for adding additional flavors
of Tasks RCU, each of which would have its own rcu_tasks struct.

Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent eacd6f04

kernel/rcu/tasks.h

+46 −27
Original line number Diff line number Diff line
@@ -7,6 +7,30 @@ 


#ifdef CONFIG_TASKS_RCU



/**

 * Definition for a Tasks-RCU-like mechanism.

 * @cbs_head: Head of callback list.

 * @cbs_tail: Tail pointer for callback list.

 * @cbs_wq: Wait queue allowning new callback to get kthread's attention.

 * @cbs_lock: Lock protecting callback list.

 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.

 */

struct rcu_tasks {

	struct rcu_head *cbs_head;

	struct rcu_head **cbs_tail;

	struct wait_queue_head cbs_wq;

	raw_spinlock_t cbs_lock;

	struct task_struct *kthread_ptr;

};



#define DEFINE_RCU_TASKS(name)						\

static struct rcu_tasks name =						\

{									\

	.cbs_tail = &name.cbs_head,					\

	.cbs_wq = __WAIT_QUEUE_HEAD_INITIALIZER(name.cbs_wq),		\

	.cbs_lock = __RAW_SPIN_LOCK_UNLOCKED(name.cbs_lock),		\

}



/*

 * Simple variant of RCU whose quiescent states are voluntary context

 * switch, cond_resched_rcu_qs(), user-space execution, and idle.
@@ -18,12 +42,7 @@ 
 * rates from multiple CPUs.  If this is required, per-CPU callback lists

 * will be needed.

 */



/* Global list of callbacks and associated lock. */

static struct rcu_head *rcu_tasks_cbs_head;

static struct rcu_head **rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;

static DECLARE_WAIT_QUEUE_HEAD(rcu_tasks_cbs_wq);

static DEFINE_RAW_SPINLOCK(rcu_tasks_cbs_lock);

DEFINE_RCU_TASKS(rcu_tasks);



/* Track exiting tasks in order to allow them to be waited for. */

DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu);
@@ -33,8 +52,6 @@ DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu); 
static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;

module_param(rcu_task_stall_timeout, int, 0644);



static struct task_struct *rcu_tasks_kthread_ptr;



/**

 * call_rcu_tasks() - Queue an RCU for invocation task-based grace period

 * @rhp: structure to be used for queueing the RCU updates.
@@ -57,17 +74,18 @@ void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func) 
{

	unsigned long flags;

	bool needwake;

	struct rcu_tasks *rtp = &rcu_tasks;



	rhp->next = NULL;

	rhp->func = func;

	raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);

	needwake = !rcu_tasks_cbs_head;

	WRITE_ONCE(*rcu_tasks_cbs_tail, rhp);

	rcu_tasks_cbs_tail = &rhp->next;

	raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);

	raw_spin_lock_irqsave(&rtp->cbs_lock, flags);

	needwake = !rtp->cbs_head;

	WRITE_ONCE(*rtp->cbs_tail, rhp);

	rtp->cbs_tail = &rhp->next;

	raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);

	/* We can't create the thread unless interrupts are enabled. */

	if (needwake && READ_ONCE(rcu_tasks_kthread_ptr))

		wake_up(&rcu_tasks_cbs_wq);

	if (needwake && READ_ONCE(rtp->kthread_ptr))

		wake_up(&rtp->cbs_wq);

}

EXPORT_SYMBOL_GPL(call_rcu_tasks);
@@ -169,10 +187,12 @@ static int __noreturn rcu_tasks_kthread(void *arg) 
	struct rcu_head *list;

	struct rcu_head *next;

	LIST_HEAD(rcu_tasks_holdouts);

	struct rcu_tasks *rtp = arg;

	int fract;



	/* Run on housekeeping CPUs by default.  Sysadm can move if desired. */

	housekeeping_affine(current, HK_FLAG_RCU);

	WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start!



	/*

	 * Each pass through the following loop makes one check for
@@ -183,17 +203,17 @@ static int __noreturn rcu_tasks_kthread(void *arg) 
	for (;;) {



		/* Pick up any new callbacks. */

		raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);

		list = rcu_tasks_cbs_head;

		rcu_tasks_cbs_head = NULL;

		rcu_tasks_cbs_tail = &rcu_tasks_cbs_head;

		raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);

		raw_spin_lock_irqsave(&rtp->cbs_lock, flags);

		list = rtp->cbs_head;

		rtp->cbs_head = NULL;

		rtp->cbs_tail = &rtp->cbs_head;

		raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);



		/* If there were none, wait a bit and start over. */

		if (!list) {

			wait_event_interruptible(rcu_tasks_cbs_wq,

						 READ_ONCE(rcu_tasks_cbs_head));

			if (!rcu_tasks_cbs_head) {

			wait_event_interruptible(rtp->cbs_wq,

						 READ_ONCE(rtp->cbs_head));

			if (!rtp->cbs_head) {

				WARN_ON(signal_pending(current));

				schedule_timeout_interruptible(HZ/10);

			}
@@ -211,7 +231,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) 
		 *

		 * This synchronize_rcu() also dispenses with the

		 * need for a memory barrier on the first store to

		 * ->rcu_tasks_holdout, as it forces the store to happen

		 * t->rcu_tasks_holdout, as it forces the store to happen

		 * after the beginning of the grace period.

		 */

		synchronize_rcu();
@@ -325,11 +345,10 @@ static int __init rcu_spawn_tasks_kthread(void) 
{

	struct task_struct *t;



	t = kthread_run(rcu_tasks_kthread, NULL, "rcu_tasks_kthread");

	t = kthread_run(rcu_tasks_kthread, &rcu_tasks, "rcu_tasks_kthread");

	if (WARN_ONCE(IS_ERR(t), "%s: Could not start Tasks-RCU grace-period kthread, OOM is now expected behavior\n", __func__))

		return 0;

	smp_mb(); /* Ensure others see full kthread. */

	WRITE_ONCE(rcu_tasks_kthread_ptr, t);

	return 0;

}

core_initcall(rcu_spawn_tasks_kthread);