Commit f32276a3 authored by Uladzislau Rezki (Sony)'s avatar Uladzislau Rezki (Sony) Committed by Paul E. McKenney
Browse files

rcu/kvfree: Add debug check for GP complete for kfree_rcu_cpu list 



Under low-memory conditions, kvfree_rcu() will use each object's
rcu_head structure to queue objects in a singly linked list headed by
the kfree_rcu_cpu structure's ->head field.  This list is passed to
call_rcu() as a unit, but there is no indication of which grace period
this list needs to wait for.  This in turn prevents adding debug checks
in the kfree_rcu_work() as was done for the two page-of-pointers channels
in the kfree_rcu_cpu structure.

This commit therefore adds a ->head_free_gp_snap field to the
kfree_rcu_cpu_work structure to record this grace-period number.  It also
adds a WARN_ON_ONCE() to kfree_rcu_monitor() that checks to make sure
that the required grace period has in fact elapsed.

[ paulmck: Fix kerneldoc issue raised by Stephen Rothwell. ]

Signed-off-by: default avatarUladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent cdfa0f6f

kernel/rcu/tree.c

+7 −1
Original line number Diff line number Diff line
@@ -2773,6 +2773,7 @@ struct kvfree_rcu_bulk_data { 
 * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests

 * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period

 * @head_free: List of kfree_rcu() objects waiting for a grace period

 * @head_free_gp_snap: Grace-period snapshot to check for attempted premature frees.

 * @bulk_head_free: Bulk-List of kvfree_rcu() objects waiting for a grace period

 * @krcp: Pointer to @kfree_rcu_cpu structure

 */
@@ -2780,6 +2781,7 @@ struct kvfree_rcu_bulk_data { 
struct kfree_rcu_cpu_work {

	struct rcu_work rcu_work;

	struct rcu_head *head_free;

	struct rcu_gp_oldstate head_free_gp_snap;

	struct list_head bulk_head_free[FREE_N_CHANNELS];

	struct kfree_rcu_cpu *krcp;

};
@@ -2985,6 +2987,7 @@ static void kfree_rcu_work(struct work_struct *work) 
	struct rcu_head *head;

	struct kfree_rcu_cpu *krcp;

	struct kfree_rcu_cpu_work *krwp;

	struct rcu_gp_oldstate head_gp_snap;

	int i;



	krwp = container_of(to_rcu_work(work),
@@ -2999,6 +3002,7 @@ static void kfree_rcu_work(struct work_struct *work) 
	// Channel 3.

	head = krwp->head_free;

	krwp->head_free = NULL;

	head_gp_snap = krwp->head_free_gp_snap;

	raw_spin_unlock_irqrestore(&krcp->lock, flags);



	// Handle the first two channels.
@@ -3015,6 +3019,7 @@ static void kfree_rcu_work(struct work_struct *work) 
	 * queued on a linked list through their rcu_head structures.

	 * This list is named "Channel 3".

	 */

	if (head && !WARN_ON_ONCE(!poll_state_synchronize_rcu_full(&head_gp_snap)))

		kvfree_rcu_list(head);

}
@@ -3147,6 +3152,7 @@ static void kfree_rcu_monitor(struct work_struct *work) 
			// objects queued on the linked list.

			if (!krwp->head_free) {

				krwp->head_free = krcp->head;

				get_state_synchronize_rcu_full(&krwp->head_free_gp_snap);

				atomic_set(&krcp->head_count, 0);

				WRITE_ONCE(krcp->head, NULL);

			}