Commit 2ffcb6fc authored by Martin KaFai Lau's avatar Martin KaFai Lau Committed by Alexei Starovoitov
Browse files

bpf: Refactor codes into bpf_local_storage_destroy 



This patch first renames bpf_local_storage_unlink_nolock to
bpf_local_storage_destroy(). It better reflects that it is only
used when the storage's owner (sk/task/cgrp/inode) is being kfree().

All bpf_local_storage_destroy's caller is taking the spin lock and
then free the storage. This patch also moves these two steps into
the bpf_local_storage_destroy.

This is a preparation work for a later patch that uses
bpf_mem_cache_alloc/free in the bpf_local_storage.

Signed-off-by: default avatarMartin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230308065936.1550103-3-martin.lau@linux.dev


Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 4cbd23cc

include/linux/bpf_local_storage.h

+1 −1
Original line number Diff line number Diff line
@@ -128,7 +128,7 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, 
			 struct bpf_local_storage_map *smap,

			 bool cacheit_lockit);



bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage);

void bpf_local_storage_destroy(struct bpf_local_storage *local_storage);



void bpf_local_storage_map_free(struct bpf_map *map,

				struct bpf_local_storage_cache *cache,

kernel/bpf/bpf_cgrp_storage.c

+1 −8
Original line number Diff line number Diff line
@@ -46,8 +46,6 @@ static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner) 
void bpf_cgrp_storage_free(struct cgroup *cgroup)

{

	struct bpf_local_storage *local_storage;

	bool free_cgroup_storage = false;

	unsigned long flags;



	rcu_read_lock();

	local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
@@ -57,14 +55,9 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup) 
	}



	bpf_cgrp_storage_lock();

	raw_spin_lock_irqsave(&local_storage->lock, flags);

	free_cgroup_storage = bpf_local_storage_unlink_nolock(local_storage);

	raw_spin_unlock_irqrestore(&local_storage->lock, flags);

	bpf_local_storage_destroy(local_storage);

	bpf_cgrp_storage_unlock();

	rcu_read_unlock();



	if (free_cgroup_storage)

		kfree_rcu(local_storage, rcu);

}



static struct bpf_local_storage_data *

kernel/bpf/bpf_inode_storage.c

+1 −7
Original line number Diff line number Diff line
@@ -57,7 +57,6 @@ static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 
void bpf_inode_storage_free(struct inode *inode)

{

	struct bpf_local_storage *local_storage;

	bool free_inode_storage = false;

	struct bpf_storage_blob *bsb;



	bsb = bpf_inode(inode);
@@ -72,13 +71,8 @@ void bpf_inode_storage_free(struct inode *inode) 
		return;

	}



	raw_spin_lock_bh(&local_storage->lock);

	free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);

	raw_spin_unlock_bh(&local_storage->lock);

	bpf_local_storage_destroy(local_storage);

	rcu_read_unlock();



	if (free_inode_storage)

		kfree_rcu(local_storage, rcu);

}



static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)

kernel/bpf/bpf_local_storage.c

+6 −2
Original line number Diff line number Diff line
@@ -652,11 +652,12 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map, 
	return 0;

}



bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage)

void bpf_local_storage_destroy(struct bpf_local_storage *local_storage)

{

	struct bpf_local_storage_elem *selem;

	bool free_storage = false;

	struct hlist_node *n;

	unsigned long flags;



	/* Neither the bpf_prog nor the bpf_map's syscall

	 * could be modifying the local_storage->list now.
@@ -667,6 +668,7 @@ bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage) 
	 * when unlinking elem from the local_storage->list and

	 * the map's bucket->list.

	 */

	raw_spin_lock_irqsave(&local_storage->lock, flags);

	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {

		/* Always unlink from map before unlinking from

		 * local_storage.
@@ -681,8 +683,10 @@ bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage) 
		free_storage = bpf_selem_unlink_storage_nolock(

			local_storage, selem, false, false);

	}

	raw_spin_unlock_irqrestore(&local_storage->lock, flags);



	return free_storage;

	if (free_storage)

		kfree_rcu(local_storage, rcu);

}



u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map)

kernel/bpf/bpf_task_storage.c

+1 −8
Original line number Diff line number Diff line
@@ -72,8 +72,6 @@ task_storage_lookup(struct task_struct *task, struct bpf_map *map, 
void bpf_task_storage_free(struct task_struct *task)

{

	struct bpf_local_storage *local_storage;

	bool free_task_storage = false;

	unsigned long flags;



	rcu_read_lock();
@@ -84,14 +82,9 @@ void bpf_task_storage_free(struct task_struct *task) 
	}



	bpf_task_storage_lock();

	raw_spin_lock_irqsave(&local_storage->lock, flags);

	free_task_storage = bpf_local_storage_unlink_nolock(local_storage);

	raw_spin_unlock_irqrestore(&local_storage->lock, flags);

	bpf_local_storage_destroy(local_storage);

	bpf_task_storage_unlock();

	rcu_read_unlock();



	if (free_task_storage)

		kfree_rcu(local_storage, rcu);

}



static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)