Commit 94688e8e authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Andrew Morton
Browse files

mm: remove folio_pincount_ptr() and head_compound_pincount() 

We can use folio->_pincount directly, since all users are guarded by tests
of compound/large.

Link: https://lkml.kernel.org/r/20230111142915.1001531-2-willy@infradead.org


Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7d4a8be0

Documentation/core-api/pin_user_pages.rst

+14 −15
Original line number Diff line number Diff line
@@ -55,18 +55,17 @@ flags the caller provides. The caller is required to pass in a non-null struct 
pages* array, and the function then pins pages by incrementing each by a special

value: GUP_PIN_COUNTING_BIAS.



For compound pages, the GUP_PIN_COUNTING_BIAS scheme is not used. Instead,

an exact form of pin counting is achieved, by using the 2nd struct page

in the compound page. A new struct page field, compound_pincount, has

been added in order to support this.



This approach for compound pages avoids the counting upper limit problems that

are discussed below. Those limitations would have been aggravated severely by

huge pages, because each tail page adds a refcount to the head page. And in

fact, testing revealed that, without a separate compound_pincount field,

page overflows were seen in some huge page stress tests.



This also means that huge pages and compound pages do not suffer

For large folios, the GUP_PIN_COUNTING_BIAS scheme is not used. Instead,

the extra space available in the struct folio is used to store the

pincount directly.



This approach for large folios avoids the counting upper limit problems

that are discussed below. Those limitations would have been aggravated

severely by huge pages, because each tail page adds a refcount to the

head page. And in fact, testing revealed that, without a separate pincount

field, refcount overflows were seen in some huge page stress tests.



This also means that huge pages and large folios do not suffer

from the false positives problem that is mentioned below.::



 Function
@@ -264,9 +263,9 @@ place.) 
Other diagnostics

=================



dump_page() has been enhanced slightly, to handle these new counting

fields, and to better report on compound pages in general. Specifically,

for compound pages, the exact (compound_pincount) pincount is reported.

dump_page() has been enhanced slightly to handle these new counting

fields, and to better report on large folios in general.  Specifically,

for large folios, the exact pincount is reported.



References

==========

include/linux/mm.h

+2 −12
Original line number Diff line number Diff line
@@ -1011,11 +1011,6 @@ static inline void folio_set_compound_dtor(struct folio *folio, 


void destroy_large_folio(struct folio *folio);



static inline int head_compound_pincount(struct page *head)

{

	return atomic_read(compound_pincount_ptr(head));

}



static inline void set_compound_order(struct page *page, unsigned int order)

{

	page[1].compound_order = order;
@@ -1641,11 +1636,6 @@ static inline struct folio *pfn_folio(unsigned long pfn) 
	return page_folio(pfn_to_page(pfn));

}



static inline atomic_t *folio_pincount_ptr(struct folio *folio)

{

	return &folio_page(folio, 1)->compound_pincount;

}



/**

 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.

 * @folio: The folio.
@@ -1663,7 +1653,7 @@ static inline atomic_t *folio_pincount_ptr(struct folio *folio) 
 * expected to be able to deal gracefully with a false positive.

 *

 * For large folios, the result will be exactly correct. That's because

 * we have more tracking data available: the compound_pincount is used

 * we have more tracking data available: the _pincount field is used

 * instead of the GUP_PIN_COUNTING_BIAS scheme.

 *

 * For more information, please see Documentation/core-api/pin_user_pages.rst.
@@ -1674,7 +1664,7 @@ static inline atomic_t *folio_pincount_ptr(struct folio *folio) 
static inline bool folio_maybe_dma_pinned(struct folio *folio)

{

	if (folio_test_large(folio))

		return atomic_read(folio_pincount_ptr(folio)) > 0;

		return atomic_read(&folio->_pincount) > 0;



	/*

	 * folio_ref_count() is signed. If that refcount overflows, then

include/linux/mm_types.h

+0 −5
Original line number Diff line number Diff line
@@ -443,11 +443,6 @@ static inline atomic_t *subpages_mapcount_ptr(struct page *page) 
	return &page[1].subpages_mapcount;

}



static inline atomic_t *compound_pincount_ptr(struct page *page)

{

	return &page[1].compound_pincount;

}



/*

 * Used for sizing the vmemmap region on some architectures

 */

mm/debug.c

+2 −2
Original line number Diff line number Diff line
@@ -94,11 +94,11 @@ static void __dump_page(struct page *page) 
			page, page_ref_count(head), mapcount, mapping,

			page_to_pgoff(page), page_to_pfn(page));

	if (compound) {

		pr_warn("head:%p order:%u compound_mapcount:%d subpages_mapcount:%d compound_pincount:%d\n",

		pr_warn("head:%p order:%u compound_mapcount:%d subpages_mapcount:%d pincount:%d\n",

				head, compound_order(head),

				head_compound_mapcount(head),

				head_subpages_mapcount(head),

				head_compound_pincount(head));

				atomic_read(&folio->_pincount));

	}



#ifdef CONFIG_MEMCG

mm/gup.c

+4 −4
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ static inline struct folio *try_get_folio(struct page *page, int refs) 
 *    FOLL_GET: folio's refcount will be incremented by @refs.

 *

 *    FOLL_PIN on large folios: folio's refcount will be incremented by

 *    @refs, and its compound_pincount will be incremented by @refs.

 *    @refs, and its pincount will be incremented by @refs.

 *

 *    FOLL_PIN on single-page folios: folio's refcount will be incremented by

 *    @refs * GUP_PIN_COUNTING_BIAS.
@@ -157,7 +157,7 @@ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags) 
		 * try_get_folio() is left intact.

		 */

		if (folio_test_large(folio))

			atomic_add(refs, folio_pincount_ptr(folio));

			atomic_add(refs, &folio->_pincount);

		else

			folio_ref_add(folio,

					refs * (GUP_PIN_COUNTING_BIAS - 1));
@@ -182,7 +182,7 @@ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags) 
	if (flags & FOLL_PIN) {

		node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);

		if (folio_test_large(folio))

			atomic_sub(refs, folio_pincount_ptr(folio));

			atomic_sub(refs, &folio->_pincount);

		else

			refs *= GUP_PIN_COUNTING_BIAS;

	}
@@ -232,7 +232,7 @@ int __must_check try_grab_page(struct page *page, unsigned int flags) 
		 */

		if (folio_test_large(folio)) {

			folio_ref_add(folio, 1);

			atomic_add(1, folio_pincount_ptr(folio));

			atomic_add(1, &folio->_pincount);

		} else {

			folio_ref_add(folio, GUP_PIN_COUNTING_BIAS);

		}