TWxSoftware/twx-linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mm/page_isolation: remove migratetype from move_freepages_block_isolate()

Browse Source 
Since migratetype is no longer overwritten during pageblock isolation,
moving a pageblock out of MIGRATE_ISOLATE no longer needs a new
migratetype.

Add pageblock_isolate_and_move_free_pages() and
pageblock_unisolate_and_move_free_pages() to be explicit about the page
isolation operations.  Both share the common code in
__move_freepages_block_isolate(), which is renamed from
move_freepages_block_isolate().

Add toggle_pageblock_isolate() to flip pageblock isolation bit in
__move_freepages_block_isolate().

Make set_pageblock_migratetype() only accept non MIGRATE_ISOLATE types, so
that one should use set_pageblock_isolate() to isolate pageblocks.  As a
result, move pageblock migratetype code out of __move_freepages_block().

Link: https://lkml.kernel.org/r/20250617021115.2331563-5-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: David Hildenbrand <david@redhat.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kirill A. Shuemov <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Richard Chang <richardycc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Zi Yan
2025-06-16 22:11:12 -04:00
committed by Andrew Morton
parent 1bc3587a88
commit b1df9c5713
3 changed files with 75 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/linux/page-isolation.h
+2 -3
View File
@@ -44,10 +44,9 @@ static inline void set_pageblock_isolate(struct page *page)
void __meminit init_pageblock_migratetype(struct page *page,
enum migratetype migratetype,
bool isolate);
void set_pageblock_migratetype(struct page *page, enum migratetype migratetype);
bool move_freepages_block_isolate(struct zone *zone, struct page *page,
int migratetype);
bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page);
bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page);
int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
int migratetype, int flags);
mm/page_alloc.c
+63 -17
View File
@@ -525,8 +525,8 @@ void clear_pfnblock_bit(const struct page *page, unsigned long pfn,
* @page: The page within the block of interest
* @migratetype: migratetype to set
*/
__always_inline void set_pageblock_migratetype(struct page *page,
enum migratetype migratetype)
static void set_pageblock_migratetype(struct page *page,
enum migratetype migratetype)
{
if (unlikely(page_group_by_mobility_disabled &&
migratetype < MIGRATE_PCPTYPES))
@@ -534,9 +534,13 @@ __always_inline void set_pageblock_migratetype(struct page *page,
#ifdef CONFIG_MEMORY_ISOLATION
if (migratetype == MIGRATE_ISOLATE) {
set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
VM_WARN_ONCE(1,
"Use set_pageblock_isolate() for pageblock isolation");
return;
}
VM_WARN_ONCE(get_pfnblock_bit(page, page_to_pfn(page),
PB_migrate_isolate),
"Use clear_pageblock_isolate() to unisolate pageblock");
/* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */
#endif
__set_pfnblock_flags_mask(page, page_to_pfn(page),
@@ -1921,8 +1925,8 @@ static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
#endif
/*
* Change the type of a block and move all its free pages to that
* type's freelist.
* Move all free pages of a block to new type's freelist. Caller needs to
* change the block type.
*/
static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,
int old_mt, int new_mt)
@@ -1954,8 +1958,6 @@ static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,
pages_moved += 1 << order;
}
set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);
return pages_moved;
}
@@ -2013,11 +2015,16 @@ static int move_freepages_block(struct zone *zone, struct page *page,
int old_mt, int new_mt)
{
unsigned long start_pfn;
int res;
if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))
return -1;
return __move_freepages_block(zone, start_pfn, old_mt, new_mt);
res = __move_freepages_block(zone, start_pfn, old_mt, new_mt);
set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);
return res;
}
#ifdef CONFIG_MEMORY_ISOLATION
@@ -2045,11 +2052,19 @@ static unsigned long find_large_buddy(unsigned long start_pfn)
return start_pfn;
}
static inline void toggle_pageblock_isolate(struct page *page, bool isolate)
{
if (isolate)
set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
else
clear_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
}
/**
* move_freepages_block_isolate - move free pages in block for page isolation
* __move_freepages_block_isolate - move free pages in block for page isolation
* @zone: the zone
* @page: the pageblock page
* @migratetype: migratetype to set on the pageblock
* @isolate: to isolate the given pageblock or unisolate it
*
* This is similar to move_freepages_block(), but handles the special
* case encountered in page isolation, where the block of interest
@@ -2064,10 +2079,18 @@ static unsigned long find_large_buddy(unsigned long start_pfn)
*
* Returns %true if pages could be moved, %false otherwise.
*/
bool move_freepages_block_isolate(struct zone *zone, struct page *page,
int migratetype)
static bool __move_freepages_block_isolate(struct zone *zone,
struct page *page, bool isolate)
{
unsigned long start_pfn, pfn;
int from_mt;
int to_mt;
if (isolate == get_pageblock_isolate(page)) {
VM_WARN_ONCE(1, "%s a pageblock that is already in that state",
isolate ? "Isolate" : "Unisolate");
return false;
}
if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))
return false;
@@ -2084,7 +2107,7 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
del_page_from_free_list(buddy, zone, order,
get_pfnblock_migratetype(buddy, pfn));
set_pageblock_migratetype(page, migratetype);
toggle_pageblock_isolate(page, isolate);
split_large_buddy(zone, buddy, pfn, order, FPI_NONE);
return true;
}
@@ -2095,16 +2118,38 @@ bool move_freepages_block_isolate(struct zone *zone, struct page *page,
del_page_from_free_list(page, zone, order,
get_pfnblock_migratetype(page, pfn));
set_pageblock_migratetype(page, migratetype);
toggle_pageblock_isolate(page, isolate);
split_large_buddy(zone, page, pfn, order, FPI_NONE);
return true;
}
move:
__move_freepages_block(zone, start_pfn,
get_pfnblock_migratetype(page, start_pfn),
migratetype);
/* Use MIGRATETYPE_MASK to get non-isolate migratetype */
if (isolate) {
from_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
MIGRATETYPE_MASK);
to_mt = MIGRATE_ISOLATE;
} else {
from_mt = MIGRATE_ISOLATE;
to_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
MIGRATETYPE_MASK);
}
__move_freepages_block(zone, start_pfn, from_mt, to_mt);
toggle_pageblock_isolate(pfn_to_page(start_pfn), isolate);
return true;
}
bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page)
{
return __move_freepages_block_isolate(zone, page, true);
}
bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page)
{
return __move_freepages_block_isolate(zone, page, false);
}
#endif /* CONFIG_MEMORY_ISOLATION */
static void change_pageblock_range(struct page *pageblock_page,
@@ -2296,6 +2341,7 @@ try_to_claim_block(struct zone *zone, struct page *page,
if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
page_group_by_mobility_disabled) {
__move_freepages_block(zone, start_pfn, block_type, start_type);
set_pageblock_migratetype(pfn_to_page(start_pfn), start_type);
return __rmqueue_smallest(zone, order, start_type);
}
mm/page_isolation.c
+10 -11
View File
@@ -188,7 +188,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_
unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end,
migratetype, isol_flags);
if (!unmovable) {
if (!move_freepages_block_isolate(zone, page, MIGRATE_ISOLATE)) {
if (!pageblock_isolate_and_move_free_pages(zone, page)) {
spin_unlock_irqrestore(&zone->lock, flags);
return -EBUSY;
}
@@ -209,7 +209,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_
return -EBUSY;
}
static void unset_migratetype_isolate(struct page *page, int migratetype)
static void unset_migratetype_isolate(struct page *page)
{
struct zone *zone;
unsigned long flags;
@@ -262,10 +262,10 @@ static void unset_migratetype_isolate(struct page *page, int migratetype)
* Isolating this block already succeeded, so this
* should not fail on zone boundaries.
*/
WARN_ON_ONCE(!move_freepages_block_isolate(zone, page, migratetype));
WARN_ON_ONCE(!pageblock_unisolate_and_move_free_pages(zone, page));
} else {
set_pageblock_migratetype(page, migratetype);
__putback_isolated_page(page, order, migratetype);
clear_pageblock_isolate(page);
__putback_isolated_page(page, order, get_pageblock_migratetype(page));
}
zone->nr_isolate_pageblock--;
out:
@@ -383,7 +383,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
if (PageBuddy(page)) {
int order = buddy_order(page);
/* move_freepages_block_isolate() handled this */
/* pageblock_isolate_and_move_free_pages() handled this */
VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn);
pfn += 1UL << order;
@@ -433,7 +433,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
failed:
/* restore the original migratetype */
if (!skip_isolation)
unset_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype);
unset_migratetype_isolate(pfn_to_page(isolate_pageblock));
return -EBUSY;
}
@@ -504,7 +504,7 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
ret = isolate_single_pageblock(isolate_end, flags, true,
skip_isolation, migratetype);
if (ret) {
unset_migratetype_isolate(pfn_to_page(isolate_start), migratetype);
unset_migratetype_isolate(pfn_to_page(isolate_start));
return ret;
}
@@ -517,8 +517,7 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
start_pfn, end_pfn)) {
undo_isolate_page_range(isolate_start, pfn, migratetype);
unset_migratetype_isolate(
pfn_to_page(isolate_end - pageblock_nr_pages),
migratetype);
pfn_to_page(isolate_end - pageblock_nr_pages));
return -EBUSY;
}
}
@@ -548,7 +547,7 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
page = __first_valid_page(pfn, pageblock_nr_pages);
if (!page || !is_migrate_isolate_page(page))
continue;
unset_migratetype_isolate(page, migratetype);
unset_migratetype_isolate(page);
}
}
/*