Lines Matching full:cc
222 static inline bool isolation_suitable(struct compact_control *cc, in isolation_suitable() argument
225 if (cc->ignore_skip_hint) in isolation_suitable()
405 static bool test_and_set_skip(struct compact_control *cc, struct page *page, in test_and_set_skip() argument
411 if (cc->ignore_skip_hint) in test_and_set_skip()
418 if (!skip && !cc->no_set_skip_hint) in test_and_set_skip()
424 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) in update_cached_migrate() argument
426 struct zone *zone = cc->zone; in update_cached_migrate()
431 if (cc->no_set_skip_hint) in update_cached_migrate()
436 if (cc->mode != MIGRATE_ASYNC && in update_cached_migrate()
445 static void update_pageblock_skip(struct compact_control *cc, in update_pageblock_skip() argument
448 struct zone *zone = cc->zone; in update_pageblock_skip()
450 if (cc->no_set_skip_hint) in update_pageblock_skip()
463 static inline bool isolation_suitable(struct compact_control *cc, in isolation_suitable() argument
474 static inline void update_pageblock_skip(struct compact_control *cc, in update_pageblock_skip() argument
479 static void update_cached_migrate(struct compact_control *cc, unsigned long pfn) in update_cached_migrate() argument
483 static bool test_and_set_skip(struct compact_control *cc, struct page *page, in test_and_set_skip() argument
500 struct compact_control *cc) in compact_lock_irqsave() argument
504 if (cc->mode == MIGRATE_ASYNC && !cc->contended) { in compact_lock_irqsave()
508 cc->contended = true; in compact_lock_irqsave()
531 unsigned long flags, bool *locked, struct compact_control *cc) in compact_unlock_should_abort() argument
539 cc->contended = true; in compact_unlock_should_abort()
553 static unsigned long isolate_freepages_block(struct compact_control *cc, in isolate_freepages_block() argument
584 && compact_unlock_should_abort(&cc->zone->lock, flags, in isolate_freepages_block()
585 &locked, cc)) in isolate_freepages_block()
619 locked = compact_lock_irqsave(&cc->zone->lock, in isolate_freepages_block()
620 &flags, cc); in isolate_freepages_block()
635 cc->nr_freepages += isolated; in isolate_freepages_block()
638 if (!strict && cc->nr_migratepages <= cc->nr_freepages) { in isolate_freepages_block()
656 spin_unlock_irqrestore(&cc->zone->lock, flags); in isolate_freepages_block()
679 cc->total_free_scanned += nr_scanned; in isolate_freepages_block()
687 * @cc: Compaction control structure.
700 isolate_freepages_range(struct compact_control *cc, in isolate_freepages_range() argument
708 if (block_start_pfn < cc->zone->zone_start_pfn) in isolate_freepages_range()
709 block_start_pfn = cc->zone->zone_start_pfn; in isolate_freepages_range()
732 block_end_pfn, cc->zone)) in isolate_freepages_range()
735 isolated = isolate_freepages_block(cc, &isolate_start_pfn, in isolate_freepages_range()
809 * @cc: Compaction control structure.
820 * The pages are isolated on cc->migratepages list (not required to be empty),
821 * and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field
825 isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, in isolate_migratepages_block() argument
828 pg_data_t *pgdat = cc->zone->zone_pgdat; in isolate_migratepages_block()
846 if (cc->nr_migratepages) in isolate_migratepages_block()
850 if (cc->mode == MIGRATE_ASYNC) in isolate_migratepages_block()
861 if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) { in isolate_migratepages_block()
863 next_skip_pfn = block_end_pfn(low_pfn, cc->order); in isolate_migratepages_block()
888 next_skip_pfn = block_end_pfn(low_pfn, cc->order); in isolate_migratepages_block()
898 flags, &locked, cc)) { in isolate_migratepages_block()
916 if (!cc->ignore_skip_hint && get_pageblock_skip(page)) { in isolate_migratepages_block()
950 if (PageCompound(page) && !cc->alloc_contig) { in isolate_migratepages_block()
996 if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page)) in isolate_migratepages_block()
1002 &flags, cc); in isolate_migratepages_block()
1007 if (test_and_set_skip(cc, page, low_pfn)) in isolate_migratepages_block()
1020 if (unlikely(PageCompound(page) && !cc->alloc_contig)) { in isolate_migratepages_block()
1043 list_add(&page->lru, &cc->migratepages); in isolate_migratepages_block()
1044 cc->nr_migratepages += compound_nr(page); in isolate_migratepages_block()
1053 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX && in isolate_migratepages_block()
1054 !cc->rescan && !cc->contended) { in isolate_migratepages_block()
1066 * instead of migrating, as we cannot form the cc->order buddy in isolate_migratepages_block()
1074 putback_movable_pages(&cc->migratepages); in isolate_migratepages_block()
1075 cc->nr_migratepages = 0; in isolate_migratepages_block()
1085 next_skip_pfn += 1UL << cc->order; in isolate_migratepages_block()
1108 if (low_pfn == end_pfn && (!nr_isolated || cc->rescan)) { in isolate_migratepages_block()
1111 update_cached_migrate(cc, low_pfn); in isolate_migratepages_block()
1118 cc->total_migrate_scanned += nr_scanned; in isolate_migratepages_block()
1127 * @cc: Compaction control structure.
1136 isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, in isolate_migratepages_range() argument
1144 if (block_start_pfn < cc->zone->zone_start_pfn) in isolate_migratepages_range()
1145 block_start_pfn = cc->zone->zone_start_pfn; in isolate_migratepages_range()
1155 block_end_pfn, cc->zone)) in isolate_migratepages_range()
1158 pfn = isolate_migratepages_block(cc, pfn, block_end_pfn, in isolate_migratepages_range()
1164 if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX) in isolate_migratepages_range()
1174 static bool suitable_migration_source(struct compact_control *cc, in suitable_migration_source() argument
1182 if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) in suitable_migration_source()
1187 if (cc->migratetype == MIGRATE_MOVABLE) in suitable_migration_source()
1190 return block_mt == cc->migratetype; in suitable_migration_source()
1194 static bool suitable_migration_target(struct compact_control *cc, in suitable_migration_target() argument
1208 if (cc->ignore_block_suitable) in suitable_migration_target()
1220 freelist_scan_limit(struct compact_control *cc) in freelist_scan_limit() argument
1224 return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1; in freelist_scan_limit()
1231 static inline bool compact_scanners_met(struct compact_control *cc) in compact_scanners_met() argument
1233 return (cc->free_pfn >> pageblock_order) in compact_scanners_met()
1234 <= (cc->migrate_pfn >> pageblock_order); in compact_scanners_met()
1273 fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated) in fast_isolate_around() argument
1279 if (cc->nr_freepages >= cc->nr_migratepages) in fast_isolate_around()
1283 if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC) in fast_isolate_around()
1287 start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn); in fast_isolate_around()
1288 end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)); in fast_isolate_around()
1290 page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone); in fast_isolate_around()
1296 isolate_freepages_block(cc, &start_pfn, pfn, &cc->freepages, 1, false); in fast_isolate_around()
1297 if (cc->nr_freepages >= cc->nr_migratepages) in fast_isolate_around()
1304 isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false); in fast_isolate_around()
1307 if (cc->nr_freepages < cc->nr_migratepages) in fast_isolate_around()
1312 static int next_search_order(struct compact_control *cc, int order) in next_search_order() argument
1316 order = cc->order - 1; in next_search_order()
1319 if (order == cc->search_order) { in next_search_order()
1320 cc->search_order--; in next_search_order()
1321 if (cc->search_order < 0) in next_search_order()
1322 cc->search_order = cc->order - 1; in next_search_order()
1330 fast_isolate_freepages(struct compact_control *cc) in fast_isolate_freepages() argument
1332 unsigned int limit = min(1U, freelist_scan_limit(cc) >> 1); in fast_isolate_freepages()
1342 if (cc->order <= 0) in fast_isolate_freepages()
1343 return cc->free_pfn; in fast_isolate_freepages()
1349 if (cc->free_pfn >= cc->zone->compact_init_free_pfn) { in fast_isolate_freepages()
1358 distance = (cc->free_pfn - cc->migrate_pfn); in fast_isolate_freepages()
1359 low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2)); in fast_isolate_freepages()
1360 min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1)); in fast_isolate_freepages()
1369 cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order); in fast_isolate_freepages()
1371 for (order = cc->search_order; in fast_isolate_freepages()
1373 order = next_search_order(cc, order)) { in fast_isolate_freepages()
1374 struct free_area *area = &cc->zone->free_area[order]; in fast_isolate_freepages()
1384 spin_lock_irqsave(&cc->zone->lock, flags); in fast_isolate_freepages()
1395 cc->zone->zone_start_pfn); in fast_isolate_freepages()
1398 cc->fast_search_fail = 0; in fast_isolate_freepages()
1399 cc->search_order = order; in fast_isolate_freepages()
1431 cc->nr_freepages += nr_isolated; in fast_isolate_freepages()
1432 list_add_tail(&page->lru, &cc->freepages); in fast_isolate_freepages()
1436 order = cc->search_order + 1; in fast_isolate_freepages()
1441 spin_unlock_irqrestore(&cc->zone->lock, flags); in fast_isolate_freepages()
1452 cc->fast_search_fail++; in fast_isolate_freepages()
1461 cc->free_pfn = highest; in fast_isolate_freepages()
1463 if (cc->direct_compaction && pfn_valid(min_pfn)) { in fast_isolate_freepages()
1466 zone_end_pfn(cc->zone)), in fast_isolate_freepages()
1467 cc->zone); in fast_isolate_freepages()
1468 cc->free_pfn = min_pfn; in fast_isolate_freepages()
1474 if (highest && highest >= cc->zone->compact_cached_free_pfn) { in fast_isolate_freepages()
1476 cc->zone->compact_cached_free_pfn = highest; in fast_isolate_freepages()
1479 cc->total_free_scanned += nr_scanned; in fast_isolate_freepages()
1481 return cc->free_pfn; in fast_isolate_freepages()
1484 fast_isolate_around(cc, low_pfn, nr_isolated); in fast_isolate_freepages()
1492 static void isolate_freepages(struct compact_control *cc) in isolate_freepages() argument
1494 struct zone *zone = cc->zone; in isolate_freepages()
1500 struct list_head *freelist = &cc->freepages; in isolate_freepages()
1504 isolate_start_pfn = fast_isolate_freepages(cc); in isolate_freepages()
1505 if (cc->nr_freepages) in isolate_freepages()
1519 isolate_start_pfn = cc->free_pfn; in isolate_freepages()
1523 low_pfn = pageblock_end_pfn(cc->migrate_pfn); in isolate_freepages()
1524 stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1; in isolate_freepages()
1528 * pages on cc->migratepages. We stop searching if the migrate in isolate_freepages()
1550 if (!suitable_migration_target(cc, page)) in isolate_freepages()
1554 if (!isolation_suitable(cc, page)) in isolate_freepages()
1558 nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn, in isolate_freepages()
1563 update_pageblock_skip(cc, page, block_start_pfn); in isolate_freepages()
1566 if (cc->nr_freepages >= cc->nr_migratepages) { in isolate_freepages()
1598 cc->free_pfn = isolate_start_pfn; in isolate_freepages()
1612 struct compact_control *cc = (struct compact_control *)data; in compaction_alloc() local
1615 if (list_empty(&cc->freepages)) { in compaction_alloc()
1616 isolate_freepages(cc); in compaction_alloc()
1618 if (list_empty(&cc->freepages)) in compaction_alloc()
1622 freepage = list_entry(cc->freepages.next, struct page, lru); in compaction_alloc()
1624 cc->nr_freepages--; in compaction_alloc()
1636 struct compact_control *cc = (struct compact_control *)data; in compaction_free() local
1638 list_add(&page->lru, &cc->freepages); in compaction_free()
1639 cc->nr_freepages++; in compaction_free()
1660 update_fast_start_pfn(struct compact_control *cc, unsigned long pfn) in update_fast_start_pfn() argument
1662 if (cc->fast_start_pfn == ULONG_MAX) in update_fast_start_pfn()
1665 if (!cc->fast_start_pfn) in update_fast_start_pfn()
1666 cc->fast_start_pfn = pfn; in update_fast_start_pfn()
1668 cc->fast_start_pfn = min(cc->fast_start_pfn, pfn); in update_fast_start_pfn()
1672 reinit_migrate_pfn(struct compact_control *cc) in reinit_migrate_pfn() argument
1674 if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX) in reinit_migrate_pfn()
1675 return cc->migrate_pfn; in reinit_migrate_pfn()
1677 cc->migrate_pfn = cc->fast_start_pfn; in reinit_migrate_pfn()
1678 cc->fast_start_pfn = ULONG_MAX; in reinit_migrate_pfn()
1680 return cc->migrate_pfn; in reinit_migrate_pfn()
1688 static unsigned long fast_find_migrateblock(struct compact_control *cc) in fast_find_migrateblock() argument
1690 unsigned int limit = freelist_scan_limit(cc); in fast_find_migrateblock()
1693 unsigned long pfn = cc->migrate_pfn; in fast_find_migrateblock()
1699 if (cc->ignore_skip_hint) in fast_find_migrateblock()
1707 if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn)) in fast_find_migrateblock()
1715 if (cc->order <= PAGE_ALLOC_COSTLY_ORDER) in fast_find_migrateblock()
1724 if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE) in fast_find_migrateblock()
1733 distance = (cc->free_pfn - cc->migrate_pfn) >> 1; in fast_find_migrateblock()
1734 if (cc->migrate_pfn != cc->zone->zone_start_pfn) in fast_find_migrateblock()
1736 high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance); in fast_find_migrateblock()
1738 for (order = cc->order - 1; in fast_find_migrateblock()
1741 struct free_area *area = &cc->zone->free_area[order]; in fast_find_migrateblock()
1749 spin_lock_irqsave(&cc->zone->lock, flags); in fast_find_migrateblock()
1773 update_fast_start_pfn(cc, free_pfn); in fast_find_migrateblock()
1775 if (pfn < cc->zone->zone_start_pfn) in fast_find_migrateblock()
1776 pfn = cc->zone->zone_start_pfn; in fast_find_migrateblock()
1777 cc->fast_search_fail = 0; in fast_find_migrateblock()
1783 spin_unlock_irqrestore(&cc->zone->lock, flags); in fast_find_migrateblock()
1786 cc->total_migrate_scanned += nr_scanned; in fast_find_migrateblock()
1793 cc->fast_search_fail++; in fast_find_migrateblock()
1794 pfn = reinit_migrate_pfn(cc); in fast_find_migrateblock()
1804 static isolate_migrate_t isolate_migratepages(struct compact_control *cc) in isolate_migratepages() argument
1812 (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0); in isolate_migratepages()
1820 low_pfn = fast_find_migrateblock(cc); in isolate_migratepages()
1822 if (block_start_pfn < cc->zone->zone_start_pfn) in isolate_migratepages()
1823 block_start_pfn = cc->zone->zone_start_pfn; in isolate_migratepages()
1830 fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail; in isolate_migratepages()
1839 for (; block_end_pfn <= cc->free_pfn; in isolate_migratepages()
1854 block_end_pfn, cc->zone); in isolate_migratepages()
1866 !fast_find_block && !isolation_suitable(cc, page)) in isolate_migratepages()
1877 if (!suitable_migration_source(cc, page)) { in isolate_migratepages()
1878 update_cached_migrate(cc, block_end_pfn); in isolate_migratepages()
1883 low_pfn = isolate_migratepages_block(cc, low_pfn, in isolate_migratepages()
1898 cc->migrate_pfn = low_pfn; in isolate_migratepages()
1900 return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE; in isolate_migratepages()
1990 static enum compact_result __compact_finished(struct compact_control *cc) in __compact_finished() argument
1993 const int migratetype = cc->migratetype; in __compact_finished()
1997 if (compact_scanners_met(cc)) { in __compact_finished()
1999 reset_cached_positions(cc->zone); in __compact_finished()
2007 if (cc->direct_compaction) in __compact_finished()
2008 cc->zone->compact_blockskip_flush = true; in __compact_finished()
2010 if (cc->whole_zone) in __compact_finished()
2016 if (cc->proactive_compaction) { in __compact_finished()
2020 pgdat = cc->zone->zone_pgdat; in __compact_finished()
2024 score = fragmentation_score_zone(cc->zone); in __compact_finished()
2035 if (is_via_compact_memory(cc->order)) in __compact_finished()
2044 if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages)) in __compact_finished()
2049 for (order = cc->order; order < MAX_ORDER; order++) { in __compact_finished()
2050 struct free_area *area = &cc->zone->free_area[order]; in __compact_finished()
2082 if (cc->mode == MIGRATE_ASYNC || in __compact_finished()
2083 IS_ALIGNED(cc->migrate_pfn, in __compact_finished()
2094 if (cc->contended || fatal_signal_pending(current)) in __compact_finished()
2100 static enum compact_result compact_finished(struct compact_control *cc) in compact_finished() argument
2104 ret = __compact_finished(cc); in compact_finished()
2105 trace_mm_compaction_finished(cc->zone, cc->order, ret); in compact_finished()
2233 compact_zone(struct compact_control *cc, struct capture_control *capc) in compact_zone() argument
2236 unsigned long start_pfn = cc->zone->zone_start_pfn; in compact_zone()
2237 unsigned long end_pfn = zone_end_pfn(cc->zone); in compact_zone()
2239 const bool sync = cc->mode != MIGRATE_ASYNC; in compact_zone()
2246 cc->total_migrate_scanned = 0; in compact_zone()
2247 cc->total_free_scanned = 0; in compact_zone()
2248 cc->nr_migratepages = 0; in compact_zone()
2249 cc->nr_freepages = 0; in compact_zone()
2250 INIT_LIST_HEAD(&cc->freepages); in compact_zone()
2251 INIT_LIST_HEAD(&cc->migratepages); in compact_zone()
2253 cc->migratetype = gfp_migratetype(cc->gfp_mask); in compact_zone()
2254 ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags, in compact_zone()
2255 cc->highest_zoneidx); in compact_zone()
2267 if (compaction_restarting(cc->zone, cc->order)) in compact_zone()
2268 __reset_isolation_suitable(cc->zone); in compact_zone()
2276 cc->fast_start_pfn = 0; in compact_zone()
2277 if (cc->whole_zone) { in compact_zone()
2278 cc->migrate_pfn = start_pfn; in compact_zone()
2279 cc->free_pfn = pageblock_start_pfn(end_pfn - 1); in compact_zone()
2281 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync]; in compact_zone()
2282 cc->free_pfn = cc->zone->compact_cached_free_pfn; in compact_zone()
2283 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) { in compact_zone()
2284 cc->free_pfn = pageblock_start_pfn(end_pfn - 1); in compact_zone()
2285 cc->zone->compact_cached_free_pfn = cc->free_pfn; in compact_zone()
2287 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) { in compact_zone()
2288 cc->migrate_pfn = start_pfn; in compact_zone()
2289 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; in compact_zone()
2290 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; in compact_zone()
2293 if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn) in compact_zone()
2294 cc->whole_zone = true; in compact_zone()
2308 cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1]; in compact_zone()
2310 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, in compact_zone()
2311 cc->free_pfn, end_pfn, sync); in compact_zone()
2316 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) { in compact_zone()
2318 unsigned long start_pfn = cc->migrate_pfn; in compact_zone()
2328 cc->rescan = false; in compact_zone()
2331 cc->rescan = true; in compact_zone()
2334 switch (isolate_migratepages(cc)) { in compact_zone()
2337 putback_movable_pages(&cc->migratepages); in compact_zone()
2338 cc->nr_migratepages = 0; in compact_zone()
2342 cc->zone->compact_cached_migrate_pfn[1] = in compact_zone()
2343 cc->zone->compact_cached_migrate_pfn[0]; in compact_zone()
2349 * previous cc->order aligned block. in compact_zone()
2358 err = migrate_pages(&cc->migratepages, compaction_alloc, in compact_zone()
2359 compaction_free, (unsigned long)cc, cc->mode, in compact_zone()
2362 trace_mm_compaction_migratepages(cc->nr_migratepages, err, in compact_zone()
2363 &cc->migratepages); in compact_zone()
2366 cc->nr_migratepages = 0; in compact_zone()
2368 putback_movable_pages(&cc->migratepages); in compact_zone()
2373 if (err == -ENOMEM && !compact_scanners_met(cc)) { in compact_zone()
2381 if (cc->direct_compaction && in compact_zone()
2382 (cc->mode == MIGRATE_ASYNC)) { in compact_zone()
2383 cc->migrate_pfn = block_end_pfn( in compact_zone()
2384 cc->migrate_pfn - 1, cc->order); in compact_zone()
2393 * cc->order aligned block where we migrated from? If yes, in compact_zone()
2398 if (cc->order > 0 && last_migrated_pfn) { in compact_zone()
2400 block_start_pfn(cc->migrate_pfn, cc->order); in compact_zone()
2403 lru_add_drain_cpu_zone(cc->zone); in compact_zone()
2421 if (cc->nr_freepages > 0) { in compact_zone()
2422 unsigned long free_pfn = release_freepages(&cc->freepages); in compact_zone()
2424 cc->nr_freepages = 0; in compact_zone()
2432 if (free_pfn > cc->zone->compact_cached_free_pfn) in compact_zone()
2433 cc->zone->compact_cached_free_pfn = free_pfn; in compact_zone()
2436 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned); in compact_zone()
2437 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned); in compact_zone()
2439 trace_mm_compaction_end(start_pfn, cc->migrate_pfn, in compact_zone()
2440 cc->free_pfn, end_pfn, sync, ret); in compact_zone()
2451 struct compact_control cc = { in compact_zone_order() local
2466 .cc = &cc, in compact_zone_order()
2478 ret = compact_zone(&cc, &capc); in compact_zone_order()
2480 VM_BUG_ON(!list_empty(&cc.freepages)); in compact_zone_order()
2481 VM_BUG_ON(!list_empty(&cc.migratepages)); in compact_zone_order()
2588 struct compact_control cc = { in proactive_compact_node() local
2602 cc.zone = zone; in proactive_compact_node()
2604 compact_zone(&cc, NULL); in proactive_compact_node()
2606 VM_BUG_ON(!list_empty(&cc.freepages)); in proactive_compact_node()
2607 VM_BUG_ON(!list_empty(&cc.migratepages)); in proactive_compact_node()
2617 struct compact_control cc = { in compact_node() local
2632 cc.zone = zone; in compact_node()
2634 compact_zone(&cc, NULL); in compact_node()
2636 VM_BUG_ON(!list_empty(&cc.freepages)); in compact_node()
2637 VM_BUG_ON(!list_empty(&cc.migratepages)); in compact_node()
2763 struct compact_control cc = { in kcompactd_do_work() local
2771 trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order, in kcompactd_do_work()
2772 cc.highest_zoneidx); in kcompactd_do_work()
2775 for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) { in kcompactd_do_work()
2782 if (compaction_deferred(zone, cc.order)) in kcompactd_do_work()
2785 if (compaction_suitable(zone, cc.order, 0, zoneid) != in kcompactd_do_work()
2792 cc.zone = zone; in kcompactd_do_work()
2793 status = compact_zone(&cc, NULL); in kcompactd_do_work()
2796 compaction_defer_reset(zone, cc.order, false); in kcompactd_do_work()
2801 * order >= cc.order. This is ratelimited by the in kcompactd_do_work()
2810 defer_compaction(zone, cc.order); in kcompactd_do_work()
2814 cc.total_migrate_scanned); in kcompactd_do_work()
2816 cc.total_free_scanned); in kcompactd_do_work()
2818 VM_BUG_ON(!list_empty(&cc.freepages)); in kcompactd_do_work()
2819 VM_BUG_ON(!list_empty(&cc.migratepages)); in kcompactd_do_work()
2827 if (pgdat->kcompactd_max_order <= cc.order) in kcompactd_do_work()
2829 if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx) in kcompactd_do_work()