Lines Matching full:ac
352 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
1691 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
1694 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
1697 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
1698 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
1700 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
1701 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
1702 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
1706 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
1708 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
1709 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
1710 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
1719 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
1720 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
1721 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
1722 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
1724 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
1726 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
1727 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
1735 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_use_best_found()
1736 ext4_mb_new_preallocation(ac); in ext4_mb_use_best_found()
1740 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
1744 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
1745 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
1746 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
1750 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
1755 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
1756 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
1757 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
1767 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
1774 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
1790 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
1794 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
1795 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
1798 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1799 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
1800 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
1802 ac->ac_found++; in ext4_mb_measure_extent()
1807 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
1809 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1818 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
1846 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
1850 int ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
1853 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
1859 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
1863 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1867 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
1868 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
1871 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
1878 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
1881 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
1884 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
1885 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
1888 if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) in ext4_mb_find_by_goal()
1893 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
1902 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1903 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
1904 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
1907 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { in ext4_mb_find_by_goal()
1910 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + in ext4_mb_find_by_goal()
1914 ac->ac_found++; in ext4_mb_find_by_goal()
1915 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1916 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1918 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
1920 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1921 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1922 ac->ac_found++; in ext4_mb_find_by_goal()
1923 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1924 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1925 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
1929 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
1930 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
1931 ac->ac_found++; in ext4_mb_find_by_goal()
1932 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
1933 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
1935 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
1946 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
1949 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
1956 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
1957 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { in ext4_mb_simple_scan_group()
1966 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, in ext4_mb_simple_scan_group()
1969 ext4_mark_group_bitmap_corrupted(ac->ac_sb, in ext4_mb_simple_scan_group()
1974 ac->ac_found++; in ext4_mb_simple_scan_group()
1976 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
1977 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
1978 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
1980 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
1982 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
1997 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
2000 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
2012 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
2030 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
2048 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
2054 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
2062 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
2065 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2087 ac->ac_found++; in ext4_mb_scan_aligned()
2089 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2090 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
2103 static bool ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
2107 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2108 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2125 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2128 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2133 if (free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2136 if (ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) in ext4_mb_good_group()
2139 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2144 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2148 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2165 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, in ext4_mb_good_group_nolock() argument
2168 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group_nolock()
2169 struct super_block *sb = ac->ac_sb; in ext4_mb_good_group_nolock()
2171 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; in ext4_mb_good_group_nolock()
2180 if (cr <= 2 && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group_nolock()
2214 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_good_group_nolock()
2300 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
2311 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2315 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2318 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2321 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
2322 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2325 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2329 * ac->ac_2order is set only if the fe_len is a power of 2 in ext4_mb_regular_allocator()
2330 * if ac->ac_2order is set we also set criteria to 0 so that we in ext4_mb_regular_allocator()
2333 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2334 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2346 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) in ext4_mb_regular_allocator()
2347 ac->ac_2order = array_index_nospec(i - 1, in ext4_mb_regular_allocator()
2352 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2355 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2356 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2361 cr = ac->ac_2order ? 0 : 1; in ext4_mb_regular_allocator()
2367 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2368 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2373 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2410 ret = ext4_mb_good_group_nolock(ac, group, cr); in ext4_mb_regular_allocator()
2427 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2434 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2436 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2438 !(ac->ac_g_ex.fe_len % sbi->s_stripe)) in ext4_mb_regular_allocator()
2439 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2441 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2446 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2451 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2452 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2457 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2458 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2466 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, in ext4_mb_regular_allocator()
2467 ac->ac_b_ex.fe_len, lost); in ext4_mb_regular_allocator()
2469 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2470 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2471 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2472 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2473 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2479 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2483 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, in ext4_mb_regular_allocator()
2484 ac->ac_flags, cr, err); in ext4_mb_regular_allocator()
3192 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3196 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
3207 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
3208 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
3210 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
3213 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3226 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
3230 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
3238 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
3240 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3241 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { in ext4_mb_mark_diskspace_used()
3248 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3249 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3250 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3251 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3258 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3262 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
3263 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
3268 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3269 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3275 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
3277 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
3279 ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); in ext4_mb_mark_diskspace_used()
3280 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
3282 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3283 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3287 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
3294 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3295 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
3423 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
3425 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
3426 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
3429 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
3430 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
3438 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
3441 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
3447 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_normalize_request()
3452 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
3456 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
3461 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
3464 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
3465 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
3469 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
3473 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_normalize_request()
3475 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
3476 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
3503 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3507 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3510 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, in ext4_mb_normalize_request()
3512 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
3516 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
3517 size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3518 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
3529 start = max(start, rounddown(ac->ac_o_ex.fe_logical, in ext4_mb_normalize_request()
3530 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); in ext4_mb_normalize_request()
3544 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
3545 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
3562 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3566 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || in ext4_mb_normalize_request()
3567 ac->ac_o_ex.fe_logical < pa->pa_lstart)); in ext4_mb_normalize_request()
3577 if (pa_end <= ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3580 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3596 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
3604 if (start + size <= ac->ac_o_ex.fe_logical && in ext4_mb_normalize_request()
3605 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
3606 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
3609 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
3612 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
3618 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
3619 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
3624 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
3625 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
3626 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
3627 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3631 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
3632 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
3633 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
3634 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
3637 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, in ext4_mb_normalize_request()
3641 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
3643 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
3645 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { in ext4_mb_collect_stats()
3647 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
3648 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
3650 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
3651 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
3652 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
3654 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
3658 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
3659 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
3661 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
3668 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3670 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
3672 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
3677 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
3679 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
3689 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3690 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
3691 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
3692 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
3697 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
3703 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
3706 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
3712 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
3714 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
3716 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
3717 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
3718 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
3719 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
3720 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
3727 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); in ext4_mb_use_inode_pa()
3733 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
3736 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
3738 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
3739 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
3740 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
3741 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
3742 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
3743 ac->ac_pa = pa; in ext4_mb_use_group_pa()
3751 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", in ext4_mb_use_group_pa()
3788 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
3790 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
3792 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
3798 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
3807 if (ac->ac_o_ex.fe_logical < pa->pa_lstart || in ext4_mb_use_preallocated()
3808 ac->ac_o_ex.fe_logical >= (pa->pa_lstart + in ext4_mb_use_preallocated()
3813 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
3822 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_use_preallocated()
3824 ac->ac_criteria = 10; in ext4_mb_use_preallocated()
3833 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
3837 lg = ac->ac_lg; in ext4_mb_use_preallocated()
3840 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
3845 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
3856 pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
3866 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
3867 ac->ac_criteria = 20; in ext4_mb_use_preallocated()
3972 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
4032 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
4034 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
4041 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4042 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
4043 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
4044 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_inode_pa()
4046 pa = ac->ac_pa; in ext4_mb_new_inode_pa()
4048 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { in ext4_mb_new_inode_pa()
4057 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
4058 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
4063 winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
4066 wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
4071 offs = ac->ac_o_ex.fe_logical % in ext4_mb_new_inode_pa()
4072 EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4076 ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - in ext4_mb_new_inode_pa()
4078 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
4079 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4084 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_inode_pa()
4086 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
4087 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
4088 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
4098 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
4100 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
4103 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
4104 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
4107 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
4121 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
4123 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
4129 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
4130 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
4131 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
4132 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_group_pa()
4134 pa = ac->ac_pa; in ext4_mb_new_group_pa()
4138 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_group_pa()
4140 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
4142 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
4152 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
4154 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
4157 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
4158 lg = ac->ac_lg; in ext4_mb_new_group_pa()
4172 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
4174 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
4175 ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
4177 ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
4481 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) in ext4_mb_pa_alloc() argument
4490 ac->ac_pa = pa; in ext4_mb_pa_alloc()
4494 static void ext4_mb_pa_free(struct ext4_allocation_context *ac) in ext4_mb_pa_free() argument
4496 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_pa_free()
4499 ac->ac_pa = NULL; in ext4_mb_pa_free()
4536 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
4538 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
4546 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
4550 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
4551 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
4552 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
4553 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
4554 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
4555 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
4556 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
4557 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
4558 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
4559 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
4560 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
4561 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
4562 (int)ac->ac_criteria); in ext4_mb_show_ac()
4563 mb_debug(sb, "%u found", ac->ac_found); in ext4_mb_show_ac()
4571 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
4573 ext4_mb_show_pa(ac->ac_sb); in ext4_mb_show_ac()
4585 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
4587 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
4588 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
4591 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
4594 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
4597 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_group_or_file()
4598 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
4602 !inode_is_open_for_write(ac->ac_inode)) { in ext4_mb_group_or_file()
4603 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
4608 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4615 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
4619 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
4625 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
4628 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
4631 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
4635 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
4661 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
4662 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
4663 ac->ac_sb = sb; in ext4_mb_initialize_context()
4664 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
4665 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
4666 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
4667 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
4668 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
4669 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
4670 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
4674 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
4676 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " in ext4_mb_initialize_context()
4679 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
4773 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
4776 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
4777 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
4778 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
4841 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
4843 struct inode *inode = ac->ac_inode; in ext4_mb_release_context()
4845 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
4846 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
4851 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4852 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
4853 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4854 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
4867 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
4881 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
4883 if (ac->ac_bitmap_page) in ext4_mb_release_context()
4884 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
4885 if (ac->ac_buddy_page) in ext4_mb_release_context()
4886 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
4887 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
4888 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
4889 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
4922 struct ext4_allocation_context *ac, u64 *seq) in ext4_mb_discard_preallocations_should_retry() argument
4928 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_discard_preallocations_should_retry()
4934 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { in ext4_mb_discard_preallocations_should_retry()
4935 ac->ac_flags |= EXT4_MB_STRICT_CHECK; in ext4_mb_discard_preallocations_should_retry()
4956 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
5014 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
5015 if (!ac) { in ext4_mb_new_blocks()
5021 *errp = ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
5027 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
5029 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
5030 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
5031 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
5033 *errp = ext4_mb_pa_alloc(ac); in ext4_mb_new_blocks()
5038 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
5042 * ac->ac_status == AC_STATUS_FOUND. in ext4_mb_new_blocks()
5043 * And error from above mean ac->ac_status != AC_STATUS_FOUND in ext4_mb_new_blocks()
5047 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5048 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
5051 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
5052 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) in ext4_mb_new_blocks()
5053 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5055 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
5056 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
5058 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
5061 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
5062 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
5066 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) in ext4_mb_new_blocks()
5072 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5078 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
5080 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
5082 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
5084 if (ac) in ext4_mb_new_blocks()
5085 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()