344 static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,  in pcpu_next_md_free_region()  argument
347 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_md_free_region()
348 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_md_free_region()
373 			*bit_off = pcpu_block_off_to_off(i,  in pcpu_next_md_free_region()
381 		*bit_off = (i + 1) * PCPU_BITMAP_BLOCK_BITS - block->right_free;  in pcpu_next_md_free_region()
400 				 int align, int *bit_off, int *bits)  in pcpu_next_fit_region()  argument
402 	int i = pcpu_off_to_block_index(*bit_off);  in pcpu_next_fit_region()
403 	int block_off = pcpu_off_to_block_off(*bit_off);  in pcpu_next_fit_region()
432 			*bit_off = pcpu_block_off_to_off(i, start);  in pcpu_next_fit_region()
438 		*bit_off = ALIGN(PCPU_BITMAP_BLOCK_BITS - block->right_free,  in pcpu_next_fit_region()
440 		*bits = PCPU_BITMAP_BLOCK_BITS - *bit_off;  in pcpu_next_fit_region()
441 		*bit_off = pcpu_block_off_to_off(i, *bit_off);  in pcpu_next_fit_region()
447 	*bit_off = pcpu_chunk_map_bits(chunk);  in pcpu_next_fit_region()
456 #define pcpu_for_each_md_free_region(chunk, bit_off, bits)		\  argument
457 	for (pcpu_next_md_free_region((chunk), &(bit_off), &(bits));	\
458 	     (bit_off) < pcpu_chunk_map_bits((chunk));			\
459 	     (bit_off) += (bits) + 1,					\
460 	     pcpu_next_md_free_region((chunk), &(bit_off), &(bits)))
462 #define pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits)     \  argument
463 	for (pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
465 	     (bit_off) < pcpu_chunk_map_bits((chunk));			      \
466 	     (bit_off) += (bits),					      \
467 	     pcpu_next_fit_region((chunk), (alloc_bits), (align), &(bit_off), \
668 static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_scan()  argument
671 	int s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_scan()
679 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_scan()
704 	int bit_off, bits;  in pcpu_chunk_refresh_hint()  local
708 		bit_off = chunk_md->scan_hint_start + chunk_md->scan_hint;  in pcpu_chunk_refresh_hint()
713 		bit_off = chunk_md->first_free;  in pcpu_chunk_refresh_hint()
718 	pcpu_for_each_md_free_region(chunk, bit_off, bits)  in pcpu_chunk_refresh_hint()
719 		pcpu_block_update(chunk_md, bit_off, bit_off + bits);  in pcpu_chunk_refresh_hint()
765 static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_alloc()  argument
780 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_alloc()
781 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_alloc()
782 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_alloc()
783 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_alloc()
877 				bit_off,  in pcpu_block_update_hint_alloc()
878 				bit_off + bits))  in pcpu_block_update_hint_alloc()
889 				bit_off,  in pcpu_block_update_hint_alloc()
890 				bit_off + bits))  in pcpu_block_update_hint_alloc()
912 static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,  in pcpu_block_update_hint_free()  argument
927 	s_index = pcpu_off_to_block_index(bit_off);  in pcpu_block_update_hint_free()
928 	e_index = pcpu_off_to_block_index(bit_off + bits - 1);  in pcpu_block_update_hint_free()
929 	s_off = pcpu_off_to_block_off(bit_off);  in pcpu_block_update_hint_free()
930 	e_off = pcpu_off_to_block_off(bit_off + bits - 1) + 1;  in pcpu_block_update_hint_free()
1022 static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,  in pcpu_is_populated()  argument
1027 	page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1028 	page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);  in pcpu_is_populated()
1062 	int bit_off, bits, next_off;  in pcpu_find_block_fit()  local
1070 	bit_off = ALIGN(chunk_md->contig_hint_start, align) -  in pcpu_find_block_fit()
1072 	if (bit_off + alloc_bits > chunk_md->contig_hint)  in pcpu_find_block_fit()
1075 	bit_off = pcpu_next_hint(chunk_md, alloc_bits);  in pcpu_find_block_fit()
1077 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {  in pcpu_find_block_fit()
1078 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,  in pcpu_find_block_fit()
1082 		bit_off = next_off;  in pcpu_find_block_fit()
1086 	if (bit_off == pcpu_chunk_map_bits(chunk))  in pcpu_find_block_fit()
1089 	return bit_off;  in pcpu_find_block_fit()
1173 	int bit_off, end, oslot;  in pcpu_alloc_area()  local
1184 	bit_off = pcpu_find_zero_area(chunk->alloc_map, end, start, alloc_bits,  in pcpu_alloc_area()
1186 	if (bit_off >= end)  in pcpu_alloc_area()
1193 	bitmap_set(chunk->alloc_map, bit_off, alloc_bits);  in pcpu_alloc_area()
1196 	set_bit(bit_off, chunk->bound_map);  in pcpu_alloc_area()
1197 	bitmap_clear(chunk->bound_map, bit_off + 1, alloc_bits - 1);  in pcpu_alloc_area()
1198 	set_bit(bit_off + alloc_bits, chunk->bound_map);  in pcpu_alloc_area()
1203 	if (bit_off == chunk_md->first_free)  in pcpu_alloc_area()
1207 					bit_off + alloc_bits);  in pcpu_alloc_area()
1209 	pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);  in pcpu_alloc_area()
1213 	return bit_off * PCPU_MIN_ALLOC_SIZE;  in pcpu_alloc_area()
1230 	int bit_off, bits, end, oslot, freed;  in pcpu_free_area()  local
1237 	bit_off = off / PCPU_MIN_ALLOC_SIZE;  in pcpu_free_area()
1241 			    bit_off + 1);  in pcpu_free_area()
1242 	bits = end - bit_off;  in pcpu_free_area()
1243 	bitmap_clear(chunk->alloc_map, bit_off, bits);  in pcpu_free_area()
1251 	chunk_md->first_free = min(chunk_md->first_free, bit_off);  in pcpu_free_area()
1253 	pcpu_block_update_hint_free(chunk, bit_off, bits);  in pcpu_free_area()