651 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)  in kvm_destroy_dirty_bitmap()  argument
653 	if (!memslot->dirty_bitmap)  in kvm_destroy_dirty_bitmap()
656 	kvfree(memslot->dirty_bitmap);  in kvm_destroy_dirty_bitmap()
657 	memslot->dirty_bitmap = NULL;  in kvm_destroy_dirty_bitmap()
672 	struct kvm_memory_slot *memslot;  in kvm_free_memslots()  local
677 	kvm_for_each_memslot(memslot, slots)  in kvm_free_memslots()
678 		kvm_free_memslot(kvm, memslot);  in kvm_free_memslots()
965 static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)  in kvm_alloc_dirty_bitmap()  argument
967 	unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);  in kvm_alloc_dirty_bitmap()
969 	memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);  in kvm_alloc_dirty_bitmap()
970 	if (!memslot->dirty_bitmap)  in kvm_alloc_dirty_bitmap()
981 				      struct kvm_memory_slot *memslot)  in kvm_memslot_delete()  argument
986 	if (WARN_ON(slots->id_to_index[memslot->id] == -1))  in kvm_memslot_delete()
994 	for (i = slots->id_to_index[memslot->id]; i < slots->used_slots; i++) {  in kvm_memslot_delete()
998 	mslots[i] = *memslot;  in kvm_memslot_delete()
999 	slots->id_to_index[memslot->id] = -1;  in kvm_memslot_delete()
1019 					    struct kvm_memory_slot *memslot)  in kvm_memslot_move_backward()  argument
1024 	if (WARN_ON_ONCE(slots->id_to_index[memslot->id] == -1) ||  in kvm_memslot_move_backward()
1033 	for (i = slots->id_to_index[memslot->id]; i < slots->used_slots - 1; i++) {  in kvm_memslot_move_backward()
1034 		if (memslot->base_gfn > mslots[i + 1].base_gfn)  in kvm_memslot_move_backward()
1037 		WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn);  in kvm_memslot_move_backward()
1054 					   struct kvm_memory_slot *memslot,  in kvm_memslot_move_forward()  argument
1061 		if (memslot->base_gfn < mslots[i - 1].base_gfn)  in kvm_memslot_move_forward()
1064 		WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn);  in kvm_memslot_move_forward()
1115 			    struct kvm_memory_slot *memslot,  in update_memslots()  argument
1121 		kvm_memslot_delete(slots, memslot);  in update_memslots()
1126 			i = kvm_memslot_move_backward(slots, memslot);  in update_memslots()
1127 		i = kvm_memslot_move_forward(slots, memslot, i);  in update_memslots()
1133 		slots->memslots[i] = *memslot;  in update_memslots()
1134 		slots->id_to_index[memslot->id] = i;  in update_memslots()
1459 		      int *is_dirty, struct kvm_memory_slot **memslot)  in kvm_get_dirty_log()  argument
1466 	*memslot = NULL;  in kvm_get_dirty_log()
1475 	*memslot = id_to_memslot(slots, id);  in kvm_get_dirty_log()
1476 	if (!(*memslot) || !(*memslot)->dirty_bitmap)  in kvm_get_dirty_log()
1479 	kvm_arch_sync_dirty_log(kvm, *memslot);  in kvm_get_dirty_log()
1481 	n = kvm_dirty_bitmap_bytes(*memslot);  in kvm_get_dirty_log()
1484 		any = (*memslot)->dirty_bitmap[i];  in kvm_get_dirty_log()
1486 	if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))  in kvm_get_dirty_log()
1520 	struct kvm_memory_slot *memslot;  in kvm_get_dirty_log_protect()  local
1533 	memslot = id_to_memslot(slots, id);  in kvm_get_dirty_log_protect()
1534 	if (!memslot || !memslot->dirty_bitmap)  in kvm_get_dirty_log_protect()
1537 	dirty_bitmap = memslot->dirty_bitmap;  in kvm_get_dirty_log_protect()
1539 	kvm_arch_sync_dirty_log(kvm, memslot);  in kvm_get_dirty_log_protect()
1541 	n = kvm_dirty_bitmap_bytes(memslot);  in kvm_get_dirty_log_protect()
1554 		dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);  in kvm_get_dirty_log_protect()
1570 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,  in kvm_get_dirty_log_protect()
1577 		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);  in kvm_get_dirty_log_protect()
1627 	struct kvm_memory_slot *memslot;  in kvm_clear_dirty_log_protect()  local
1644 	memslot = id_to_memslot(slots, id);  in kvm_clear_dirty_log_protect()
1645 	if (!memslot || !memslot->dirty_bitmap)  in kvm_clear_dirty_log_protect()
1648 	dirty_bitmap = memslot->dirty_bitmap;  in kvm_clear_dirty_log_protect()
1652 	if (log->first_page > memslot->npages ||  in kvm_clear_dirty_log_protect()
1653 	    log->num_pages > memslot->npages - log->first_page ||  in kvm_clear_dirty_log_protect()
1654 	    (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))  in kvm_clear_dirty_log_protect()
1657 	kvm_arch_sync_dirty_log(kvm, memslot);  in kvm_clear_dirty_log_protect()
1660 	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);  in kvm_clear_dirty_log_protect()
1683 			kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,  in kvm_clear_dirty_log_protect()
1690 		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);  in kvm_clear_dirty_log_protect()
1722 	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);  in kvm_is_visible_gfn()  local
1724 	return kvm_is_visible_memslot(memslot);  in kvm_is_visible_gfn()
1730 	struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_is_visible_gfn()  local
1732 	return kvm_is_visible_memslot(memslot);  in kvm_vcpu_is_visible_gfn()
2262 static void __kvm_unmap_gfn(struct kvm_memory_slot *memslot,  in __kvm_unmap_gfn()  argument
2287 		mark_page_dirty_in_slot(memslot, map->gfn);  in __kvm_unmap_gfn()
2496 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,  in __kvm_write_guest_page()  argument
2502 	addr = gfn_to_hva_memslot(memslot, gfn);  in __kvm_write_guest_page()
2508 	mark_page_dirty_in_slot(memslot, gfn);  in __kvm_write_guest_page()
2595 		ghc->memslot = __gfn_to_memslot(slots, start_gfn);  in __kvm_gfn_to_hva_cache_init()
2596 		ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,  in __kvm_gfn_to_hva_cache_init()
2606 		ghc->memslot = NULL;  in __kvm_gfn_to_hva_cache_init()
2640 	if (unlikely(!ghc->memslot))  in kvm_write_guest_offset_cached()
2646 	mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);  in kvm_write_guest_offset_cached()
2678 	if (unlikely(!ghc->memslot))  in kvm_read_guest_offset_cached()
2723 void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn)  in mark_page_dirty_in_slot()  argument
2725 	if (memslot && memslot->dirty_bitmap) {  in mark_page_dirty_in_slot()
2726 		unsigned long rel_gfn = gfn - memslot->base_gfn;  in mark_page_dirty_in_slot()
2728 		set_bit_le(rel_gfn, memslot->dirty_bitmap);  in mark_page_dirty_in_slot()
2735 	struct kvm_memory_slot *memslot;  in mark_page_dirty()  local
2737 	memslot = gfn_to_memslot(kvm, gfn);  in mark_page_dirty()
2738 	mark_page_dirty_in_slot(memslot, gfn);  in mark_page_dirty()
2744 	struct kvm_memory_slot *memslot;  in kvm_vcpu_mark_page_dirty()  local
2746 	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_mark_page_dirty()
2747 	mark_page_dirty_in_slot(memslot, gfn);  in kvm_vcpu_mark_page_dirty()