106 	u64			(*get_msi_base)(struct its_device *its_dev);
227 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_get_event_id()  local
228 	return d->hwirq - its_dev->event_map.lpi_base;  in its_get_event_id()
231 static struct its_collection *dev_event_to_col(struct its_device *its_dev,  in dev_event_to_col()  argument
234 	struct its_node *its = its_dev->its;  in dev_event_to_col()
236 	return its->collections + its_dev->event_map.col_map[event];  in dev_event_to_col()
239 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,  in dev_event_to_vlpi_map()  argument
242 	if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))  in dev_event_to_vlpi_map()
245 	return &its_dev->event_map.vlpi_maps[event];  in dev_event_to_vlpi_map()
251 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in get_vlpi_map()  local
254 		return dev_event_to_vlpi_map(its_dev, event);  in get_vlpi_map()
280 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in irq_to_cpuid_lock()  local
281 		cpu = its_dev->event_map.col_map[its_get_event_id(d)];  in irq_to_cpuid_lock()
1435 		struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in direct_lpi_inv()  local
1437 		WARN_ON(!is_v4_1(its_dev->its));  in direct_lpi_inv()
1459 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in lpi_update_config()  local
1463 	    (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))  in lpi_update_config()
1466 		its_send_inv(its_dev, its_get_event_id(d));  in lpi_update_config()
1468 		its_send_vinv(its_dev, its_get_event_id(d));  in lpi_update_config()
1473 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_vlpi_set_doorbell()  local
1481 	if (is_v4_1(its_dev->its))  in its_vlpi_set_doorbell()
1484 	map = dev_event_to_vlpi_map(its_dev, event);  in its_vlpi_set_doorbell()
1501 	its_send_vmovi(its_dev, event);  in its_vlpi_set_doorbell()
1568 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_select_cpu()  local
1575 	node = its_dev->its->numa_node;  in its_select_cpu()
1604 			if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))  in its_select_cpu()
1622 		if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&  in its_select_cpu()
1638 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_set_affinity()  local
1647 	prev_cpu = its_dev->event_map.col_map[id];  in its_set_affinity()
1660 		target_col = &its_dev->its->collections[cpu];  in its_set_affinity()
1661 		its_send_movi(its_dev, target_col, id);  in its_set_affinity()
1662 		its_dev->event_map.col_map[id] = cpu;  in its_set_affinity()
1675 static u64 its_irq_get_msi_base(struct its_device *its_dev)  in its_irq_get_msi_base()  argument
1677 	struct its_node *its = its_dev->its;  in its_irq_get_msi_base()
1684 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_irq_compose_msi_msg()  local
1688 	its = its_dev->its;  in its_irq_compose_msi_msg()
1689 	addr = its->get_msi_base(its_dev);  in its_irq_compose_msi_msg()
1702 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_irq_set_irqchip_state()  local
1710 			its_send_vint(its_dev, event);  in its_irq_set_irqchip_state()
1712 			its_send_vclear(its_dev, event);  in its_irq_set_irqchip_state()
1715 			its_send_int(its_dev, event);  in its_irq_set_irqchip_state()
1717 			its_send_clear(its_dev, event);  in its_irq_set_irqchip_state()
1803 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_vlpi_map()  local
1810 	raw_spin_lock(&its_dev->event_map.vlpi_lock);  in its_vlpi_map()
1812 	if (!its_dev->event_map.vm) {  in its_vlpi_map()
1815 		maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),  in its_vlpi_map()
1822 		its_dev->event_map.vm = info->map->vm;  in its_vlpi_map()
1823 		its_dev->event_map.vlpi_maps = maps;  in its_vlpi_map()
1824 	} else if (its_dev->event_map.vm != info->map->vm) {  in its_vlpi_map()
1830 	its_dev->event_map.vlpi_maps[event] = *info->map;  in its_vlpi_map()
1834 		its_send_vmovi(its_dev, event);  in its_vlpi_map()
1837 		its_map_vm(its_dev->its, info->map->vm);  in its_vlpi_map()
1849 		its_send_discard(its_dev, event);  in its_vlpi_map()
1852 		its_send_vmapti(its_dev, event);  in its_vlpi_map()
1855 		its_dev->event_map.nr_vlpis++;  in its_vlpi_map()
1859 	raw_spin_unlock(&its_dev->event_map.vlpi_lock);  in its_vlpi_map()
1865 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_vlpi_get()  local
1869 	raw_spin_lock(&its_dev->event_map.vlpi_lock);  in its_vlpi_get()
1873 	if (!its_dev->event_map.vm || !map) {  in its_vlpi_get()
1882 	raw_spin_unlock(&its_dev->event_map.vlpi_lock);  in its_vlpi_get()
1888 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_vlpi_unmap()  local
1892 	raw_spin_lock(&its_dev->event_map.vlpi_lock);  in its_vlpi_unmap()
1894 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {  in its_vlpi_unmap()
1900 	its_send_discard(its_dev, event);  in its_vlpi_unmap()
1904 	its_send_mapti(its_dev, d->hwirq, event);  in its_vlpi_unmap()
1910 	its_unmap_vm(its_dev->its, its_dev->event_map.vm);  in its_vlpi_unmap()
1916 	if (!--its_dev->event_map.nr_vlpis) {  in its_vlpi_unmap()
1917 		its_dev->event_map.vm = NULL;  in its_vlpi_unmap()
1918 		kfree(its_dev->event_map.vlpi_maps);  in its_vlpi_unmap()
1922 	raw_spin_unlock(&its_dev->event_map.vlpi_lock);  in its_vlpi_unmap()
1928 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_vlpi_prop_update()  local
1930 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))  in its_vlpi_prop_update()
1944 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_irq_set_vcpu_affinity()  local
1948 	if (!is_v4(its_dev->its))  in its_irq_set_vcpu_affinity()
3257 	struct its_device *its_dev = NULL, *tmp;  in its_find_device()  local
3264 			its_dev = tmp;  in its_find_device()
3271 	return its_dev;  in its_find_device()
3473 static void its_free_device(struct its_device *its_dev)  in its_free_device()  argument
3477 	raw_spin_lock_irqsave(&its_dev->its->lock, flags);  in its_free_device()
3478 	list_del(&its_dev->entry);  in its_free_device()
3479 	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);  in its_free_device()
3480 	kfree(its_dev->event_map.col_map);  in its_free_device()
3484 		free_pages((unsigned long)its_dev->itt, get_order(its_dev->itt_sz));  in its_free_device()
3486 		kfree(its_dev->itt);  in its_free_device()
3488 	kfree(its_dev);  in its_free_device()
3511 	struct its_device *its_dev;  in its_msi_prepare()  local
3538 	its_dev = its_find_device(its, dev_id);  in its_msi_prepare()
3539 	if (its_dev) {  in its_msi_prepare()
3545 		its_dev->shared = true;  in its_msi_prepare()
3550 	its_dev = its_create_device(its, dev_id, nvec, true);  in its_msi_prepare()
3551 	if (!its_dev) {  in its_msi_prepare()
3559 	info->scratchpad[0].ptr = its_dev;  in its_msi_prepare()
3595 	struct its_device *its_dev = info->scratchpad[0].ptr;  in its_irq_domain_alloc()  local
3596 	struct its_node *its = its_dev->its;  in its_irq_domain_alloc()
3602 	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);  in its_irq_domain_alloc()
3606 	err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));  in its_irq_domain_alloc()
3616 					      hwirq + i, &its_irq_chip, its_dev);  in its_irq_domain_alloc()
3621 			 (int)(hwirq + i - its_dev->event_map.lpi_base),  in its_irq_domain_alloc()
3631 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_irq_domain_activate()  local
3640 	its_dev->event_map.col_map[event] = cpu;  in its_irq_domain_activate()
3644 	its_send_mapti(its_dev, d->hwirq, event);  in its_irq_domain_activate()
3651 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_irq_domain_deactivate()  local
3654 	its_dec_lpi_count(d, its_dev->event_map.col_map[event]);  in its_irq_domain_deactivate()
3656 	its_send_discard(its_dev, event);  in its_irq_domain_deactivate()
3663 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);  in its_irq_domain_free()  local
3664 	struct its_node *its = its_dev->its;  in its_irq_domain_free()
3667 	bitmap_release_region(its_dev->event_map.lpi_map,  in its_irq_domain_free()
3684 	if (!its_dev->shared &&  in its_irq_domain_free()
3685 	    bitmap_empty(its_dev->event_map.lpi_map,  in its_irq_domain_free()
3686 			 its_dev->event_map.nr_lpis)) {  in its_irq_domain_free()
3687 		its_lpi_free(its_dev->event_map.lpi_map,  in its_irq_domain_free()
3688 			     its_dev->event_map.lpi_base,  in its_irq_domain_free()
3689 			     its_dev->event_map.nr_lpis);  in its_irq_domain_free()
3692 		its_send_mapd(its_dev, 0);  in its_irq_domain_free()
3693 		its_free_device(its_dev);  in its_irq_domain_free()
4694 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)  in its_irq_get_msi_base_pre_its()  argument
4696 	struct its_node *its = its_dev->its;  in its_irq_get_msi_base_pre_its()
4705 	return its->pre_its_base + (its_dev->device_id << 2);  in its_irq_get_msi_base_pre_its()