79 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)  in efx_wanted_parallelism()  argument
89 			netif_warn(efx, probe, efx->net_dev,  in efx_wanted_parallelism()
107 		netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,  in efx_wanted_parallelism()
117 	if (efx->type->sriov_wanted) {  in efx_wanted_parallelism()
118 		if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&  in efx_wanted_parallelism()
119 		    count > efx_vf_size(efx)) {  in efx_wanted_parallelism()
120 			netif_warn(efx, probe, efx->net_dev,  in efx_wanted_parallelism()
124 				   count, efx_vf_size(efx));  in efx_wanted_parallelism()
125 			count = efx_vf_size(efx);  in efx_wanted_parallelism()
133 static int efx_allocate_msix_channels(struct efx_nic *efx,  in efx_allocate_msix_channels()  argument
156 	vec_count = pci_msix_vec_count(efx->pci_dev);  in efx_allocate_msix_channels()
167 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
170 		efx->n_xdp_channels = 0;  in efx_allocate_msix_channels()
171 		efx->xdp_tx_per_channel = 0;  in efx_allocate_msix_channels()
172 		efx->xdp_tx_queue_count = 0;  in efx_allocate_msix_channels()
173 	} else if (n_channels + n_xdp_tx > efx->max_vis) {  in efx_allocate_msix_channels()
174 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
176 			  n_xdp_tx, n_channels, efx->max_vis);  in efx_allocate_msix_channels()
177 		efx->n_xdp_channels = 0;  in efx_allocate_msix_channels()
178 		efx->xdp_tx_per_channel = 0;  in efx_allocate_msix_channels()
179 		efx->xdp_tx_queue_count = 0;  in efx_allocate_msix_channels()
181 		efx->n_xdp_channels = n_xdp_ev;  in efx_allocate_msix_channels()
182 		efx->xdp_tx_per_channel = EFX_MAX_TXQ_PER_CHANNEL;  in efx_allocate_msix_channels()
183 		efx->xdp_tx_queue_count = n_xdp_tx;  in efx_allocate_msix_channels()
185 		netif_dbg(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
191 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
194 		netif_err(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
201 	efx->n_channels = n_channels;  in efx_allocate_msix_channels()
204 	n_channels -= efx->n_xdp_channels;  in efx_allocate_msix_channels()
207 		efx->n_tx_channels =  in efx_allocate_msix_channels()
209 			    efx->max_tx_channels);  in efx_allocate_msix_channels()
210 		efx->tx_channel_offset =  in efx_allocate_msix_channels()
211 			n_channels - efx->n_tx_channels;  in efx_allocate_msix_channels()
212 		efx->n_rx_channels =  in efx_allocate_msix_channels()
214 			    efx->n_tx_channels, 1U);  in efx_allocate_msix_channels()
216 		efx->n_tx_channels = min(n_channels, efx->max_tx_channels);  in efx_allocate_msix_channels()
217 		efx->tx_channel_offset = 0;  in efx_allocate_msix_channels()
218 		efx->n_rx_channels = n_channels;  in efx_allocate_msix_channels()
221 	efx->n_rx_channels = min(efx->n_rx_channels, parallelism);  in efx_allocate_msix_channels()
222 	efx->n_tx_channels = min(efx->n_tx_channels, parallelism);  in efx_allocate_msix_channels()
224 	efx->xdp_channel_offset = n_channels;  in efx_allocate_msix_channels()
226 	netif_dbg(efx, drv, efx->net_dev,  in efx_allocate_msix_channels()
228 		  efx->n_rx_channels);  in efx_allocate_msix_channels()
230 	return efx->n_channels;  in efx_allocate_msix_channels()
236 int efx_probe_interrupts(struct efx_nic *efx)  in efx_probe_interrupts()  argument
244 		if (efx->extra_channel_type[i])  in efx_probe_interrupts()
247 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {  in efx_probe_interrupts()
248 		unsigned int parallelism = efx_wanted_parallelism(efx);  in efx_probe_interrupts()
252 		rc = efx_allocate_msix_channels(efx, efx->max_channels,  in efx_probe_interrupts()
258 			rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,  in efx_probe_interrupts()
263 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
265 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)  in efx_probe_interrupts()
266 				efx->interrupt_mode = EFX_INT_MODE_MSI;  in efx_probe_interrupts()
270 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
273 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
279 			for (i = 0; i < efx->n_channels; i++)  in efx_probe_interrupts()
280 				efx_get_channel(efx, i)->irq =  in efx_probe_interrupts()
286 	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {  in efx_probe_interrupts()
287 		efx->n_channels = 1;  in efx_probe_interrupts()
288 		efx->n_rx_channels = 1;  in efx_probe_interrupts()
289 		efx->n_tx_channels = 1;  in efx_probe_interrupts()
290 		efx->tx_channel_offset = 0;  in efx_probe_interrupts()
291 		efx->n_xdp_channels = 0;  in efx_probe_interrupts()
292 		efx->xdp_channel_offset = efx->n_channels;  in efx_probe_interrupts()
293 		rc = pci_enable_msi(efx->pci_dev);  in efx_probe_interrupts()
295 			efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;  in efx_probe_interrupts()
297 			netif_err(efx, drv, efx->net_dev,  in efx_probe_interrupts()
299 			if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)  in efx_probe_interrupts()
300 				efx->interrupt_mode = EFX_INT_MODE_LEGACY;  in efx_probe_interrupts()
307 	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {  in efx_probe_interrupts()
308 		efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);  in efx_probe_interrupts()
309 		efx->n_rx_channels = 1;  in efx_probe_interrupts()
310 		efx->n_tx_channels = 1;  in efx_probe_interrupts()
311 		efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0;  in efx_probe_interrupts()
312 		efx->n_xdp_channels = 0;  in efx_probe_interrupts()
313 		efx->xdp_channel_offset = efx->n_channels;  in efx_probe_interrupts()
314 		efx->legacy_irq = efx->pci_dev->irq;  in efx_probe_interrupts()
318 	efx->n_extra_tx_channels = 0;  in efx_probe_interrupts()
319 	j = efx->xdp_channel_offset;  in efx_probe_interrupts()
321 		if (!efx->extra_channel_type[i])  in efx_probe_interrupts()
323 		if (j <= efx->tx_channel_offset + efx->n_tx_channels) {  in efx_probe_interrupts()
324 			efx->extra_channel_type[i]->handle_no_channel(efx);  in efx_probe_interrupts()
327 			efx_get_channel(efx, j)->type =  in efx_probe_interrupts()
328 				efx->extra_channel_type[i];  in efx_probe_interrupts()
329 			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))  in efx_probe_interrupts()
330 				efx->n_extra_tx_channels++;  in efx_probe_interrupts()
334 	rss_spread = efx->n_rx_channels;  in efx_probe_interrupts()
337 	if (efx->type->sriov_wanted) {  in efx_probe_interrupts()
338 		efx->rss_spread = ((rss_spread > 1 ||  in efx_probe_interrupts()
339 				    !efx->type->sriov_wanted(efx)) ?  in efx_probe_interrupts()
340 				   rss_spread : efx_vf_size(efx));  in efx_probe_interrupts()
344 	efx->rss_spread = rss_spread;  in efx_probe_interrupts()
350 void efx_set_interrupt_affinity(struct efx_nic *efx)  in efx_set_interrupt_affinity()  argument
355 	efx_for_each_channel(channel, efx) {  in efx_set_interrupt_affinity()
357 					   pcibus_to_node(efx->pci_dev->bus));  in efx_set_interrupt_affinity()
362 void efx_clear_interrupt_affinity(struct efx_nic *efx)  in efx_clear_interrupt_affinity()  argument
366 	efx_for_each_channel(channel, efx)  in efx_clear_interrupt_affinity()
371 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))  in efx_set_interrupt_affinity()  argument
376 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))  in efx_clear_interrupt_affinity()  argument
381 void efx_remove_interrupts(struct efx_nic *efx)  in efx_remove_interrupts()  argument
386 	efx_for_each_channel(channel, efx)  in efx_remove_interrupts()
388 	pci_disable_msi(efx->pci_dev);  in efx_remove_interrupts()
389 	pci_disable_msix(efx->pci_dev);  in efx_remove_interrupts()
392 	efx->legacy_irq = 0;  in efx_remove_interrupts()
406 	struct efx_nic *efx = channel->efx;  in efx_probe_eventq()  local
409 	netif_dbg(efx, probe, efx->net_dev,  in efx_probe_eventq()
415 	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);  in efx_probe_eventq()
425 	struct efx_nic *efx = channel->efx;  in efx_init_eventq()  local
430 	netif_dbg(efx, drv, efx->net_dev,  in efx_init_eventq()
435 		efx->type->push_irq_moderation(channel);  in efx_init_eventq()
445 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,  in efx_start_eventq()
471 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_fini_eventq()
480 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_eventq()
510 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)  in efx_alloc_channel()  argument
521 	channel->efx = efx;  in efx_alloc_channel()
527 		tx_queue->efx = efx;  in efx_alloc_channel()
538 	rx_queue->efx = efx;  in efx_alloc_channel()
544 int efx_init_channels(struct efx_nic *efx)  in efx_init_channels()  argument
549 		efx->channel[i] = efx_alloc_channel(efx, i);  in efx_init_channels()
550 		if (!efx->channel[i])  in efx_init_channels()
552 		efx->msi_context[i].efx = efx;  in efx_init_channels()
553 		efx->msi_context[i].index = i;  in efx_init_channels()
557 	efx->interrupt_mode = min(efx->type->min_interrupt_mode,  in efx_init_channels()
560 	efx->max_channels = EFX_MAX_CHANNELS;  in efx_init_channels()
561 	efx->max_tx_channels = EFX_MAX_CHANNELS;  in efx_init_channels()
566 void efx_fini_channels(struct efx_nic *efx)  in efx_fini_channels()  argument
571 		if (efx->channel[i]) {  in efx_fini_channels()
572 			kfree(efx->channel[i]);  in efx_fini_channels()
573 			efx->channel[i] = NULL;  in efx_fini_channels()
625 	netif_dbg(channel->efx, probe, channel->efx->net_dev,  in efx_probe_channel()
659 	struct efx_nic *efx = channel->efx;  in efx_get_channel_name()  local
665 	if (number >= efx->xdp_channel_offset &&  in efx_get_channel_name()
666 	    !WARN_ON_ONCE(!efx->n_xdp_channels)) {  in efx_get_channel_name()
668 		number -= efx->xdp_channel_offset;  in efx_get_channel_name()
669 	} else if (efx->tx_channel_offset == 0) {  in efx_get_channel_name()
671 	} else if (number < efx->tx_channel_offset) {  in efx_get_channel_name()
675 		number -= efx->tx_channel_offset;  in efx_get_channel_name()
677 	snprintf(buf, len, "%s%s-%d", efx->name, type, number);  in efx_get_channel_name()
680 void efx_set_channel_names(struct efx_nic *efx)  in efx_set_channel_names()  argument
684 	efx_for_each_channel(channel, efx)  in efx_set_channel_names()
686 					efx->msi_context[channel->channel].name,  in efx_set_channel_names()
687 					sizeof(efx->msi_context[0].name));  in efx_set_channel_names()
690 int efx_probe_channels(struct efx_nic *efx)  in efx_probe_channels()  argument
696 	efx->next_buffer_table = 0;  in efx_probe_channels()
703 	efx_for_each_channel_rev(channel, efx) {  in efx_probe_channels()
706 			netif_err(efx, probe, efx->net_dev,  in efx_probe_channels()
712 	efx_set_channel_names(efx);  in efx_probe_channels()
717 	efx_remove_channels(efx);  in efx_probe_channels()
726 	netif_dbg(channel->efx, drv, channel->efx->net_dev,  in efx_remove_channel()
737 void efx_remove_channels(struct efx_nic *efx)  in efx_remove_channels()  argument
741 	efx_for_each_channel(channel, efx)  in efx_remove_channels()
744 	kfree(efx->xdp_tx_queues);  in efx_remove_channels()
747 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)  in efx_realloc_channels()  argument
750 			   *ptp_channel = efx_ptp_channel(efx);  in efx_realloc_channels()
751 	struct efx_ptp_data *ptp_data = efx->ptp_data;  in efx_realloc_channels()
756 	rc = efx_check_disabled(efx);  in efx_realloc_channels()
763 	efx_for_each_channel(channel, efx) {  in efx_realloc_channels()
782 	efx_device_detach_sync(efx);  in efx_realloc_channels()
783 	efx_stop_all(efx);  in efx_realloc_channels()
784 	efx_soft_disable_interrupts(efx);  in efx_realloc_channels()
788 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
789 		channel = efx->channel[i];  in efx_realloc_channels()
800 	old_rxq_entries = efx->rxq_entries;  in efx_realloc_channels()
801 	old_txq_entries = efx->txq_entries;  in efx_realloc_channels()
802 	efx->rxq_entries = rxq_entries;  in efx_realloc_channels()
803 	efx->txq_entries = txq_entries;  in efx_realloc_channels()
804 	for (i = 0; i < efx->n_channels; i++)  in efx_realloc_channels()
805 		swap(efx->channel[i], other_channel[i]);  in efx_realloc_channels()
808 	efx->next_buffer_table = next_buffer_table;  in efx_realloc_channels()
810 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
811 		channel = efx->channel[i];  in efx_realloc_channels()
817 		efx_init_napi_channel(efx->channel[i]);  in efx_realloc_channels()
821 	efx->ptp_data = NULL;  in efx_realloc_channels()
823 	for (i = 0; i < efx->n_channels; i++) {  in efx_realloc_channels()
832 	efx->ptp_data = ptp_data;  in efx_realloc_channels()
833 	rc2 = efx_soft_enable_interrupts(efx);  in efx_realloc_channels()
836 		netif_err(efx, drv, efx->net_dev,  in efx_realloc_channels()
838 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);  in efx_realloc_channels()
840 		efx_start_all(efx);  in efx_realloc_channels()
841 		efx_device_attach_if_not_resetting(efx);  in efx_realloc_channels()
847 	efx->rxq_entries = old_rxq_entries;  in efx_realloc_channels()
848 	efx->txq_entries = old_txq_entries;  in efx_realloc_channels()
849 	for (i = 0; i < efx->n_channels; i++)  in efx_realloc_channels()
850 		swap(efx->channel[i], other_channel[i]);  in efx_realloc_channels()
851 	efx_ptp_update_channel(efx, ptp_channel);  in efx_realloc_channels()
855 int efx_set_channels(struct efx_nic *efx)  in efx_set_channels()  argument
863 	if (efx->xdp_tx_queue_count) {  in efx_set_channels()
864 		EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);  in efx_set_channels()
867 		efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,  in efx_set_channels()
868 					     sizeof(*efx->xdp_tx_queues),  in efx_set_channels()
870 		if (!efx->xdp_tx_queues)  in efx_set_channels()
879 	efx_for_each_channel(channel, efx) {  in efx_set_channels()
880 		if (channel->channel < efx->n_rx_channels)  in efx_set_channels()
885 		if (channel->channel >= efx->tx_channel_offset) {  in efx_set_channels()
896 					if (xdp_queue_number < efx->xdp_tx_queue_count) {  in efx_set_channels()
897 						netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is XDP %u, HW %u\n",  in efx_set_channels()
900 						efx->xdp_tx_queues[xdp_queue_number] = tx_queue;  in efx_set_channels()
907 					netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is HW %u\n",  in efx_set_channels()
914 	WARN_ON(xdp_queue_number != efx->xdp_tx_queue_count);  in efx_set_channels()
916 	rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);  in efx_set_channels()
919 	return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);  in efx_set_channels()
924 	return channel->channel - channel->efx->tx_channel_offset <  in efx_default_channel_want_txqs()
925 		channel->efx->n_tx_channels;  in efx_default_channel_want_txqs()
932 int efx_soft_enable_interrupts(struct efx_nic *efx)  in efx_soft_enable_interrupts()  argument
937 	BUG_ON(efx->state == STATE_DISABLED);  in efx_soft_enable_interrupts()
939 	efx->irq_soft_enabled = true;  in efx_soft_enable_interrupts()
942 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
951 	efx_mcdi_mode_event(efx);  in efx_soft_enable_interrupts()
956 	efx_for_each_channel(channel, efx) {  in efx_soft_enable_interrupts()
967 void efx_soft_disable_interrupts(struct efx_nic *efx)  in efx_soft_disable_interrupts()  argument
971 	if (efx->state == STATE_DISABLED)  in efx_soft_disable_interrupts()
974 	efx_mcdi_mode_poll(efx);  in efx_soft_disable_interrupts()
976 	efx->irq_soft_enabled = false;  in efx_soft_disable_interrupts()
979 	if (efx->legacy_irq)  in efx_soft_disable_interrupts()
980 		synchronize_irq(efx->legacy_irq);  in efx_soft_disable_interrupts()
982 	efx_for_each_channel(channel, efx) {  in efx_soft_disable_interrupts()
992 	efx_mcdi_flush_async(efx);  in efx_soft_disable_interrupts()
995 int efx_enable_interrupts(struct efx_nic *efx)  in efx_enable_interrupts()  argument
1001 	BUG_ON(efx->state == STATE_DISABLED);  in efx_enable_interrupts()
1003 	if (efx->eeh_disabled_legacy_irq) {  in efx_enable_interrupts()
1004 		enable_irq(efx->legacy_irq);  in efx_enable_interrupts()
1005 		efx->eeh_disabled_legacy_irq = false;  in efx_enable_interrupts()
1008 	efx->type->irq_enable_master(efx);  in efx_enable_interrupts()
1010 	efx_for_each_channel(channel, efx) {  in efx_enable_interrupts()
1018 	rc = efx_soft_enable_interrupts(efx);  in efx_enable_interrupts()
1026 	efx_for_each_channel(channel, efx) {  in efx_enable_interrupts()
1033 	efx->type->irq_disable_non_ev(efx);  in efx_enable_interrupts()
1038 void efx_disable_interrupts(struct efx_nic *efx)  in efx_disable_interrupts()  argument
1042 	efx_soft_disable_interrupts(efx);  in efx_disable_interrupts()
1044 	efx_for_each_channel(channel, efx) {  in efx_disable_interrupts()
1049 	efx->type->irq_disable_non_ev(efx);  in efx_disable_interrupts()
1052 void efx_start_channels(struct efx_nic *efx)  in efx_start_channels()  argument
1058 	efx_for_each_channel(channel, efx) {  in efx_start_channels()
1061 			atomic_inc(&efx->active_queues);  in efx_start_channels()
1066 			atomic_inc(&efx->active_queues);  in efx_start_channels()
1076 void efx_stop_channels(struct efx_nic *efx)  in efx_stop_channels()  argument
1084 	efx_for_each_channel(channel, efx) {  in efx_stop_channels()
1089 	efx_for_each_channel(channel, efx) {  in efx_stop_channels()
1102 	if (efx->type->fini_dmaq)  in efx_stop_channels()
1103 		rc = efx->type->fini_dmaq(efx);  in efx_stop_channels()
1106 		netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");  in efx_stop_channels()
1108 		netif_dbg(efx, drv, efx->net_dev,  in efx_stop_channels()
1112 	efx_for_each_channel(channel, efx) {  in efx_stop_channels()
1177 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)  in efx_update_irq_mod()  argument
1179 	int step = efx->irq_mod_step_us;  in efx_update_irq_mod()
1184 			efx->type->push_irq_moderation(channel);  in efx_update_irq_mod()
1188 		    efx->irq_rx_moderation_us) {  in efx_update_irq_mod()
1190 			efx->type->push_irq_moderation(channel);  in efx_update_irq_mod()
1207 	struct efx_nic *efx = channel->efx;  in efx_poll()  local
1213 	netif_vdbg(efx, intr, efx->net_dev,  in efx_poll()
1223 		    efx->irq_rx_adaptive &&  in efx_poll()
1225 			efx_update_irq_mod(efx, channel);  in efx_poll()
1250 	struct efx_nic *efx = channel->efx;  in efx_init_napi_channel()  local
1252 	channel->napi_dev = efx->net_dev;  in efx_init_napi_channel()
1257 void efx_init_napi(struct efx_nic *efx)  in efx_init_napi()  argument
1261 	efx_for_each_channel(channel, efx)  in efx_init_napi()
1273 void efx_fini_napi(struct efx_nic *efx)  in efx_fini_napi()  argument
1277 	efx_for_each_channel(channel, efx)  in efx_fini_napi()