609 			     struct netsec_desc_ring *dring, u16 idx,  in netsec_set_rx_de()  argument
612 	struct netsec_de *de = dring->vaddr + DESC_SZ * idx;  in netsec_set_rx_de()
626 	dring->desc[idx].dma_addr = desc->dma_addr;  in netsec_set_rx_de()
627 	dring->desc[idx].addr = desc->addr;  in netsec_set_rx_de()
628 	dring->desc[idx].len = desc->len;  in netsec_set_rx_de()
633 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_clean_tx_dring()  local
635 	int tail = dring->tail;  in netsec_clean_tx_dring()
639 	spin_lock(&dring->lock);  in netsec_clean_tx_dring()
642 	entry = dring->vaddr + DESC_SZ * tail;  in netsec_clean_tx_dring()
649 		desc = &dring->desc[tail];  in netsec_clean_tx_dring()
681 		dring->tail = (tail + 1) % DESC_NUM;  in netsec_clean_tx_dring()
683 		tail = dring->tail;  in netsec_clean_tx_dring()
684 		entry = dring->vaddr + DESC_SZ * tail;  in netsec_clean_tx_dring()
688 	spin_unlock(&dring->lock);  in netsec_clean_tx_dring()
725 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_alloc_rx_data()  local
728 	page = page_pool_dev_alloc_pages(dring->page_pool);  in netsec_alloc_rx_data()
747 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_rx_fill()  local
751 		netsec_set_rx_de(priv, dring, idx, &dring->desc[idx]);  in netsec_rx_fill()
776 			     struct netsec_desc_ring *dring,  in netsec_set_tx_de()  argument
780 	int idx = dring->head;  in netsec_set_tx_de()
784 	de = dring->vaddr + (DESC_SZ * idx);  in netsec_set_tx_de()
802 	dring->desc[idx] = *desc;  in netsec_set_tx_de()
804 		dring->desc[idx].skb = buf;  in netsec_set_tx_de()
807 		dring->desc[idx].xdpf = buf;  in netsec_set_tx_de()
810 	dring->head = (dring->head + 1) % DESC_NUM;  in netsec_set_tx_de()
887 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_run_xdp()  local
908 			page_pool_put_page(dring->page_pool, page, sync, true);  in netsec_run_xdp()
918 			page_pool_put_page(dring->page_pool, page, sync, true);  in netsec_run_xdp()
930 		page_pool_put_page(dring->page_pool, page, sync, true);  in netsec_run_xdp()
939 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_process_rx()  local
949 	xdp.rxq = &dring->xdp_rxq;  in netsec_process_rx()
954 	dma_dir = page_pool_get_dma_dir(dring->page_pool);  in netsec_process_rx()
957 		u16 idx = dring->tail;  in netsec_process_rx()
958 		struct netsec_de *de = dring->vaddr + (DESC_SZ * idx);  in netsec_process_rx()
959 		struct netsec_desc *desc = &dring->desc[idx];  in netsec_process_rx()
989 			dring->tail = (dring->tail + 1) % DESC_NUM;  in netsec_process_rx()
1031 			page_pool_put_page(dring->page_pool, page, pkt_len,  in netsec_process_rx()
1037 		page_pool_release_page(dring->page_pool, page);  in netsec_process_rx()
1061 		dring->tail = (dring->tail + 1) % DESC_NUM;  in netsec_process_rx()
1093 static int netsec_desc_used(struct netsec_desc_ring *dring)  in netsec_desc_used()  argument
1097 	if (dring->head >= dring->tail)  in netsec_desc_used()
1098 		used = dring->head - dring->tail;  in netsec_desc_used()
1100 		used = dring->head + DESC_NUM - dring->tail;  in netsec_desc_used()
1107 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_check_stop_tx()  local
1118 		used = netsec_desc_used(dring);  in netsec_check_stop_tx()
1132 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_netdev_start_xmit()  local
1138 	spin_lock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1139 	filled = netsec_desc_used(dring);  in netsec_netdev_start_xmit()
1141 		spin_unlock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1170 		spin_unlock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1184 	netsec_set_tx_de(priv, dring, &tx_ctrl, &tx_desc, skb);  in netsec_netdev_start_xmit()
1185 	spin_unlock_bh(&dring->lock);  in netsec_netdev_start_xmit()
1193 	struct netsec_desc_ring *dring = &priv->desc_ring[id];  in netsec_uninit_pkt_dring()  local
1197 	if (!dring->vaddr || !dring->desc)  in netsec_uninit_pkt_dring()
1200 		desc = &dring->desc[idx];  in netsec_uninit_pkt_dring()
1207 			page_pool_put_full_page(dring->page_pool, page, false);  in netsec_uninit_pkt_dring()
1217 		if (xdp_rxq_info_is_reg(&dring->xdp_rxq))  in netsec_uninit_pkt_dring()
1218 			xdp_rxq_info_unreg(&dring->xdp_rxq);  in netsec_uninit_pkt_dring()
1219 		page_pool_destroy(dring->page_pool);  in netsec_uninit_pkt_dring()
1222 	memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM);  in netsec_uninit_pkt_dring()
1223 	memset(dring->vaddr, 0, DESC_SZ * DESC_NUM);  in netsec_uninit_pkt_dring()
1225 	dring->head = 0;  in netsec_uninit_pkt_dring()
1226 	dring->tail = 0;  in netsec_uninit_pkt_dring()
1234 	struct netsec_desc_ring *dring = &priv->desc_ring[id];  in netsec_free_dring()  local
1236 	if (dring->vaddr) {  in netsec_free_dring()
1238 				  dring->vaddr, dring->desc_dma);  in netsec_free_dring()
1239 		dring->vaddr = NULL;  in netsec_free_dring()
1242 	kfree(dring->desc);  in netsec_free_dring()
1243 	dring->desc = NULL;  in netsec_free_dring()
1248 	struct netsec_desc_ring *dring = &priv->desc_ring[id];  in netsec_alloc_dring()  local
1250 	dring->vaddr = dma_alloc_coherent(priv->dev, DESC_SZ * DESC_NUM,  in netsec_alloc_dring()
1251 					  &dring->desc_dma, GFP_KERNEL);  in netsec_alloc_dring()
1252 	if (!dring->vaddr)  in netsec_alloc_dring()
1255 	dring->desc = kcalloc(DESC_NUM, sizeof(*dring->desc), GFP_KERNEL);  in netsec_alloc_dring()
1256 	if (!dring->desc)  in netsec_alloc_dring()
1268 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];  in netsec_setup_tx_dring()  local
1274 		de = dring->vaddr + (DESC_SZ * i);  in netsec_setup_tx_dring()
1285 	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];  in netsec_setup_rx_dring()  local
1300 	dring->page_pool = page_pool_create(&pp_params);  in netsec_setup_rx_dring()
1301 	if (IS_ERR(dring->page_pool)) {  in netsec_setup_rx_dring()
1302 		err = PTR_ERR(dring->page_pool);  in netsec_setup_rx_dring()
1303 		dring->page_pool = NULL;  in netsec_setup_rx_dring()
1307 	err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0);  in netsec_setup_rx_dring()
1311 	err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_POOL,  in netsec_setup_rx_dring()
1312 					 dring->page_pool);  in netsec_setup_rx_dring()
1317 		struct netsec_desc *desc = &dring->desc[i];  in netsec_setup_rx_dring()