1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3
4 #include "enetc.h"
5 #include <linux/tcp.h>
6 #include <linux/udp.h>
7 #include <linux/vmalloc.h>
8
9 /* ENETC overhead: optional extension BD + 1 BD gap */
10 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
11 /* max # of chained Tx BDs is 15, including head and extension BD */
12 #define ENETC_MAX_SKB_FRAGS 13
13 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
14
15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
16 int active_offloads);
17
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
19 {
20 struct enetc_ndev_priv *priv = netdev_priv(ndev);
21 struct enetc_bdr *tx_ring;
22 int count;
23
24 tx_ring = priv->tx_ring[skb->queue_mapping];
25
26 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
27 if (unlikely(skb_linearize(skb)))
28 goto drop_packet_err;
29
30 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
31 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
32 netif_stop_subqueue(ndev, tx_ring->index);
33 return NETDEV_TX_BUSY;
34 }
35
36 enetc_lock_mdio();
37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
38 enetc_unlock_mdio();
39
40 if (unlikely(!count))
41 goto drop_packet_err;
42
43 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
44 netif_stop_subqueue(ndev, tx_ring->index);
45
46 return NETDEV_TX_OK;
47
48 drop_packet_err:
49 dev_kfree_skb_any(skb);
50 return NETDEV_TX_OK;
51 }
52
enetc_tx_csum(struct sk_buff * skb,union enetc_tx_bd * txbd)53 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
54 {
55 int l3_start, l3_hsize;
56 u16 l3_flags, l4_flags;
57
58 if (skb->ip_summed != CHECKSUM_PARTIAL)
59 return false;
60
61 switch (skb->csum_offset) {
62 case offsetof(struct tcphdr, check):
63 l4_flags = ENETC_TXBD_L4_TCP;
64 break;
65 case offsetof(struct udphdr, check):
66 l4_flags = ENETC_TXBD_L4_UDP;
67 break;
68 default:
69 skb_checksum_help(skb);
70 return false;
71 }
72
73 l3_start = skb_network_offset(skb);
74 l3_hsize = skb_network_header_len(skb);
75
76 l3_flags = 0;
77 if (skb->protocol == htons(ETH_P_IPV6))
78 l3_flags = ENETC_TXBD_L3_IPV6;
79
80 /* write BD fields */
81 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
82 txbd->l4_csoff = l4_flags;
83
84 return true;
85 }
86
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)87 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
88 struct enetc_tx_swbd *tx_swbd)
89 {
90 if (tx_swbd->is_dma_page)
91 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
92 tx_swbd->len, DMA_TO_DEVICE);
93 else
94 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
95 tx_swbd->len, DMA_TO_DEVICE);
96 tx_swbd->dma = 0;
97 }
98
enetc_free_tx_skb(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)99 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
100 struct enetc_tx_swbd *tx_swbd)
101 {
102 if (tx_swbd->dma)
103 enetc_unmap_tx_buff(tx_ring, tx_swbd);
104
105 if (tx_swbd->skb) {
106 dev_kfree_skb_any(tx_swbd->skb);
107 tx_swbd->skb = NULL;
108 }
109 }
110
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb,int active_offloads)111 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
112 int active_offloads)
113 {
114 struct enetc_tx_swbd *tx_swbd;
115 skb_frag_t *frag;
116 int len = skb_headlen(skb);
117 union enetc_tx_bd temp_bd;
118 union enetc_tx_bd *txbd;
119 bool do_vlan, do_tstamp;
120 int i, count = 0;
121 unsigned int f;
122 dma_addr_t dma;
123 u8 flags = 0;
124
125 i = tx_ring->next_to_use;
126 txbd = ENETC_TXBD(*tx_ring, i);
127 prefetchw(txbd);
128
129 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
130 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
131 goto dma_err;
132
133 temp_bd.addr = cpu_to_le64(dma);
134 temp_bd.buf_len = cpu_to_le16(len);
135 temp_bd.lstatus = 0;
136
137 tx_swbd = &tx_ring->tx_swbd[i];
138 tx_swbd->dma = dma;
139 tx_swbd->len = len;
140 tx_swbd->is_dma_page = 0;
141 count++;
142
143 do_vlan = skb_vlan_tag_present(skb);
144 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
145 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
146 tx_swbd->do_tstamp = do_tstamp;
147 tx_swbd->check_wb = tx_swbd->do_tstamp;
148
149 if (do_vlan || do_tstamp)
150 flags |= ENETC_TXBD_FLAGS_EX;
151
152 if (enetc_tx_csum(skb, &temp_bd))
153 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
154 else if (tx_ring->tsd_enable)
155 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
156
157 /* first BD needs frm_len and offload flags set */
158 temp_bd.frm_len = cpu_to_le16(skb->len);
159 temp_bd.flags = flags;
160
161 if (flags & ENETC_TXBD_FLAGS_TSE) {
162 u32 temp;
163
164 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK)
165 | (flags << ENETC_TXBD_FLAGS_OFFSET);
166 temp_bd.txstart = cpu_to_le32(temp);
167 }
168
169 if (flags & ENETC_TXBD_FLAGS_EX) {
170 u8 e_flags = 0;
171 *txbd = temp_bd;
172 enetc_clear_tx_bd(&temp_bd);
173
174 /* add extension BD for VLAN and/or timestamping */
175 flags = 0;
176 tx_swbd++;
177 txbd++;
178 i++;
179 if (unlikely(i == tx_ring->bd_count)) {
180 i = 0;
181 tx_swbd = tx_ring->tx_swbd;
182 txbd = ENETC_TXBD(*tx_ring, 0);
183 }
184 prefetchw(txbd);
185
186 if (do_vlan) {
187 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
188 temp_bd.ext.tpid = 0; /* < C-TAG */
189 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
190 }
191
192 if (do_tstamp) {
193 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
194 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
195 }
196
197 temp_bd.ext.e_flags = e_flags;
198 count++;
199 }
200
201 frag = &skb_shinfo(skb)->frags[0];
202 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
203 len = skb_frag_size(frag);
204 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
205 DMA_TO_DEVICE);
206 if (dma_mapping_error(tx_ring->dev, dma))
207 goto dma_err;
208
209 *txbd = temp_bd;
210 enetc_clear_tx_bd(&temp_bd);
211
212 flags = 0;
213 tx_swbd++;
214 txbd++;
215 i++;
216 if (unlikely(i == tx_ring->bd_count)) {
217 i = 0;
218 tx_swbd = tx_ring->tx_swbd;
219 txbd = ENETC_TXBD(*tx_ring, 0);
220 }
221 prefetchw(txbd);
222
223 temp_bd.addr = cpu_to_le64(dma);
224 temp_bd.buf_len = cpu_to_le16(len);
225
226 tx_swbd->dma = dma;
227 tx_swbd->len = len;
228 tx_swbd->is_dma_page = 1;
229 count++;
230 }
231
232 /* last BD needs 'F' bit set */
233 flags |= ENETC_TXBD_FLAGS_F;
234 temp_bd.flags = flags;
235 *txbd = temp_bd;
236
237 tx_ring->tx_swbd[i].skb = skb;
238
239 enetc_bdr_idx_inc(tx_ring, &i);
240 tx_ring->next_to_use = i;
241
242 skb_tx_timestamp(skb);
243
244 /* let H/W know BD ring has been updated */
245 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */
246
247 return count;
248
249 dma_err:
250 dev_err(tx_ring->dev, "DMA map error");
251
252 do {
253 tx_swbd = &tx_ring->tx_swbd[i];
254 enetc_free_tx_skb(tx_ring, tx_swbd);
255 if (i == 0)
256 i = tx_ring->bd_count;
257 i--;
258 } while (count--);
259
260 return 0;
261 }
262
enetc_msix(int irq,void * data)263 static irqreturn_t enetc_msix(int irq, void *data)
264 {
265 struct enetc_int_vector *v = data;
266 int i;
267
268 enetc_lock_mdio();
269
270 /* disable interrupts */
271 enetc_wr_reg_hot(v->rbier, 0);
272 enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
273
274 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
275 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
276
277 enetc_unlock_mdio();
278
279 napi_schedule(&v->napi);
280
281 return IRQ_HANDLED;
282 }
283
284 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
285 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
286 struct napi_struct *napi, int work_limit);
287
enetc_rx_dim_work(struct work_struct * w)288 static void enetc_rx_dim_work(struct work_struct *w)
289 {
290 struct dim *dim = container_of(w, struct dim, work);
291 struct dim_cq_moder moder =
292 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
293 struct enetc_int_vector *v =
294 container_of(dim, struct enetc_int_vector, rx_dim);
295
296 v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
297 dim->state = DIM_START_MEASURE;
298 }
299
enetc_rx_net_dim(struct enetc_int_vector * v)300 static void enetc_rx_net_dim(struct enetc_int_vector *v)
301 {
302 struct dim_sample dim_sample = {};
303
304 v->comp_cnt++;
305
306 if (!v->rx_napi_work)
307 return;
308
309 dim_update_sample(v->comp_cnt,
310 v->rx_ring.stats.packets,
311 v->rx_ring.stats.bytes,
312 &dim_sample);
313 net_dim(&v->rx_dim, dim_sample);
314 }
315
enetc_poll(struct napi_struct * napi,int budget)316 static int enetc_poll(struct napi_struct *napi, int budget)
317 {
318 struct enetc_int_vector
319 *v = container_of(napi, struct enetc_int_vector, napi);
320 bool complete = true;
321 int work_done;
322 int i;
323
324 enetc_lock_mdio();
325
326 for (i = 0; i < v->count_tx_rings; i++)
327 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
328 complete = false;
329
330 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
331 if (work_done == budget)
332 complete = false;
333 if (work_done)
334 v->rx_napi_work = true;
335
336 if (!complete) {
337 enetc_unlock_mdio();
338 return budget;
339 }
340
341 napi_complete_done(napi, work_done);
342
343 if (likely(v->rx_dim_en))
344 enetc_rx_net_dim(v);
345
346 v->rx_napi_work = false;
347
348 /* enable interrupts */
349 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
350
351 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
352 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
353 ENETC_TBIER_TXTIE);
354
355 enetc_unlock_mdio();
356
357 return work_done;
358 }
359
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)360 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
361 {
362 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
363
364 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
365 }
366
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)367 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
368 u64 *tstamp)
369 {
370 u32 lo, hi, tstamp_lo;
371
372 lo = enetc_rd_hot(hw, ENETC_SICTR0);
373 hi = enetc_rd_hot(hw, ENETC_SICTR1);
374 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
375 if (lo <= tstamp_lo)
376 hi -= 1;
377 *tstamp = (u64)hi << 32 | tstamp_lo;
378 }
379
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)380 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
381 {
382 struct skb_shared_hwtstamps shhwtstamps;
383
384 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
385 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
386 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
387 /* Ensure skb_mstamp_ns, which might have been populated with
388 * the txtime, is not mistaken for a software timestamp,
389 * because this will prevent the dispatch of our hardware
390 * timestamp to the socket.
391 */
392 skb->tstamp = ktime_set(0, 0);
393 skb_tstamp_tx(skb, &shhwtstamps);
394 }
395 }
396
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)397 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
398 {
399 struct net_device *ndev = tx_ring->ndev;
400 int tx_frm_cnt = 0, tx_byte_cnt = 0;
401 struct enetc_tx_swbd *tx_swbd;
402 int i, bds_to_clean;
403 bool do_tstamp;
404 u64 tstamp = 0;
405
406 i = tx_ring->next_to_clean;
407 tx_swbd = &tx_ring->tx_swbd[i];
408
409 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
410
411 do_tstamp = false;
412
413 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
414 bool is_eof = !!tx_swbd->skb;
415
416 if (unlikely(tx_swbd->check_wb)) {
417 struct enetc_ndev_priv *priv = netdev_priv(ndev);
418 union enetc_tx_bd *txbd;
419
420 txbd = ENETC_TXBD(*tx_ring, i);
421
422 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
423 tx_swbd->do_tstamp) {
424 enetc_get_tx_tstamp(&priv->si->hw, txbd,
425 &tstamp);
426 do_tstamp = true;
427 }
428 }
429
430 if (likely(tx_swbd->dma))
431 enetc_unmap_tx_buff(tx_ring, tx_swbd);
432
433 if (is_eof) {
434 if (unlikely(do_tstamp)) {
435 enetc_tstamp_tx(tx_swbd->skb, tstamp);
436 do_tstamp = false;
437 }
438 napi_consume_skb(tx_swbd->skb, napi_budget);
439 tx_swbd->skb = NULL;
440 }
441
442 tx_byte_cnt += tx_swbd->len;
443
444 bds_to_clean--;
445 tx_swbd++;
446 i++;
447 if (unlikely(i == tx_ring->bd_count)) {
448 i = 0;
449 tx_swbd = tx_ring->tx_swbd;
450 }
451
452 /* BD iteration loop end */
453 if (is_eof) {
454 tx_frm_cnt++;
455 /* re-arm interrupt source */
456 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
457 BIT(16 + tx_ring->index));
458 }
459
460 if (unlikely(!bds_to_clean))
461 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
462 }
463
464 tx_ring->next_to_clean = i;
465 tx_ring->stats.packets += tx_frm_cnt;
466 tx_ring->stats.bytes += tx_byte_cnt;
467
468 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
469 __netif_subqueue_stopped(ndev, tx_ring->index) &&
470 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
471 netif_wake_subqueue(ndev, tx_ring->index);
472 }
473
474 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
475 }
476
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)477 static bool enetc_new_page(struct enetc_bdr *rx_ring,
478 struct enetc_rx_swbd *rx_swbd)
479 {
480 struct page *page;
481 dma_addr_t addr;
482
483 page = dev_alloc_page();
484 if (unlikely(!page))
485 return false;
486
487 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
488 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
489 __free_page(page);
490
491 return false;
492 }
493
494 rx_swbd->dma = addr;
495 rx_swbd->page = page;
496 rx_swbd->page_offset = ENETC_RXB_PAD;
497
498 return true;
499 }
500
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)501 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
502 {
503 struct enetc_rx_swbd *rx_swbd;
504 union enetc_rx_bd *rxbd;
505 int i, j;
506
507 i = rx_ring->next_to_use;
508 rx_swbd = &rx_ring->rx_swbd[i];
509 rxbd = enetc_rxbd(rx_ring, i);
510
511 for (j = 0; j < buff_cnt; j++) {
512 /* try reuse page */
513 if (unlikely(!rx_swbd->page)) {
514 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
515 rx_ring->stats.rx_alloc_errs++;
516 break;
517 }
518 }
519
520 /* update RxBD */
521 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
522 rx_swbd->page_offset);
523 /* clear 'R" as well */
524 rxbd->r.lstatus = 0;
525
526 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
527 rx_swbd++;
528 i++;
529 if (unlikely(i == rx_ring->bd_count)) {
530 i = 0;
531 rx_swbd = rx_ring->rx_swbd;
532 }
533 }
534
535 if (likely(j)) {
536 rx_ring->next_to_alloc = i; /* keep track from page reuse */
537 rx_ring->next_to_use = i;
538 }
539
540 return j;
541 }
542
543 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)544 static void enetc_get_rx_tstamp(struct net_device *ndev,
545 union enetc_rx_bd *rxbd,
546 struct sk_buff *skb)
547 {
548 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
549 struct enetc_ndev_priv *priv = netdev_priv(ndev);
550 struct enetc_hw *hw = &priv->si->hw;
551 u32 lo, hi, tstamp_lo;
552 u64 tstamp;
553
554 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
555 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
556 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
557 rxbd = enetc_rxbd_ext(rxbd);
558 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
559 if (lo <= tstamp_lo)
560 hi -= 1;
561
562 tstamp = (u64)hi << 32 | tstamp_lo;
563 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
564 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
565 }
566 }
567 #endif
568
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)569 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
570 union enetc_rx_bd *rxbd, struct sk_buff *skb)
571 {
572 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
573
574 /* TODO: hashing */
575 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
576 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
577
578 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
579 skb->ip_summed = CHECKSUM_COMPLETE;
580 }
581
582 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
583 __be16 tpid = 0;
584
585 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
586 case 0:
587 tpid = htons(ETH_P_8021Q);
588 break;
589 case 1:
590 tpid = htons(ETH_P_8021AD);
591 break;
592 case 2:
593 tpid = htons(enetc_port_rd(&priv->si->hw,
594 ENETC_PCVLANR1));
595 break;
596 case 3:
597 tpid = htons(enetc_port_rd(&priv->si->hw,
598 ENETC_PCVLANR2));
599 break;
600 default:
601 break;
602 }
603
604 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
605 }
606
607 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
608 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
609 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
610 #endif
611 }
612
enetc_process_skb(struct enetc_bdr * rx_ring,struct sk_buff * skb)613 static void enetc_process_skb(struct enetc_bdr *rx_ring,
614 struct sk_buff *skb)
615 {
616 skb_record_rx_queue(skb, rx_ring->index);
617 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
618 }
619
enetc_page_reusable(struct page * page)620 static bool enetc_page_reusable(struct page *page)
621 {
622 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
623 }
624
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)625 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
626 struct enetc_rx_swbd *old)
627 {
628 struct enetc_rx_swbd *new;
629
630 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
631
632 /* next buf that may reuse a page */
633 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
634
635 /* copy page reference */
636 *new = *old;
637 }
638
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)639 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
640 int i, u16 size)
641 {
642 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
643
644 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
645 rx_swbd->page_offset,
646 size, DMA_FROM_DEVICE);
647 return rx_swbd;
648 }
649
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)650 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
651 struct enetc_rx_swbd *rx_swbd)
652 {
653 if (likely(enetc_page_reusable(rx_swbd->page))) {
654 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
655 page_ref_inc(rx_swbd->page);
656
657 enetc_reuse_page(rx_ring, rx_swbd);
658
659 /* sync for use by the device */
660 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
661 rx_swbd->page_offset,
662 ENETC_RXB_DMA_SIZE,
663 DMA_FROM_DEVICE);
664 } else {
665 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
666 PAGE_SIZE, DMA_FROM_DEVICE);
667 }
668
669 rx_swbd->page = NULL;
670 }
671
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)672 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
673 int i, u16 size)
674 {
675 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
676 struct sk_buff *skb;
677 void *ba;
678
679 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
680 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
681 if (unlikely(!skb)) {
682 rx_ring->stats.rx_alloc_errs++;
683 return NULL;
684 }
685
686 skb_reserve(skb, ENETC_RXB_PAD);
687 __skb_put(skb, size);
688
689 enetc_put_rx_buff(rx_ring, rx_swbd);
690
691 return skb;
692 }
693
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)694 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
695 u16 size, struct sk_buff *skb)
696 {
697 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
698
699 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
700 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
701
702 enetc_put_rx_buff(rx_ring, rx_swbd);
703 }
704
705 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
706
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)707 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
708 struct napi_struct *napi, int work_limit)
709 {
710 int rx_frm_cnt = 0, rx_byte_cnt = 0;
711 int cleaned_cnt, i;
712
713 cleaned_cnt = enetc_bd_unused(rx_ring);
714 /* next descriptor to process */
715 i = rx_ring->next_to_clean;
716
717 while (likely(rx_frm_cnt < work_limit)) {
718 union enetc_rx_bd *rxbd;
719 struct sk_buff *skb;
720 u32 bd_status;
721 u16 size;
722
723 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
724 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
725
726 /* update ENETC's consumer index */
727 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
728 cleaned_cnt -= count;
729 }
730
731 rxbd = enetc_rxbd(rx_ring, i);
732 bd_status = le32_to_cpu(rxbd->r.lstatus);
733 if (!bd_status)
734 break;
735
736 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
737 dma_rmb(); /* for reading other rxbd fields */
738 size = le16_to_cpu(rxbd->r.buf_len);
739 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
740 if (!skb)
741 break;
742
743 enetc_get_offloads(rx_ring, rxbd, skb);
744
745 cleaned_cnt++;
746
747 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
748 if (unlikely(++i == rx_ring->bd_count))
749 i = 0;
750
751 if (unlikely(bd_status &
752 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
753 dev_kfree_skb(skb);
754 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
755 dma_rmb();
756 bd_status = le32_to_cpu(rxbd->r.lstatus);
757
758 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
759 if (unlikely(++i == rx_ring->bd_count))
760 i = 0;
761 }
762
763 rx_ring->ndev->stats.rx_dropped++;
764 rx_ring->ndev->stats.rx_errors++;
765
766 break;
767 }
768
769 /* not last BD in frame? */
770 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
771 bd_status = le32_to_cpu(rxbd->r.lstatus);
772 size = ENETC_RXB_DMA_SIZE;
773
774 if (bd_status & ENETC_RXBD_LSTATUS_F) {
775 dma_rmb();
776 size = le16_to_cpu(rxbd->r.buf_len);
777 }
778
779 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
780
781 cleaned_cnt++;
782
783 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
784 if (unlikely(++i == rx_ring->bd_count))
785 i = 0;
786 }
787
788 rx_byte_cnt += skb->len;
789
790 enetc_process_skb(rx_ring, skb);
791
792 napi_gro_receive(napi, skb);
793
794 rx_frm_cnt++;
795 }
796
797 rx_ring->next_to_clean = i;
798
799 rx_ring->stats.packets += rx_frm_cnt;
800 rx_ring->stats.bytes += rx_byte_cnt;
801
802 return rx_frm_cnt;
803 }
804
805 /* Probing and Init */
806 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)807 void enetc_get_si_caps(struct enetc_si *si)
808 {
809 struct enetc_hw *hw = &si->hw;
810 u32 val;
811
812 /* find out how many of various resources we have to work with */
813 val = enetc_rd(hw, ENETC_SICAPR0);
814 si->num_rx_rings = (val >> 16) & 0xff;
815 si->num_tx_rings = val & 0xff;
816
817 val = enetc_rd(hw, ENETC_SIRFSCAPR);
818 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
819 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
820
821 si->num_rss = 0;
822 val = enetc_rd(hw, ENETC_SIPCAPR0);
823 if (val & ENETC_SIPCAPR0_RSS) {
824 u32 rss;
825
826 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
827 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
828 }
829
830 if (val & ENETC_SIPCAPR0_QBV)
831 si->hw_features |= ENETC_SI_F_QBV;
832
833 if (val & ENETC_SIPCAPR0_PSFP)
834 si->hw_features |= ENETC_SI_F_PSFP;
835 }
836
enetc_dma_alloc_bdr(struct enetc_bdr * r,size_t bd_size)837 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
838 {
839 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
840 &r->bd_dma_base, GFP_KERNEL);
841 if (!r->bd_base)
842 return -ENOMEM;
843
844 /* h/w requires 128B alignment */
845 if (!IS_ALIGNED(r->bd_dma_base, 128)) {
846 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
847 r->bd_dma_base);
848 return -EINVAL;
849 }
850
851 return 0;
852 }
853
enetc_alloc_txbdr(struct enetc_bdr * txr)854 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
855 {
856 int err;
857
858 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
859 if (!txr->tx_swbd)
860 return -ENOMEM;
861
862 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
863 if (err) {
864 vfree(txr->tx_swbd);
865 return err;
866 }
867
868 txr->next_to_clean = 0;
869 txr->next_to_use = 0;
870
871 return 0;
872 }
873
enetc_free_txbdr(struct enetc_bdr * txr)874 static void enetc_free_txbdr(struct enetc_bdr *txr)
875 {
876 int size, i;
877
878 for (i = 0; i < txr->bd_count; i++)
879 enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
880
881 size = txr->bd_count * sizeof(union enetc_tx_bd);
882
883 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
884 txr->bd_base = NULL;
885
886 vfree(txr->tx_swbd);
887 txr->tx_swbd = NULL;
888 }
889
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)890 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
891 {
892 int i, err;
893
894 for (i = 0; i < priv->num_tx_rings; i++) {
895 err = enetc_alloc_txbdr(priv->tx_ring[i]);
896
897 if (err)
898 goto fail;
899 }
900
901 return 0;
902
903 fail:
904 while (i-- > 0)
905 enetc_free_txbdr(priv->tx_ring[i]);
906
907 return err;
908 }
909
enetc_free_tx_resources(struct enetc_ndev_priv * priv)910 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
911 {
912 int i;
913
914 for (i = 0; i < priv->num_tx_rings; i++)
915 enetc_free_txbdr(priv->tx_ring[i]);
916 }
917
enetc_alloc_rxbdr(struct enetc_bdr * rxr,bool extended)918 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
919 {
920 size_t size = sizeof(union enetc_rx_bd);
921 int err;
922
923 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
924 if (!rxr->rx_swbd)
925 return -ENOMEM;
926
927 if (extended)
928 size *= 2;
929
930 err = enetc_dma_alloc_bdr(rxr, size);
931 if (err) {
932 vfree(rxr->rx_swbd);
933 return err;
934 }
935
936 rxr->next_to_clean = 0;
937 rxr->next_to_use = 0;
938 rxr->next_to_alloc = 0;
939 rxr->ext_en = extended;
940
941 return 0;
942 }
943
enetc_free_rxbdr(struct enetc_bdr * rxr)944 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
945 {
946 int size;
947
948 size = rxr->bd_count * sizeof(union enetc_rx_bd);
949
950 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
951 rxr->bd_base = NULL;
952
953 vfree(rxr->rx_swbd);
954 rxr->rx_swbd = NULL;
955 }
956
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv)957 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
958 {
959 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
960 int i, err;
961
962 for (i = 0; i < priv->num_rx_rings; i++) {
963 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
964
965 if (err)
966 goto fail;
967 }
968
969 return 0;
970
971 fail:
972 while (i-- > 0)
973 enetc_free_rxbdr(priv->rx_ring[i]);
974
975 return err;
976 }
977
enetc_free_rx_resources(struct enetc_ndev_priv * priv)978 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
979 {
980 int i;
981
982 for (i = 0; i < priv->num_rx_rings; i++)
983 enetc_free_rxbdr(priv->rx_ring[i]);
984 }
985
enetc_free_tx_ring(struct enetc_bdr * tx_ring)986 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
987 {
988 int i;
989
990 if (!tx_ring->tx_swbd)
991 return;
992
993 for (i = 0; i < tx_ring->bd_count; i++) {
994 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
995
996 enetc_free_tx_skb(tx_ring, tx_swbd);
997 }
998
999 tx_ring->next_to_clean = 0;
1000 tx_ring->next_to_use = 0;
1001 }
1002
enetc_free_rx_ring(struct enetc_bdr * rx_ring)1003 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1004 {
1005 int i;
1006
1007 if (!rx_ring->rx_swbd)
1008 return;
1009
1010 for (i = 0; i < rx_ring->bd_count; i++) {
1011 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1012
1013 if (!rx_swbd->page)
1014 continue;
1015
1016 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
1017 PAGE_SIZE, DMA_FROM_DEVICE);
1018 __free_page(rx_swbd->page);
1019 rx_swbd->page = NULL;
1020 }
1021
1022 rx_ring->next_to_clean = 0;
1023 rx_ring->next_to_use = 0;
1024 rx_ring->next_to_alloc = 0;
1025 }
1026
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)1027 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1028 {
1029 int i;
1030
1031 for (i = 0; i < priv->num_rx_rings; i++)
1032 enetc_free_rx_ring(priv->rx_ring[i]);
1033
1034 for (i = 0; i < priv->num_tx_rings; i++)
1035 enetc_free_tx_ring(priv->tx_ring[i]);
1036 }
1037
enetc_alloc_cbdr(struct device * dev,struct enetc_cbdr * cbdr)1038 int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1039 {
1040 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1041
1042 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
1043 GFP_KERNEL);
1044 if (!cbdr->bd_base)
1045 return -ENOMEM;
1046
1047 /* h/w requires 128B alignment */
1048 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
1049 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1050 return -EINVAL;
1051 }
1052
1053 cbdr->next_to_clean = 0;
1054 cbdr->next_to_use = 0;
1055
1056 return 0;
1057 }
1058
enetc_free_cbdr(struct device * dev,struct enetc_cbdr * cbdr)1059 void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1060 {
1061 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1062
1063 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1064 cbdr->bd_base = NULL;
1065 }
1066
enetc_setup_cbdr(struct enetc_hw * hw,struct enetc_cbdr * cbdr)1067 void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
1068 {
1069 /* set CBDR cache attributes */
1070 enetc_wr(hw, ENETC_SICAR2,
1071 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1072
1073 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
1074 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
1075 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
1076
1077 enetc_wr(hw, ENETC_SICBDRPIR, 0);
1078 enetc_wr(hw, ENETC_SICBDRCIR, 0);
1079
1080 /* enable ring */
1081 enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
1082
1083 cbdr->pir = hw->reg + ENETC_SICBDRPIR;
1084 cbdr->cir = hw->reg + ENETC_SICBDRCIR;
1085 }
1086
enetc_clear_cbdr(struct enetc_hw * hw)1087 void enetc_clear_cbdr(struct enetc_hw *hw)
1088 {
1089 enetc_wr(hw, ENETC_SICBDRMR, 0);
1090 }
1091
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)1092 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1093 {
1094 int *rss_table;
1095 int i;
1096
1097 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1098 if (!rss_table)
1099 return -ENOMEM;
1100
1101 /* Set up RSS table defaults */
1102 for (i = 0; i < si->num_rss; i++)
1103 rss_table[i] = i % num_groups;
1104
1105 enetc_set_rss_table(si, rss_table, si->num_rss);
1106
1107 kfree(rss_table);
1108
1109 return 0;
1110 }
1111
enetc_configure_si(struct enetc_ndev_priv * priv)1112 int enetc_configure_si(struct enetc_ndev_priv *priv)
1113 {
1114 struct enetc_si *si = priv->si;
1115 struct enetc_hw *hw = &si->hw;
1116 int err;
1117
1118 /* set SI cache attributes */
1119 enetc_wr(hw, ENETC_SICAR0,
1120 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1121 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1122 /* enable SI */
1123 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1124
1125 if (si->num_rss) {
1126 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1127 if (err)
1128 return err;
1129 }
1130
1131 return 0;
1132 }
1133
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)1134 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1135 {
1136 struct enetc_si *si = priv->si;
1137 int cpus = num_online_cpus();
1138
1139 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1140 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1141
1142 /* Enable all available TX rings in order to configure as many
1143 * priorities as possible, when needed.
1144 * TODO: Make # of TX rings run-time configurable
1145 */
1146 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1147 priv->num_tx_rings = si->num_tx_rings;
1148 priv->bdr_int_num = cpus;
1149 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1150 priv->tx_ictt = ENETC_TXIC_TIMETHR;
1151
1152 /* SI specific */
1153 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1154 }
1155
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)1156 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1157 {
1158 struct enetc_si *si = priv->si;
1159 int err;
1160
1161 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1162 if (err)
1163 return err;
1164
1165 enetc_setup_cbdr(&si->hw, &si->cbd_ring);
1166
1167 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1168 GFP_KERNEL);
1169 if (!priv->cls_rules) {
1170 err = -ENOMEM;
1171 goto err_alloc_cls;
1172 }
1173
1174 return 0;
1175
1176 err_alloc_cls:
1177 enetc_clear_cbdr(&si->hw);
1178 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1179
1180 return err;
1181 }
1182
enetc_free_si_resources(struct enetc_ndev_priv * priv)1183 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1184 {
1185 struct enetc_si *si = priv->si;
1186
1187 enetc_clear_cbdr(&si->hw);
1188 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1189
1190 kfree(priv->cls_rules);
1191 }
1192
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1193 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1194 {
1195 int idx = tx_ring->index;
1196 u32 tbmr;
1197
1198 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1199 lower_32_bits(tx_ring->bd_dma_base));
1200
1201 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1202 upper_32_bits(tx_ring->bd_dma_base));
1203
1204 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1205 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1206 ENETC_RTBLENR_LEN(tx_ring->bd_count));
1207
1208 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1209 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1210 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1211
1212 /* enable Tx ints by setting pkt thr to 1 */
1213 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1214
1215 tbmr = ENETC_TBMR_EN | ENETC_TBMR_SET_PRIO(tx_ring->prio);
1216 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1217 tbmr |= ENETC_TBMR_VIH;
1218
1219 /* enable ring */
1220 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1221
1222 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1223 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1224 tx_ring->idr = hw->reg + ENETC_SITXIDR;
1225 }
1226
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1227 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1228 {
1229 int idx = rx_ring->index;
1230 u32 rbmr;
1231
1232 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1233 lower_32_bits(rx_ring->bd_dma_base));
1234
1235 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1236 upper_32_bits(rx_ring->bd_dma_base));
1237
1238 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1239 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1240 ENETC_RTBLENR_LEN(rx_ring->bd_count));
1241
1242 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1243
1244 /* Also prepare the consumer index in case page allocation never
1245 * succeeds. In that case, hardware will never advance producer index
1246 * to match consumer index, and will drop all frames.
1247 */
1248 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1249 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
1250
1251 /* enable Rx ints by setting pkt thr to 1 */
1252 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1253
1254 rbmr = ENETC_RBMR_EN;
1255
1256 if (rx_ring->ext_en)
1257 rbmr |= ENETC_RBMR_BDS;
1258
1259 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1260 rbmr |= ENETC_RBMR_VTE;
1261
1262 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1263 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1264
1265 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1266 /* update ENETC's consumer index */
1267 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, rx_ring->next_to_use);
1268
1269 /* enable ring */
1270 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1271 }
1272
enetc_setup_bdrs(struct enetc_ndev_priv * priv)1273 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1274 {
1275 struct enetc_hw *hw = &priv->si->hw;
1276 int i;
1277
1278 for (i = 0; i < priv->num_tx_rings; i++)
1279 enetc_setup_txbdr(hw, priv->tx_ring[i]);
1280
1281 for (i = 0; i < priv->num_rx_rings; i++)
1282 enetc_setup_rxbdr(hw, priv->rx_ring[i]);
1283 }
1284
enetc_clear_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1285 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1286 {
1287 int idx = rx_ring->index;
1288
1289 /* disable EN bit on ring */
1290 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1291 }
1292
enetc_clear_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1293 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1294 {
1295 int delay = 8, timeout = 100;
1296 int idx = tx_ring->index;
1297
1298 /* disable EN bit on ring */
1299 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1300
1301 /* wait for busy to clear */
1302 while (delay < timeout &&
1303 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1304 msleep(delay);
1305 delay *= 2;
1306 }
1307
1308 if (delay >= timeout)
1309 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1310 idx);
1311 }
1312
enetc_clear_bdrs(struct enetc_ndev_priv * priv)1313 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1314 {
1315 struct enetc_hw *hw = &priv->si->hw;
1316 int i;
1317
1318 for (i = 0; i < priv->num_tx_rings; i++)
1319 enetc_clear_txbdr(hw, priv->tx_ring[i]);
1320
1321 for (i = 0; i < priv->num_rx_rings; i++)
1322 enetc_clear_rxbdr(hw, priv->rx_ring[i]);
1323
1324 udelay(1);
1325 }
1326
enetc_setup_irqs(struct enetc_ndev_priv * priv)1327 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1328 {
1329 struct pci_dev *pdev = priv->si->pdev;
1330 struct enetc_hw *hw = &priv->si->hw;
1331 int i, j, err;
1332
1333 for (i = 0; i < priv->bdr_int_num; i++) {
1334 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1335 struct enetc_int_vector *v = priv->int_vector[i];
1336 int entry = ENETC_BDR_INT_BASE_IDX + i;
1337
1338 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1339 priv->ndev->name, i);
1340 err = request_irq(irq, enetc_msix, 0, v->name, v);
1341 if (err) {
1342 dev_err(priv->dev, "request_irq() failed!\n");
1343 goto irq_err;
1344 }
1345 disable_irq(irq);
1346
1347 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1348 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1349 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1350
1351 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1352
1353 for (j = 0; j < v->count_tx_rings; j++) {
1354 int idx = v->tx_ring[j].index;
1355
1356 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1357 }
1358 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
1359 }
1360
1361 return 0;
1362
1363 irq_err:
1364 while (i--) {
1365 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1366
1367 irq_set_affinity_hint(irq, NULL);
1368 free_irq(irq, priv->int_vector[i]);
1369 }
1370
1371 return err;
1372 }
1373
enetc_free_irqs(struct enetc_ndev_priv * priv)1374 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1375 {
1376 struct pci_dev *pdev = priv->si->pdev;
1377 int i;
1378
1379 for (i = 0; i < priv->bdr_int_num; i++) {
1380 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1381
1382 irq_set_affinity_hint(irq, NULL);
1383 free_irq(irq, priv->int_vector[i]);
1384 }
1385 }
1386
enetc_setup_interrupts(struct enetc_ndev_priv * priv)1387 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1388 {
1389 struct enetc_hw *hw = &priv->si->hw;
1390 u32 icpt, ictt;
1391 int i;
1392
1393 /* enable Tx & Rx event indication */
1394 if (priv->ic_mode &
1395 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1396 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1397 /* init to non-0 minimum, will be adjusted later */
1398 ictt = 0x1;
1399 } else {
1400 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1401 ictt = 0;
1402 }
1403
1404 for (i = 0; i < priv->num_rx_rings; i++) {
1405 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1406 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1407 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1408 }
1409
1410 if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1411 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1412 else
1413 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1414
1415 for (i = 0; i < priv->num_tx_rings; i++) {
1416 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1417 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1418 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1419 }
1420 }
1421
enetc_clear_interrupts(struct enetc_ndev_priv * priv)1422 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1423 {
1424 struct enetc_hw *hw = &priv->si->hw;
1425 int i;
1426
1427 for (i = 0; i < priv->num_tx_rings; i++)
1428 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
1429
1430 for (i = 0; i < priv->num_rx_rings; i++)
1431 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
1432 }
1433
enetc_phylink_connect(struct net_device * ndev)1434 static int enetc_phylink_connect(struct net_device *ndev)
1435 {
1436 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1437 struct ethtool_eee edata;
1438 int err;
1439
1440 if (!priv->phylink)
1441 return 0; /* phy-less mode */
1442
1443 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1444 if (err) {
1445 dev_err(&ndev->dev, "could not attach to PHY\n");
1446 return err;
1447 }
1448
1449 /* disable EEE autoneg, until ENETC driver supports it */
1450 memset(&edata, 0, sizeof(struct ethtool_eee));
1451 phylink_ethtool_set_eee(priv->phylink, &edata);
1452
1453 return 0;
1454 }
1455
enetc_start(struct net_device * ndev)1456 void enetc_start(struct net_device *ndev)
1457 {
1458 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1459 int i;
1460
1461 enetc_setup_interrupts(priv);
1462
1463 for (i = 0; i < priv->bdr_int_num; i++) {
1464 int irq = pci_irq_vector(priv->si->pdev,
1465 ENETC_BDR_INT_BASE_IDX + i);
1466
1467 napi_enable(&priv->int_vector[i]->napi);
1468 enable_irq(irq);
1469 }
1470
1471 if (priv->phylink)
1472 phylink_start(priv->phylink);
1473 else
1474 netif_carrier_on(ndev);
1475
1476 netif_tx_start_all_queues(ndev);
1477 }
1478
enetc_open(struct net_device * ndev)1479 int enetc_open(struct net_device *ndev)
1480 {
1481 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1482 int err;
1483
1484 err = enetc_setup_irqs(priv);
1485 if (err)
1486 return err;
1487
1488 err = enetc_phylink_connect(ndev);
1489 if (err)
1490 goto err_phy_connect;
1491
1492 err = enetc_alloc_tx_resources(priv);
1493 if (err)
1494 goto err_alloc_tx;
1495
1496 err = enetc_alloc_rx_resources(priv);
1497 if (err)
1498 goto err_alloc_rx;
1499
1500 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1501 if (err)
1502 goto err_set_queues;
1503
1504 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1505 if (err)
1506 goto err_set_queues;
1507
1508 enetc_setup_bdrs(priv);
1509 enetc_start(ndev);
1510
1511 return 0;
1512
1513 err_set_queues:
1514 enetc_free_rx_resources(priv);
1515 err_alloc_rx:
1516 enetc_free_tx_resources(priv);
1517 err_alloc_tx:
1518 if (priv->phylink)
1519 phylink_disconnect_phy(priv->phylink);
1520 err_phy_connect:
1521 enetc_free_irqs(priv);
1522
1523 return err;
1524 }
1525
enetc_stop(struct net_device * ndev)1526 void enetc_stop(struct net_device *ndev)
1527 {
1528 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1529 int i;
1530
1531 netif_tx_stop_all_queues(ndev);
1532
1533 for (i = 0; i < priv->bdr_int_num; i++) {
1534 int irq = pci_irq_vector(priv->si->pdev,
1535 ENETC_BDR_INT_BASE_IDX + i);
1536
1537 disable_irq(irq);
1538 napi_synchronize(&priv->int_vector[i]->napi);
1539 napi_disable(&priv->int_vector[i]->napi);
1540 }
1541
1542 if (priv->phylink)
1543 phylink_stop(priv->phylink);
1544 else
1545 netif_carrier_off(ndev);
1546
1547 enetc_clear_interrupts(priv);
1548 }
1549
enetc_close(struct net_device * ndev)1550 int enetc_close(struct net_device *ndev)
1551 {
1552 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1553
1554 enetc_stop(ndev);
1555 enetc_clear_bdrs(priv);
1556
1557 if (priv->phylink)
1558 phylink_disconnect_phy(priv->phylink);
1559 enetc_free_rxtx_rings(priv);
1560 enetc_free_rx_resources(priv);
1561 enetc_free_tx_resources(priv);
1562 enetc_free_irqs(priv);
1563
1564 return 0;
1565 }
1566
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)1567 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1568 {
1569 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1570 struct tc_mqprio_qopt *mqprio = type_data;
1571 struct enetc_hw *hw = &priv->si->hw;
1572 struct enetc_bdr *tx_ring;
1573 u8 num_tc;
1574 int i;
1575
1576 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1577 num_tc = mqprio->num_tc;
1578
1579 if (!num_tc) {
1580 netdev_reset_tc(ndev);
1581 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1582
1583 /* Reset all ring priorities to 0 */
1584 for (i = 0; i < priv->num_tx_rings; i++) {
1585 tx_ring = priv->tx_ring[i];
1586 tx_ring->prio = 0;
1587 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
1588 }
1589
1590 return 0;
1591 }
1592
1593 /* Check if we have enough BD rings available to accommodate all TCs */
1594 if (num_tc > priv->num_tx_rings) {
1595 netdev_err(ndev, "Max %d traffic classes supported\n",
1596 priv->num_tx_rings);
1597 return -EINVAL;
1598 }
1599
1600 /* For the moment, we use only one BD ring per TC.
1601 *
1602 * Configure num_tc BD rings with increasing priorities.
1603 */
1604 for (i = 0; i < num_tc; i++) {
1605 tx_ring = priv->tx_ring[i];
1606 tx_ring->prio = i;
1607 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
1608 }
1609
1610 /* Reset the number of netdev queues based on the TC count */
1611 netif_set_real_num_tx_queues(ndev, num_tc);
1612
1613 netdev_set_num_tc(ndev, num_tc);
1614
1615 /* Each TC is associated with one netdev queue */
1616 for (i = 0; i < num_tc; i++)
1617 netdev_set_tc_queue(ndev, i, 1, i);
1618
1619 return 0;
1620 }
1621
enetc_setup_tc(struct net_device * ndev,enum tc_setup_type type,void * type_data)1622 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1623 void *type_data)
1624 {
1625 switch (type) {
1626 case TC_SETUP_QDISC_MQPRIO:
1627 return enetc_setup_tc_mqprio(ndev, type_data);
1628 case TC_SETUP_QDISC_TAPRIO:
1629 return enetc_setup_tc_taprio(ndev, type_data);
1630 case TC_SETUP_QDISC_CBS:
1631 return enetc_setup_tc_cbs(ndev, type_data);
1632 case TC_SETUP_QDISC_ETF:
1633 return enetc_setup_tc_txtime(ndev, type_data);
1634 case TC_SETUP_BLOCK:
1635 return enetc_setup_tc_psfp(ndev, type_data);
1636 default:
1637 return -EOPNOTSUPP;
1638 }
1639 }
1640
enetc_get_stats(struct net_device * ndev)1641 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1642 {
1643 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1644 struct net_device_stats *stats = &ndev->stats;
1645 unsigned long packets = 0, bytes = 0;
1646 int i;
1647
1648 for (i = 0; i < priv->num_rx_rings; i++) {
1649 packets += priv->rx_ring[i]->stats.packets;
1650 bytes += priv->rx_ring[i]->stats.bytes;
1651 }
1652
1653 stats->rx_packets = packets;
1654 stats->rx_bytes = bytes;
1655 bytes = 0;
1656 packets = 0;
1657
1658 for (i = 0; i < priv->num_tx_rings; i++) {
1659 packets += priv->tx_ring[i]->stats.packets;
1660 bytes += priv->tx_ring[i]->stats.bytes;
1661 }
1662
1663 stats->tx_packets = packets;
1664 stats->tx_bytes = bytes;
1665
1666 return stats;
1667 }
1668
enetc_set_rss(struct net_device * ndev,int en)1669 static int enetc_set_rss(struct net_device *ndev, int en)
1670 {
1671 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1672 struct enetc_hw *hw = &priv->si->hw;
1673 u32 reg;
1674
1675 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1676
1677 reg = enetc_rd(hw, ENETC_SIMR);
1678 reg &= ~ENETC_SIMR_RSSE;
1679 reg |= (en) ? ENETC_SIMR_RSSE : 0;
1680 enetc_wr(hw, ENETC_SIMR, reg);
1681
1682 return 0;
1683 }
1684
enetc_enable_rxvlan(struct net_device * ndev,bool en)1685 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
1686 {
1687 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1688 struct enetc_hw *hw = &priv->si->hw;
1689 int i;
1690
1691 for (i = 0; i < priv->num_rx_rings; i++)
1692 enetc_bdr_enable_rxvlan(hw, i, en);
1693 }
1694
enetc_enable_txvlan(struct net_device * ndev,bool en)1695 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
1696 {
1697 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1698 struct enetc_hw *hw = &priv->si->hw;
1699 int i;
1700
1701 for (i = 0; i < priv->num_tx_rings; i++)
1702 enetc_bdr_enable_txvlan(hw, i, en);
1703 }
1704
enetc_set_features(struct net_device * ndev,netdev_features_t features)1705 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
1706 {
1707 netdev_features_t changed = ndev->features ^ features;
1708
1709 if (changed & NETIF_F_RXHASH)
1710 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1711
1712 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1713 enetc_enable_rxvlan(ndev,
1714 !!(features & NETIF_F_HW_VLAN_CTAG_RX));
1715
1716 if (changed & NETIF_F_HW_VLAN_CTAG_TX)
1717 enetc_enable_txvlan(ndev,
1718 !!(features & NETIF_F_HW_VLAN_CTAG_TX));
1719 }
1720
1721 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)1722 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1723 {
1724 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1725 struct hwtstamp_config config;
1726 int ao;
1727
1728 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1729 return -EFAULT;
1730
1731 switch (config.tx_type) {
1732 case HWTSTAMP_TX_OFF:
1733 priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1734 break;
1735 case HWTSTAMP_TX_ON:
1736 priv->active_offloads |= ENETC_F_TX_TSTAMP;
1737 break;
1738 default:
1739 return -ERANGE;
1740 }
1741
1742 ao = priv->active_offloads;
1743 switch (config.rx_filter) {
1744 case HWTSTAMP_FILTER_NONE:
1745 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1746 break;
1747 default:
1748 priv->active_offloads |= ENETC_F_RX_TSTAMP;
1749 config.rx_filter = HWTSTAMP_FILTER_ALL;
1750 }
1751
1752 if (netif_running(ndev) && ao != priv->active_offloads) {
1753 enetc_close(ndev);
1754 enetc_open(ndev);
1755 }
1756
1757 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1758 -EFAULT : 0;
1759 }
1760
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)1761 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1762 {
1763 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1764 struct hwtstamp_config config;
1765
1766 config.flags = 0;
1767
1768 if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1769 config.tx_type = HWTSTAMP_TX_ON;
1770 else
1771 config.tx_type = HWTSTAMP_TX_OFF;
1772
1773 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1774 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1775
1776 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1777 -EFAULT : 0;
1778 }
1779 #endif
1780
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)1781 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1782 {
1783 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1784 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1785 if (cmd == SIOCSHWTSTAMP)
1786 return enetc_hwtstamp_set(ndev, rq);
1787 if (cmd == SIOCGHWTSTAMP)
1788 return enetc_hwtstamp_get(ndev, rq);
1789 #endif
1790
1791 if (!priv->phylink)
1792 return -EOPNOTSUPP;
1793
1794 return phylink_mii_ioctl(priv->phylink, rq, cmd);
1795 }
1796
enetc_alloc_msix(struct enetc_ndev_priv * priv)1797 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1798 {
1799 struct pci_dev *pdev = priv->si->pdev;
1800 int v_tx_rings;
1801 int i, n, err, nvec;
1802
1803 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1804 /* allocate MSIX for both messaging and Rx/Tx interrupts */
1805 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1806
1807 if (n < 0)
1808 return n;
1809
1810 if (n != nvec)
1811 return -EPERM;
1812
1813 /* # of tx rings per int vector */
1814 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1815
1816 for (i = 0; i < priv->bdr_int_num; i++) {
1817 struct enetc_int_vector *v;
1818 struct enetc_bdr *bdr;
1819 int j;
1820
1821 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
1822 if (!v) {
1823 err = -ENOMEM;
1824 goto fail;
1825 }
1826
1827 priv->int_vector[i] = v;
1828
1829 /* init defaults for adaptive IC */
1830 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
1831 v->rx_ictt = 0x1;
1832 v->rx_dim_en = true;
1833 }
1834 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
1835 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1836 NAPI_POLL_WEIGHT);
1837 v->count_tx_rings = v_tx_rings;
1838
1839 for (j = 0; j < v_tx_rings; j++) {
1840 int idx;
1841
1842 /* default tx ring mapping policy */
1843 if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1844 idx = 2 * j + i; /* 2 CPUs */
1845 else
1846 idx = j + i * v_tx_rings; /* default */
1847
1848 __set_bit(idx, &v->tx_rings_map);
1849 bdr = &v->tx_ring[j];
1850 bdr->index = idx;
1851 bdr->ndev = priv->ndev;
1852 bdr->dev = priv->dev;
1853 bdr->bd_count = priv->tx_bd_count;
1854 priv->tx_ring[idx] = bdr;
1855 }
1856
1857 bdr = &v->rx_ring;
1858 bdr->index = i;
1859 bdr->ndev = priv->ndev;
1860 bdr->dev = priv->dev;
1861 bdr->bd_count = priv->rx_bd_count;
1862 priv->rx_ring[i] = bdr;
1863 }
1864
1865 return 0;
1866
1867 fail:
1868 while (i--) {
1869 netif_napi_del(&priv->int_vector[i]->napi);
1870 cancel_work_sync(&priv->int_vector[i]->rx_dim.work);
1871 kfree(priv->int_vector[i]);
1872 }
1873
1874 pci_free_irq_vectors(pdev);
1875
1876 return err;
1877 }
1878
enetc_free_msix(struct enetc_ndev_priv * priv)1879 void enetc_free_msix(struct enetc_ndev_priv *priv)
1880 {
1881 int i;
1882
1883 for (i = 0; i < priv->bdr_int_num; i++) {
1884 struct enetc_int_vector *v = priv->int_vector[i];
1885
1886 netif_napi_del(&v->napi);
1887 cancel_work_sync(&v->rx_dim.work);
1888 }
1889
1890 for (i = 0; i < priv->num_rx_rings; i++)
1891 priv->rx_ring[i] = NULL;
1892
1893 for (i = 0; i < priv->num_tx_rings; i++)
1894 priv->tx_ring[i] = NULL;
1895
1896 for (i = 0; i < priv->bdr_int_num; i++) {
1897 kfree(priv->int_vector[i]);
1898 priv->int_vector[i] = NULL;
1899 }
1900
1901 /* disable all MSIX for this device */
1902 pci_free_irq_vectors(priv->si->pdev);
1903 }
1904
enetc_kfree_si(struct enetc_si * si)1905 static void enetc_kfree_si(struct enetc_si *si)
1906 {
1907 char *p = (char *)si - si->pad;
1908
1909 kfree(p);
1910 }
1911
enetc_detect_errata(struct enetc_si * si)1912 static void enetc_detect_errata(struct enetc_si *si)
1913 {
1914 if (si->pdev->revision == ENETC_REV1)
1915 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1916 ENETC_ERR_UCMCSWP;
1917 }
1918
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)1919 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1920 {
1921 struct enetc_si *si, *p;
1922 struct enetc_hw *hw;
1923 size_t alloc_size;
1924 int err, len;
1925
1926 pcie_flr(pdev);
1927 err = pci_enable_device_mem(pdev);
1928 if (err) {
1929 dev_err(&pdev->dev, "device enable failed\n");
1930 return err;
1931 }
1932
1933 /* set up for high or low dma */
1934 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1935 if (err) {
1936 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1937 if (err) {
1938 dev_err(&pdev->dev,
1939 "DMA configuration failed: 0x%x\n", err);
1940 goto err_dma;
1941 }
1942 }
1943
1944 err = pci_request_mem_regions(pdev, name);
1945 if (err) {
1946 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1947 goto err_pci_mem_reg;
1948 }
1949
1950 pci_set_master(pdev);
1951
1952 alloc_size = sizeof(struct enetc_si);
1953 if (sizeof_priv) {
1954 /* align priv to 32B */
1955 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1956 alloc_size += sizeof_priv;
1957 }
1958 /* force 32B alignment for enetc_si */
1959 alloc_size += ENETC_SI_ALIGN - 1;
1960
1961 p = kzalloc(alloc_size, GFP_KERNEL);
1962 if (!p) {
1963 err = -ENOMEM;
1964 goto err_alloc_si;
1965 }
1966
1967 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1968 si->pad = (char *)si - (char *)p;
1969
1970 pci_set_drvdata(pdev, si);
1971 si->pdev = pdev;
1972 hw = &si->hw;
1973
1974 len = pci_resource_len(pdev, ENETC_BAR_REGS);
1975 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1976 if (!hw->reg) {
1977 err = -ENXIO;
1978 dev_err(&pdev->dev, "ioremap() failed\n");
1979 goto err_ioremap;
1980 }
1981 if (len > ENETC_PORT_BASE)
1982 hw->port = hw->reg + ENETC_PORT_BASE;
1983 if (len > ENETC_GLOBAL_BASE)
1984 hw->global = hw->reg + ENETC_GLOBAL_BASE;
1985
1986 enetc_detect_errata(si);
1987
1988 return 0;
1989
1990 err_ioremap:
1991 enetc_kfree_si(si);
1992 err_alloc_si:
1993 pci_release_mem_regions(pdev);
1994 err_pci_mem_reg:
1995 err_dma:
1996 pci_disable_device(pdev);
1997
1998 return err;
1999 }
2000
enetc_pci_remove(struct pci_dev * pdev)2001 void enetc_pci_remove(struct pci_dev *pdev)
2002 {
2003 struct enetc_si *si = pci_get_drvdata(pdev);
2004 struct enetc_hw *hw = &si->hw;
2005
2006 iounmap(hw->reg);
2007 enetc_kfree_si(si);
2008 pci_release_mem_regions(pdev);
2009 pci_disable_device(pdev);
2010 }
2011