1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/skbuff.h>
7 #include <linux/module.h>
8 #include <linux/highmem.h>
9 #include <linux/ip.h>
10 #include <net/ipv6.h>
11 #include <linux/netdevice.h>
12 #include <crypto/aes.h>
13 #include "chcr_ktls.h"
14
15 static LIST_HEAD(uld_ctx_list);
16 static DEFINE_MUTEX(dev_mutex);
17
18 /* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
19 * @skb: skb
20 * @start: start offset.
21 * @len: how much data to send after @start
22 */
chcr_get_nfrags_to_send(struct sk_buff * skb,u32 start,u32 len)23 static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
24 {
25 struct skb_shared_info *si = skb_shinfo(skb);
26 u32 frag_size, skb_linear_data_len = skb_headlen(skb);
27 u8 nfrags = 0, frag_idx = 0;
28 skb_frag_t *frag;
29
30 /* if its a linear skb then return 1 */
31 if (!skb_is_nonlinear(skb))
32 return 1;
33
34 if (unlikely(start < skb_linear_data_len)) {
35 frag_size = min(len, skb_linear_data_len - start);
36 start = 0;
37 } else {
38 start -= skb_linear_data_len;
39
40 frag = &si->frags[frag_idx];
41 frag_size = skb_frag_size(frag);
42 while (start >= frag_size) {
43 start -= frag_size;
44 frag_idx++;
45 frag = &si->frags[frag_idx];
46 frag_size = skb_frag_size(frag);
47 }
48 frag_size = min(len, skb_frag_size(frag) - start);
49 }
50 len -= frag_size;
51 nfrags++;
52
53 while (len) {
54 frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
55 len -= frag_size;
56 nfrags++;
57 frag_idx++;
58 }
59 return nfrags;
60 }
61
62 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
63 static void clear_conn_resources(struct chcr_ktls_info *tx_info);
64 /*
65 * chcr_ktls_save_keys: calculate and save crypto keys.
66 * @tx_info - driver specific tls info.
67 * @crypto_info - tls crypto information.
68 * @direction - TX/RX direction.
69 * return - SUCCESS/FAILURE.
70 */
chcr_ktls_save_keys(struct chcr_ktls_info * tx_info,struct tls_crypto_info * crypto_info,enum tls_offload_ctx_dir direction)71 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
72 struct tls_crypto_info *crypto_info,
73 enum tls_offload_ctx_dir direction)
74 {
75 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
76 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
77 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
78 struct ktls_key_ctx *kctx = &tx_info->key_ctx;
79 struct crypto_aes_ctx aes_ctx;
80 unsigned char *key, *salt;
81
82 switch (crypto_info->cipher_type) {
83 case TLS_CIPHER_AES_GCM_128:
84 info_128_gcm =
85 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
86 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
87 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
88 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
89 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
90 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
91 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
92
93 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
94 key = info_128_gcm->key;
95 salt = info_128_gcm->salt;
96 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
97
98 /* The SCMD fields used when encrypting a full TLS
99 * record. Its a one time calculation till the
100 * connection exists.
101 */
102 tx_info->scmd0_seqno_numivs =
103 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
104 SCMD_CIPH_AUTH_SEQ_CTRL_F |
105 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
106 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
107 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
108 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
109 SCMD_NUM_IVS_V(1);
110
111 /* keys will be sent inline. */
112 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
113
114 /* The SCMD fields used when encrypting a partial TLS
115 * record (no trailer and possibly a truncated payload).
116 */
117 tx_info->scmd0_short_seqno_numivs =
118 SCMD_CIPH_AUTH_SEQ_CTRL_F |
119 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
120 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
121 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
122
123 tx_info->scmd0_short_ivgen_hdrlen =
124 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
125
126 break;
127
128 default:
129 pr_err("GCM: cipher type 0x%x not supported\n",
130 crypto_info->cipher_type);
131 ret = -EINVAL;
132 goto out;
133 }
134
135 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
136 roundup(keylen, 16) + ghash_size;
137 /* Calculate the H = CIPH(K, 0 repeated 16 times).
138 * It will go in key context
139 */
140
141 ret = aes_expandkey(&aes_ctx, key, keylen);
142 if (ret)
143 goto out;
144
145 memset(ghash_h, 0, ghash_size);
146 aes_encrypt(&aes_ctx, ghash_h, ghash_h);
147 memzero_explicit(&aes_ctx, sizeof(aes_ctx));
148
149 /* fill the Key context */
150 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
151 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
152 mac_key_size,
153 key_ctx_size >> 4);
154 } else {
155 ret = -EINVAL;
156 goto out;
157 }
158
159 memcpy(kctx->salt, salt, tx_info->salt_size);
160 memcpy(kctx->key, key, keylen);
161 memcpy(kctx->key + keylen, ghash_h, ghash_size);
162 tx_info->key_ctx_len = key_ctx_size;
163
164 out:
165 return ret;
166 }
167
168 /*
169 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
170 * @sk - tcp socket.
171 * @tx_info - driver specific tls info.
172 * @atid - connection active tid.
173 * return - send success/failure.
174 */
chcr_ktls_act_open_req(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)175 static int chcr_ktls_act_open_req(struct sock *sk,
176 struct chcr_ktls_info *tx_info,
177 int atid)
178 {
179 struct inet_sock *inet = inet_sk(sk);
180 struct cpl_t6_act_open_req *cpl6;
181 struct cpl_act_open_req *cpl;
182 struct sk_buff *skb;
183 unsigned int len;
184 int qid_atid;
185 u64 options;
186
187 len = sizeof(*cpl6);
188 skb = alloc_skb(len, GFP_KERNEL);
189 if (unlikely(!skb))
190 return -ENOMEM;
191 /* mark it a control pkt */
192 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
193
194 cpl6 = __skb_put_zero(skb, len);
195 cpl = (struct cpl_act_open_req *)cpl6;
196 INIT_TP_WR(cpl6, 0);
197 qid_atid = TID_QID_V(tx_info->rx_qid) |
198 TID_TID_V(atid);
199 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
200 cpl->local_port = inet->inet_sport;
201 cpl->peer_port = inet->inet_dport;
202 cpl->local_ip = inet->inet_rcv_saddr;
203 cpl->peer_ip = inet->inet_daddr;
204
205 /* fill first 64 bit option field. */
206 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
207 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
208 cpl->opt0 = cpu_to_be64(options);
209
210 /* next 64 bit option field. */
211 options =
212 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
213 cpl->opt2 = htonl(options);
214
215 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
216 }
217
218 #if IS_ENABLED(CONFIG_IPV6)
219 /*
220 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
221 * @sk - tcp socket.
222 * @tx_info - driver specific tls info.
223 * @atid - connection active tid.
224 * return - send success/failure.
225 */
chcr_ktls_act_open_req6(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)226 static int chcr_ktls_act_open_req6(struct sock *sk,
227 struct chcr_ktls_info *tx_info,
228 int atid)
229 {
230 struct inet_sock *inet = inet_sk(sk);
231 struct cpl_t6_act_open_req6 *cpl6;
232 struct cpl_act_open_req6 *cpl;
233 struct sk_buff *skb;
234 unsigned int len;
235 int qid_atid;
236 u64 options;
237
238 len = sizeof(*cpl6);
239 skb = alloc_skb(len, GFP_KERNEL);
240 if (unlikely(!skb))
241 return -ENOMEM;
242 /* mark it a control pkt */
243 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
244
245 cpl6 = __skb_put_zero(skb, len);
246 cpl = (struct cpl_act_open_req6 *)cpl6;
247 INIT_TP_WR(cpl6, 0);
248 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
249 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
250 cpl->local_port = inet->inet_sport;
251 cpl->peer_port = inet->inet_dport;
252 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
253 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
254 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
255 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
256
257 /* first 64 bit option field. */
258 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
259 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
260 cpl->opt0 = cpu_to_be64(options);
261 /* next 64 bit option field. */
262 options =
263 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
264 cpl->opt2 = htonl(options);
265
266 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
267 }
268 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
269
270 /*
271 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
272 * @sk - tcp socket.
273 * @tx_info - driver specific tls info.
274 * return: NET_TX_OK/NET_XMIT_DROP
275 */
chcr_setup_connection(struct sock * sk,struct chcr_ktls_info * tx_info)276 static int chcr_setup_connection(struct sock *sk,
277 struct chcr_ktls_info *tx_info)
278 {
279 struct tid_info *t = &tx_info->adap->tids;
280 int atid, ret = 0;
281
282 atid = cxgb4_alloc_atid(t, tx_info);
283 if (atid == -1)
284 return -EINVAL;
285
286 tx_info->atid = atid;
287
288 if (tx_info->ip_family == AF_INET) {
289 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
290 #if IS_ENABLED(CONFIG_IPV6)
291 } else {
292 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
293 &sk->sk_v6_rcv_saddr,
294 1);
295 if (ret)
296 return ret;
297 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
298 #endif
299 }
300
301 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
302 * success, if any other return type clear atid and return that failure.
303 */
304 if (ret) {
305 if (ret == NET_XMIT_CN) {
306 ret = 0;
307 } else {
308 #if IS_ENABLED(CONFIG_IPV6)
309 /* clear clip entry */
310 if (tx_info->ip_family == AF_INET6)
311 cxgb4_clip_release(tx_info->netdev,
312 (const u32 *)
313 &sk->sk_v6_rcv_saddr,
314 1);
315 #endif
316 cxgb4_free_atid(t, atid);
317 }
318 }
319
320 return ret;
321 }
322
323 /*
324 * chcr_set_tcb_field: update tcb fields.
325 * @tx_info - driver specific tls info.
326 * @word - TCB word.
327 * @mask - TCB word related mask.
328 * @val - TCB word related value.
329 * @no_reply - set 1 if not looking for TP response.
330 */
chcr_set_tcb_field(struct chcr_ktls_info * tx_info,u16 word,u64 mask,u64 val,int no_reply)331 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
332 u64 mask, u64 val, int no_reply)
333 {
334 struct cpl_set_tcb_field *req;
335 struct sk_buff *skb;
336
337 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
338 if (!skb)
339 return -ENOMEM;
340
341 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
342 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
343 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
344 NO_REPLY_V(no_reply));
345 req->word_cookie = htons(TCB_WORD_V(word));
346 req->mask = cpu_to_be64(mask);
347 req->val = cpu_to_be64(val);
348
349 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
350 return cxgb4_ofld_send(tx_info->netdev, skb);
351 }
352
353 /*
354 * chcr_ktls_dev_del: call back for tls_dev_del.
355 * Remove the tid and l2t entry and close the connection.
356 * it per connection basis.
357 * @netdev - net device.
358 * @tls_cts - tls context.
359 * @direction - TX/RX crypto direction
360 */
chcr_ktls_dev_del(struct net_device * netdev,struct tls_context * tls_ctx,enum tls_offload_ctx_dir direction)361 static void chcr_ktls_dev_del(struct net_device *netdev,
362 struct tls_context *tls_ctx,
363 enum tls_offload_ctx_dir direction)
364 {
365 struct chcr_ktls_ofld_ctx_tx *tx_ctx =
366 chcr_get_ktls_tx_context(tls_ctx);
367 struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
368 struct ch_ktls_port_stats_debug *port_stats;
369 struct chcr_ktls_uld_ctx *u_ctx;
370
371 if (!tx_info)
372 return;
373
374 u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle;
375 if (u_ctx && u_ctx->detach)
376 return;
377 /* clear l2t entry */
378 if (tx_info->l2te)
379 cxgb4_l2t_release(tx_info->l2te);
380
381 #if IS_ENABLED(CONFIG_IPV6)
382 /* clear clip entry */
383 if (tx_info->ip_family == AF_INET6)
384 cxgb4_clip_release(netdev, (const u32 *)
385 &tx_info->sk->sk_v6_rcv_saddr,
386 1);
387 #endif
388
389 /* clear tid */
390 if (tx_info->tid != -1) {
391 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
392 tx_info->tid, tx_info->ip_family);
393
394 xa_erase(&u_ctx->tid_list, tx_info->tid);
395 }
396
397 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
398 atomic64_inc(&port_stats->ktls_tx_connection_close);
399 kvfree(tx_info);
400 tx_ctx->chcr_info = NULL;
401 /* release module refcount */
402 module_put(THIS_MODULE);
403 }
404
405 /*
406 * chcr_ktls_dev_add: call back for tls_dev_add.
407 * Create a tcb entry for TP. Also add l2t entry for the connection. And
408 * generate keys & save those keys locally.
409 * @netdev - net device.
410 * @tls_cts - tls context.
411 * @direction - TX/RX crypto direction
412 * return: SUCCESS/FAILURE.
413 */
chcr_ktls_dev_add(struct net_device * netdev,struct sock * sk,enum tls_offload_ctx_dir direction,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)414 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
415 enum tls_offload_ctx_dir direction,
416 struct tls_crypto_info *crypto_info,
417 u32 start_offload_tcp_sn)
418 {
419 struct tls_context *tls_ctx = tls_get_ctx(sk);
420 struct ch_ktls_port_stats_debug *port_stats;
421 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
422 struct chcr_ktls_uld_ctx *u_ctx;
423 struct chcr_ktls_info *tx_info;
424 struct dst_entry *dst;
425 struct adapter *adap;
426 struct port_info *pi;
427 struct neighbour *n;
428 u8 daaddr[16];
429 int ret = -1;
430
431 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
432
433 pi = netdev_priv(netdev);
434 adap = pi->adapter;
435 port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
436 atomic64_inc(&port_stats->ktls_tx_connection_open);
437 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
438
439 if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
440 pr_err("not expecting for RX direction\n");
441 goto out;
442 }
443
444 if (tx_ctx->chcr_info)
445 goto out;
446
447 if (u_ctx && u_ctx->detach)
448 goto out;
449
450 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
451 if (!tx_info)
452 goto out;
453
454 tx_info->sk = sk;
455 spin_lock_init(&tx_info->lock);
456 /* initialize tid and atid to -1, 0 is a also a valid id. */
457 tx_info->tid = -1;
458 tx_info->atid = -1;
459
460 tx_info->adap = adap;
461 tx_info->netdev = netdev;
462 tx_info->first_qset = pi->first_qset;
463 tx_info->tx_chan = pi->tx_chan;
464 tx_info->smt_idx = pi->smt_idx;
465 tx_info->port_id = pi->port_id;
466 tx_info->prev_ack = 0;
467 tx_info->prev_win = 0;
468
469 tx_info->rx_qid = chcr_get_first_rx_qid(adap);
470 if (unlikely(tx_info->rx_qid < 0))
471 goto free_tx_info;
472
473 tx_info->prev_seq = start_offload_tcp_sn;
474 tx_info->tcp_start_seq_number = start_offload_tcp_sn;
475
476 /* save crypto keys */
477 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
478 if (ret < 0)
479 goto free_tx_info;
480
481 /* get peer ip */
482 if (sk->sk_family == AF_INET) {
483 memcpy(daaddr, &sk->sk_daddr, 4);
484 tx_info->ip_family = AF_INET;
485 #if IS_ENABLED(CONFIG_IPV6)
486 } else {
487 if (!sk->sk_ipv6only &&
488 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
489 memcpy(daaddr, &sk->sk_daddr, 4);
490 tx_info->ip_family = AF_INET;
491 } else {
492 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
493 tx_info->ip_family = AF_INET6;
494 }
495 #endif
496 }
497
498 /* get the l2t index */
499 dst = sk_dst_get(sk);
500 if (!dst) {
501 pr_err("DST entry not found\n");
502 goto free_tx_info;
503 }
504 n = dst_neigh_lookup(dst, daaddr);
505 if (!n || !n->dev) {
506 pr_err("neighbour not found\n");
507 dst_release(dst);
508 goto free_tx_info;
509 }
510 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
511
512 neigh_release(n);
513 dst_release(dst);
514
515 if (!tx_info->l2te) {
516 pr_err("l2t entry not found\n");
517 goto free_tx_info;
518 }
519
520 /* Driver shouldn't be removed until any single connection exists */
521 if (!try_module_get(THIS_MODULE))
522 goto free_l2t;
523
524 init_completion(&tx_info->completion);
525 /* create a filter and call cxgb4_l2t_send to send the packet out, which
526 * will take care of updating l2t entry in hw if not already done.
527 */
528 tx_info->open_state = CH_KTLS_OPEN_PENDING;
529
530 if (chcr_setup_connection(sk, tx_info))
531 goto put_module;
532
533 /* Wait for reply */
534 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
535 spin_lock_bh(&tx_info->lock);
536 if (tx_info->open_state) {
537 /* need to wait for hw response, can't free tx_info yet. */
538 if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
539 tx_info->pending_close = true;
540 else
541 spin_unlock_bh(&tx_info->lock);
542 /* if in pending close, free the lock after the cleanup */
543 goto put_module;
544 }
545 spin_unlock_bh(&tx_info->lock);
546
547 /* initialize tcb */
548 reinit_completion(&tx_info->completion);
549 /* mark it pending for hw response */
550 tx_info->open_state = CH_KTLS_OPEN_PENDING;
551
552 if (chcr_init_tcb_fields(tx_info))
553 goto free_tid;
554
555 /* Wait for reply */
556 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
557 spin_lock_bh(&tx_info->lock);
558 if (tx_info->open_state) {
559 /* need to wait for hw response, can't free tx_info yet. */
560 tx_info->pending_close = true;
561 /* free the lock after cleanup */
562 goto free_tid;
563 }
564 spin_unlock_bh(&tx_info->lock);
565
566 if (!cxgb4_check_l2t_valid(tx_info->l2te))
567 goto free_tid;
568
569 atomic64_inc(&port_stats->ktls_tx_ctx);
570 tx_ctx->chcr_info = tx_info;
571
572 return 0;
573
574 free_tid:
575 #if IS_ENABLED(CONFIG_IPV6)
576 /* clear clip entry */
577 if (tx_info->ip_family == AF_INET6)
578 cxgb4_clip_release(netdev, (const u32 *)
579 &sk->sk_v6_rcv_saddr,
580 1);
581 #endif
582 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
583 tx_info->tid, tx_info->ip_family);
584
585 xa_erase(&u_ctx->tid_list, tx_info->tid);
586
587 put_module:
588 /* release module refcount */
589 module_put(THIS_MODULE);
590 free_l2t:
591 cxgb4_l2t_release(tx_info->l2te);
592 free_tx_info:
593 if (tx_info->pending_close)
594 spin_unlock_bh(&tx_info->lock);
595 else
596 kvfree(tx_info);
597 out:
598 atomic64_inc(&port_stats->ktls_tx_connection_fail);
599 return -1;
600 }
601
602 /*
603 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
604 * handling.
605 * @tx_info - driver specific tls info.
606 * return: NET_TX_OK/NET_XMIT_DROP
607 */
chcr_init_tcb_fields(struct chcr_ktls_info * tx_info)608 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
609 {
610 int ret = 0;
611
612 /* set tcb in offload and bypass */
613 ret =
614 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
615 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
616 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
617 if (ret)
618 return ret;
619 /* reset snd_una and snd_next fields in tcb */
620 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
621 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
622 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
623 0, 1);
624 if (ret)
625 return ret;
626
627 /* reset send max */
628 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
629 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
630 0, 1);
631 if (ret)
632 return ret;
633
634 /* update l2t index and request for tp reply to confirm tcb is
635 * initialised to handle tx traffic.
636 */
637 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
638 TCB_L2T_IX_V(TCB_L2T_IX_M),
639 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
640 return ret;
641 }
642
643 /*
644 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
645 */
chcr_ktls_cpl_act_open_rpl(struct adapter * adap,unsigned char * input)646 static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
647 unsigned char *input)
648 {
649 const struct cpl_act_open_rpl *p = (void *)input;
650 struct chcr_ktls_info *tx_info = NULL;
651 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
652 struct chcr_ktls_uld_ctx *u_ctx;
653 unsigned int atid, tid, status;
654 struct tls_context *tls_ctx;
655 struct tid_info *t;
656 int ret = 0;
657
658 tid = GET_TID(p);
659 status = AOPEN_STATUS_G(ntohl(p->atid_status));
660 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
661
662 t = &adap->tids;
663 tx_info = lookup_atid(t, atid);
664
665 if (!tx_info || tx_info->atid != atid) {
666 pr_err("%s: incorrect tx_info or atid\n", __func__);
667 return -1;
668 }
669
670 cxgb4_free_atid(t, atid);
671 tx_info->atid = -1;
672
673 spin_lock(&tx_info->lock);
674 /* HW response is very close, finish pending cleanup */
675 if (tx_info->pending_close) {
676 spin_unlock(&tx_info->lock);
677 if (!status) {
678 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
679 tid, tx_info->ip_family);
680 }
681 kvfree(tx_info);
682 return 0;
683 }
684
685 if (!status) {
686 tx_info->tid = tid;
687 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
688 /* Adding tid */
689 tls_ctx = tls_get_ctx(tx_info->sk);
690 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
691 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
692 if (u_ctx) {
693 ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx,
694 GFP_NOWAIT);
695 if (ret < 0) {
696 pr_err("%s: Failed to allocate tid XA entry = %d\n",
697 __func__, tx_info->tid);
698 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
699 goto out;
700 }
701 }
702 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
703 } else {
704 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
705 }
706 out:
707 spin_unlock(&tx_info->lock);
708
709 complete(&tx_info->completion);
710 return ret;
711 }
712
713 /*
714 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
715 */
chcr_ktls_cpl_set_tcb_rpl(struct adapter * adap,unsigned char * input)716 static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
717 {
718 const struct cpl_set_tcb_rpl *p = (void *)input;
719 struct chcr_ktls_info *tx_info = NULL;
720 struct tid_info *t;
721 u32 tid;
722
723 tid = GET_TID(p);
724
725 t = &adap->tids;
726 tx_info = lookup_tid(t, tid);
727
728 if (!tx_info || tx_info->tid != tid) {
729 pr_err("%s: incorrect tx_info or tid\n", __func__);
730 return -1;
731 }
732
733 spin_lock(&tx_info->lock);
734 if (tx_info->pending_close) {
735 spin_unlock(&tx_info->lock);
736 kvfree(tx_info);
737 return 0;
738 }
739 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
740 spin_unlock(&tx_info->lock);
741
742 complete(&tx_info->completion);
743 return 0;
744 }
745
__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,u32 tid,void * pos,u16 word,struct sge_eth_txq * q,u64 mask,u64 val,u32 reply)746 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
747 u32 tid, void *pos, u16 word,
748 struct sge_eth_txq *q, u64 mask,
749 u64 val, u32 reply)
750 {
751 struct cpl_set_tcb_field_core *cpl;
752 struct ulptx_idata *idata;
753 struct ulp_txpkt *txpkt;
754
755 /* ULP_TXPKT */
756 txpkt = pos;
757 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
758 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
759 ULP_TXPKT_FID_V(q->q.cntxt_id) |
760 ULP_TXPKT_RO_F);
761 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
762
763 /* ULPTX_IDATA sub-command */
764 idata = (struct ulptx_idata *)(txpkt + 1);
765 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
766 idata->len = htonl(sizeof(*cpl));
767 pos = idata + 1;
768
769 cpl = pos;
770 /* CPL_SET_TCB_FIELD */
771 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
772 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
773 NO_REPLY_V(!reply));
774 cpl->word_cookie = htons(TCB_WORD_V(word));
775 cpl->mask = cpu_to_be64(mask);
776 cpl->val = cpu_to_be64(val);
777
778 /* ULPTX_NOOP */
779 idata = (struct ulptx_idata *)(cpl + 1);
780 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
781 idata->len = htonl(0);
782 pos = idata + 1;
783
784 return pos;
785 }
786
787
788 /*
789 * chcr_write_cpl_set_tcb_ulp: update tcb values.
790 * TCB is responsible to create tcp headers, so all the related values
791 * should be correctly updated.
792 * @tx_info - driver specific tls info.
793 * @q - tx queue on which packet is going out.
794 * @tid - TCB identifier.
795 * @pos - current index where should we start writing.
796 * @word - TCB word.
797 * @mask - TCB word related mask.
798 * @val - TCB word related value.
799 * @reply - set 1 if looking for TP response.
800 * return - next position to write.
801 */
chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tid,void * pos,u16 word,u64 mask,u64 val,u32 reply)802 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
803 struct sge_eth_txq *q, u32 tid,
804 void *pos, u16 word, u64 mask,
805 u64 val, u32 reply)
806 {
807 int left = (void *)q->q.stat - pos;
808
809 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
810 if (!left) {
811 pos = q->q.desc;
812 } else {
813 u8 buf[48] = {0};
814
815 __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
816 mask, val, reply);
817
818 return chcr_copy_to_txd(buf, &q->q, pos,
819 CHCR_SET_TCB_FIELD_LEN);
820 }
821 }
822
823 pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
824 mask, val, reply);
825
826 /* check again if we are at the end of the queue */
827 if (left == CHCR_SET_TCB_FIELD_LEN)
828 pos = q->q.desc;
829
830 return pos;
831 }
832
833 /*
834 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
835 * with updated values like tcp seq, ack, window etc.
836 * @tx_info - driver specific tls info.
837 * @q - TX queue.
838 * @tcp_seq
839 * @tcp_ack
840 * @tcp_win
841 * return: NETDEV_TX_BUSY/NET_TX_OK.
842 */
chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u64 tcp_seq,u64 tcp_ack,u64 tcp_win,bool offset)843 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
844 struct sge_eth_txq *q, u64 tcp_seq,
845 u64 tcp_ack, u64 tcp_win, bool offset)
846 {
847 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
848 struct ch_ktls_port_stats_debug *port_stats;
849 u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
850 struct fw_ulptx_wr *wr;
851 int credits;
852 void *pos;
853
854 wr_len = sizeof(*wr);
855 /* there can be max 4 cpls, check if we have enough credits */
856 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
857 ndesc = DIV_ROUND_UP(len, 64);
858
859 credits = chcr_txq_avail(&q->q) - ndesc;
860 if (unlikely(credits < 0)) {
861 chcr_eth_txq_stop(q);
862 return NETDEV_TX_BUSY;
863 }
864
865 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
866 chcr_eth_txq_stop(q);
867 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
868 }
869
870 pos = &q->q.desc[q->q.pidx];
871 /* make space for WR, we'll fill it later when we know all the cpls
872 * being sent out and have complete length.
873 */
874 wr = pos;
875 pos += wr_len;
876 /* update tx_max if its a re-transmit or the first wr */
877 if (first_wr || tcp_seq != tx_info->prev_seq) {
878 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
879 TCB_TX_MAX_W,
880 TCB_TX_MAX_V(TCB_TX_MAX_M),
881 TCB_TX_MAX_V(tcp_seq), 0);
882 cpl++;
883 }
884 /* reset snd una if it's a re-transmit pkt */
885 if (tcp_seq != tx_info->prev_seq || offset) {
886 /* reset snd_una */
887 port_stats =
888 &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
889 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
890 TCB_SND_UNA_RAW_W,
891 TCB_SND_UNA_RAW_V
892 (TCB_SND_UNA_RAW_M),
893 TCB_SND_UNA_RAW_V(0), 0);
894 if (tcp_seq != tx_info->prev_seq)
895 atomic64_inc(&port_stats->ktls_tx_ooo);
896 cpl++;
897 }
898 /* update ack */
899 if (first_wr || tx_info->prev_ack != tcp_ack) {
900 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
901 TCB_RCV_NXT_W,
902 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
903 TCB_RCV_NXT_V(tcp_ack), 0);
904 tx_info->prev_ack = tcp_ack;
905 cpl++;
906 }
907 /* update receive window */
908 if (first_wr || tx_info->prev_win != tcp_win) {
909 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
910 TCB_RCV_WND_W,
911 TCB_RCV_WND_V(TCB_RCV_WND_M),
912 TCB_RCV_WND_V(tcp_win), 0);
913 tx_info->prev_win = tcp_win;
914 cpl++;
915 }
916
917 if (cpl) {
918 /* get the actual length */
919 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
920 /* ULPTX wr */
921 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
922 wr->cookie = 0;
923 /* fill len in wr field */
924 wr->flowid_len16 = htonl(wr_mid |
925 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
926
927 ndesc = DIV_ROUND_UP(len, 64);
928 chcr_txq_advance(&q->q, ndesc);
929 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
930 }
931 return 0;
932 }
933
934 /*
935 * chcr_ktls_get_tx_flits
936 * returns number of flits to be sent out, it includes key context length, WR
937 * size and skb fragments.
938 */
939 static unsigned int
chcr_ktls_get_tx_flits(u32 nr_frags,unsigned int key_ctx_len)940 chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
941 {
942 return chcr_sgl_len(nr_frags) +
943 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
944 }
945
946 /*
947 * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
948 * other than timestamp.
949 * @skb - skb contains partial record..
950 * return: 1 / 0
951 */
952 static int
chcr_ktls_check_tcp_options(struct tcphdr * tcp)953 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
954 {
955 int cnt, opt, optlen;
956 u_char *cp;
957
958 cp = (u_char *)(tcp + 1);
959 cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
960 for (; cnt > 0; cnt -= optlen, cp += optlen) {
961 opt = cp[0];
962 if (opt == TCPOPT_EOL)
963 break;
964 if (opt == TCPOPT_NOP) {
965 optlen = 1;
966 } else {
967 if (cnt < 2)
968 break;
969 optlen = cp[1];
970 if (optlen < 2 || optlen > cnt)
971 break;
972 }
973 switch (opt) {
974 case TCPOPT_NOP:
975 break;
976 default:
977 return 1;
978 }
979 }
980 return 0;
981 }
982
983 /*
984 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
985 * send out separately.
986 * @tx_info - driver specific tls info.
987 * @skb - skb contains partial record..
988 * @q - TX queue.
989 * @tx_chan - channel number.
990 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
991 */
992 static int
chcr_ktls_write_tcp_options(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q,uint32_t tx_chan)993 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
994 struct sge_eth_txq *q, uint32_t tx_chan)
995 {
996 struct fw_eth_tx_pkt_wr *wr;
997 struct cpl_tx_pkt_core *cpl;
998 u32 ctrl, iplen, maclen;
999 struct ipv6hdr *ip6;
1000 unsigned int ndesc;
1001 struct tcphdr *tcp;
1002 int len16, pktlen;
1003 struct iphdr *ip;
1004 u32 wr_mid = 0;
1005 int credits;
1006 u8 buf[150];
1007 u64 cntrl1;
1008 void *pos;
1009
1010 iplen = skb_network_header_len(skb);
1011 maclen = skb_mac_header_len(skb);
1012
1013 /* packet length = eth hdr len + ip hdr len + tcp hdr len
1014 * (including options).
1015 */
1016 pktlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
1017
1018 ctrl = sizeof(*cpl) + pktlen;
1019 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
1020 /* check how many descriptors needed */
1021 ndesc = DIV_ROUND_UP(len16, 4);
1022
1023 credits = chcr_txq_avail(&q->q) - ndesc;
1024 if (unlikely(credits < 0)) {
1025 chcr_eth_txq_stop(q);
1026 return NETDEV_TX_BUSY;
1027 }
1028
1029 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1030 chcr_eth_txq_stop(q);
1031 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1032 }
1033
1034 pos = &q->q.desc[q->q.pidx];
1035 wr = pos;
1036
1037 /* Firmware work request header */
1038 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1039 FW_WR_IMMDLEN_V(ctrl));
1040
1041 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1042 wr->r3 = 0;
1043
1044 cpl = (void *)(wr + 1);
1045
1046 /* CPL header */
1047 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1048 TXPKT_PF_V(tx_info->adap->pf));
1049 cpl->pack = 0;
1050 cpl->len = htons(pktlen);
1051
1052 memcpy(buf, skb->data, pktlen);
1053 if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1054 /* we need to correct ip header len */
1055 ip = (struct iphdr *)(buf + maclen);
1056 ip->tot_len = htons(pktlen - maclen);
1057 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1058 } else {
1059 ip6 = (struct ipv6hdr *)(buf + maclen);
1060 ip6->payload_len = htons(pktlen - maclen - iplen);
1061 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1062 }
1063
1064 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1065 TXPKT_IPHDR_LEN_V(iplen);
1066 /* checksum offload */
1067 cpl->ctrl1 = cpu_to_be64(cntrl1);
1068
1069 pos = cpl + 1;
1070
1071 /* now take care of the tcp header, if fin is not set then clear push
1072 * bit as well, and if fin is set, it will be sent at the last so we
1073 * need to update the tcp sequence number as per the last packet.
1074 */
1075 tcp = (struct tcphdr *)(buf + maclen + iplen);
1076
1077 if (!tcp->fin)
1078 tcp->psh = 0;
1079 else
1080 tcp->seq = htonl(tx_info->prev_seq);
1081
1082 chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1083
1084 chcr_txq_advance(&q->q, ndesc);
1085 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1086 return 0;
1087 }
1088
1089 /*
1090 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1091 * received has partial end part of the record, send out the complete record, so
1092 * that crypto block will be able to generate TAG/HASH.
1093 * @skb - segment which has complete or partial end part.
1094 * @tx_info - driver specific tls info.
1095 * @q - TX queue.
1096 * @tcp_seq
1097 * @tcp_push - tcp push bit.
1098 * @mss - segment size.
1099 * return: NETDEV_TX_BUSY/NET_TX_OK.
1100 */
chcr_ktls_xmit_wr_complete(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool is_last_wr,u32 data_len,u32 skb_offset,u32 nfrags,bool tcp_push,u32 mss)1101 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1102 struct chcr_ktls_info *tx_info,
1103 struct sge_eth_txq *q, u32 tcp_seq,
1104 bool is_last_wr, u32 data_len,
1105 u32 skb_offset, u32 nfrags,
1106 bool tcp_push, u32 mss)
1107 {
1108 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1109 struct adapter *adap = tx_info->adap;
1110 int credits, left, last_desc;
1111 struct tx_sw_desc *sgl_sdesc;
1112 struct cpl_tx_data *tx_data;
1113 struct cpl_tx_sec_pdu *cpl;
1114 struct ulptx_idata *idata;
1115 struct ulp_txpkt *ulptx;
1116 struct fw_ulptx_wr *wr;
1117 void *pos;
1118 u64 *end;
1119
1120 /* get the number of flits required */
1121 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1122 /* number of descriptors */
1123 ndesc = chcr_flits_to_desc(flits);
1124 /* check if enough credits available */
1125 credits = chcr_txq_avail(&q->q) - ndesc;
1126 if (unlikely(credits < 0)) {
1127 chcr_eth_txq_stop(q);
1128 return NETDEV_TX_BUSY;
1129 }
1130
1131 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1132 /* Credits are below the threshold vaues, stop the queue after
1133 * injecting the Work Request for this packet.
1134 */
1135 chcr_eth_txq_stop(q);
1136 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1137 }
1138
1139 last_desc = q->q.pidx + ndesc - 1;
1140 if (last_desc >= q->q.size)
1141 last_desc -= q->q.size;
1142 sgl_sdesc = &q->q.sdesc[last_desc];
1143
1144 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1145 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1146 q->mapping_err++;
1147 return NETDEV_TX_BUSY;
1148 }
1149
1150 if (!is_last_wr)
1151 skb_get(skb);
1152
1153 pos = &q->q.desc[q->q.pidx];
1154 end = (u64 *)pos + flits;
1155 /* FW_ULPTX_WR */
1156 wr = pos;
1157 /* WR will need len16 */
1158 len16 = DIV_ROUND_UP(flits, 2);
1159 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1160 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1161 wr->cookie = 0;
1162 pos += sizeof(*wr);
1163 /* ULP_TXPKT */
1164 ulptx = pos;
1165 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1166 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1167 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1168 ULP_TXPKT_RO_F);
1169 ulptx->len = htonl(len16 - 1);
1170 /* ULPTX_IDATA sub-command */
1171 idata = (struct ulptx_idata *)(ulptx + 1);
1172 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1173 /* idata length will include cpl_tx_sec_pdu + key context size +
1174 * cpl_tx_data header.
1175 */
1176 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1177 sizeof(*tx_data));
1178 /* SEC CPL */
1179 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1180 cpl->op_ivinsrtofst =
1181 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1182 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1183 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1184 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1185 cpl->pldlen = htonl(data_len);
1186
1187 /* encryption should start after tls header size + iv size */
1188 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1189
1190 cpl->aadstart_cipherstop_hi =
1191 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1192 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1193 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1194
1195 /* authentication will also start after tls header + iv size */
1196 cpl->cipherstop_lo_authinsert =
1197 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1198 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1199 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1200
1201 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1202 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1203 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1204 cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1205
1206 pos = cpl + 1;
1207 /* check if space left to fill the keys */
1208 left = (void *)q->q.stat - pos;
1209 if (!left) {
1210 left = (void *)end - (void *)q->q.stat;
1211 pos = q->q.desc;
1212 end = pos + left;
1213 }
1214
1215 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1216 tx_info->key_ctx_len);
1217 left = (void *)q->q.stat - pos;
1218
1219 if (!left) {
1220 left = (void *)end - (void *)q->q.stat;
1221 pos = q->q.desc;
1222 end = pos + left;
1223 }
1224 /* CPL_TX_DATA */
1225 tx_data = (void *)pos;
1226 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1227 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1228
1229 tx_data->rsvd = htonl(tcp_seq);
1230
1231 tx_data->flags = htonl(TX_BYPASS_F);
1232 if (tcp_push)
1233 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1234
1235 /* check left again, it might go beyond queue limit */
1236 pos = tx_data + 1;
1237 left = (void *)q->q.stat - pos;
1238
1239 /* check the position again */
1240 if (!left) {
1241 left = (void *)end - (void *)q->q.stat;
1242 pos = q->q.desc;
1243 end = pos + left;
1244 }
1245
1246 /* send the complete packet except the header */
1247 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1248 skb_offset, data_len);
1249 sgl_sdesc->skb = skb;
1250
1251 chcr_txq_advance(&q->q, ndesc);
1252 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1253 atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1254
1255 return 0;
1256 }
1257
1258 /*
1259 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1260 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1261 * and then only send it out.
1262 *
1263 * @skb - skb contains partial record..
1264 * @tx_info - driver specific tls info.
1265 * @q - TX queue.
1266 * @tcp_seq
1267 * @tcp_push - tcp push bit.
1268 * @mss - segment size.
1269 * @tls_rec_offset - offset from start of the tls record.
1270 * @perior_data - data before the current segment, required to make this record
1271 * 16 byte aligned.
1272 * @prior_data_len - prior_data length (less than 16)
1273 * return: NETDEV_TX_BUSY/NET_TX_OK.
1274 */
chcr_ktls_xmit_wr_short(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool tcp_push,u32 mss,u32 tls_rec_offset,u8 * prior_data,u32 prior_data_len,u32 data_len,u32 skb_offset)1275 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1276 struct chcr_ktls_info *tx_info,
1277 struct sge_eth_txq *q,
1278 u32 tcp_seq, bool tcp_push, u32 mss,
1279 u32 tls_rec_offset, u8 *prior_data,
1280 u32 prior_data_len, u32 data_len,
1281 u32 skb_offset)
1282 {
1283 u32 len16, wr_mid = 0, cipher_start, nfrags;
1284 struct adapter *adap = tx_info->adap;
1285 unsigned int flits = 0, ndesc;
1286 int credits, left, last_desc;
1287 struct tx_sw_desc *sgl_sdesc;
1288 struct cpl_tx_data *tx_data;
1289 struct cpl_tx_sec_pdu *cpl;
1290 struct ulptx_idata *idata;
1291 struct ulp_txpkt *ulptx;
1292 struct fw_ulptx_wr *wr;
1293 __be64 iv_record;
1294 void *pos;
1295 u64 *end;
1296
1297 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1298 /* get the number of flits required, it's a partial record so 2 flits
1299 * (AES_BLOCK_SIZE) will be added.
1300 */
1301 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1302 /* get the correct 8 byte IV of this record */
1303 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1304 /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1305 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1306 * data will be added.
1307 */
1308 if (prior_data_len)
1309 flits += 2;
1310 /* number of descriptors */
1311 ndesc = chcr_flits_to_desc(flits);
1312 /* check if enough credits available */
1313 credits = chcr_txq_avail(&q->q) - ndesc;
1314 if (unlikely(credits < 0)) {
1315 chcr_eth_txq_stop(q);
1316 return NETDEV_TX_BUSY;
1317 }
1318
1319 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1320 chcr_eth_txq_stop(q);
1321 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1322 }
1323
1324 last_desc = q->q.pidx + ndesc - 1;
1325 if (last_desc >= q->q.size)
1326 last_desc -= q->q.size;
1327 sgl_sdesc = &q->q.sdesc[last_desc];
1328
1329 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1330 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1331 q->mapping_err++;
1332 return NETDEV_TX_BUSY;
1333 }
1334
1335 pos = &q->q.desc[q->q.pidx];
1336 end = (u64 *)pos + flits;
1337 /* FW_ULPTX_WR */
1338 wr = pos;
1339 /* WR will need len16 */
1340 len16 = DIV_ROUND_UP(flits, 2);
1341 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1342 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1343 wr->cookie = 0;
1344 pos += sizeof(*wr);
1345 /* ULP_TXPKT */
1346 ulptx = pos;
1347 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1348 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1349 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1350 ULP_TXPKT_RO_F);
1351 ulptx->len = htonl(len16 - 1);
1352 /* ULPTX_IDATA sub-command */
1353 idata = (struct ulptx_idata *)(ulptx + 1);
1354 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1355 /* idata length will include cpl_tx_sec_pdu + key context size +
1356 * cpl_tx_data header.
1357 */
1358 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1359 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1360 /* SEC CPL */
1361 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1362 /* cipher start will have tls header + iv size extra if its a header
1363 * part of tls record. else only 16 byte IV will be added.
1364 */
1365 cipher_start =
1366 AES_BLOCK_LEN + 1 +
1367 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1368
1369 cpl->op_ivinsrtofst =
1370 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1371 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1372 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1373 cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1374 cpl->aadstart_cipherstop_hi =
1375 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1376 cpl->cipherstop_lo_authinsert = 0;
1377 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1378 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1379 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1380 cpl->scmd1 = 0;
1381
1382 pos = cpl + 1;
1383 /* check if space left to fill the keys */
1384 left = (void *)q->q.stat - pos;
1385 if (!left) {
1386 left = (void *)end - (void *)q->q.stat;
1387 pos = q->q.desc;
1388 end = pos + left;
1389 }
1390
1391 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1392 tx_info->key_ctx_len);
1393 left = (void *)q->q.stat - pos;
1394
1395 if (!left) {
1396 left = (void *)end - (void *)q->q.stat;
1397 pos = q->q.desc;
1398 end = pos + left;
1399 }
1400 /* CPL_TX_DATA */
1401 tx_data = (void *)pos;
1402 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1403 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1404 TX_LENGTH_V(data_len + prior_data_len));
1405 tx_data->rsvd = htonl(tcp_seq);
1406 tx_data->flags = htonl(TX_BYPASS_F);
1407 if (tcp_push)
1408 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1409
1410 /* check left again, it might go beyond queue limit */
1411 pos = tx_data + 1;
1412 left = (void *)q->q.stat - pos;
1413
1414 /* check the position again */
1415 if (!left) {
1416 left = (void *)end - (void *)q->q.stat;
1417 pos = q->q.desc;
1418 end = pos + left;
1419 }
1420 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1421 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1422 */
1423 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1424 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1425 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1426 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1427 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1428
1429 pos += 16;
1430 /* Prior_data_len will always be less than 16 bytes, fill the
1431 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1432 * to 0.
1433 */
1434 if (prior_data_len)
1435 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1436 /* send the complete packet except the header */
1437 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1438 skb_offset, data_len);
1439 sgl_sdesc->skb = skb;
1440
1441 chcr_txq_advance(&q->q, ndesc);
1442 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1443
1444 return 0;
1445 }
1446
1447 /*
1448 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1449 * only plain text (only tls header and iv)
1450 * @tx_info - driver specific tls info.
1451 * @skb - skb contains partial record..
1452 * @tcp_seq
1453 * @mss - segment size.
1454 * @tcp_push - tcp push bit.
1455 * @q - TX queue.
1456 * @port_id : port number
1457 * @perior_data - data before the current segment, required to make this record
1458 * 16 byte aligned.
1459 * @prior_data_len - prior_data length (less than 16)
1460 * return: NETDEV_TX_BUSY/NET_TX_OK.
1461 */
chcr_ktls_tx_plaintxt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,u32 tcp_seq,u32 mss,bool tcp_push,struct sge_eth_txq * q,u32 port_id,u8 * prior_data,u32 data_len,u32 skb_offset,u32 prior_data_len)1462 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1463 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1464 bool tcp_push, struct sge_eth_txq *q,
1465 u32 port_id, u8 *prior_data,
1466 u32 data_len, u32 skb_offset,
1467 u32 prior_data_len)
1468 {
1469 int credits, left, len16, last_desc;
1470 unsigned int flits = 0, ndesc;
1471 struct tx_sw_desc *sgl_sdesc;
1472 struct cpl_tx_data *tx_data;
1473 struct ulptx_idata *idata;
1474 struct ulp_txpkt *ulptx;
1475 struct fw_ulptx_wr *wr;
1476 u32 wr_mid = 0, nfrags;
1477 void *pos;
1478 u64 *end;
1479
1480 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1481 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1482 flits += chcr_sgl_len(nfrags);
1483 if (prior_data_len)
1484 flits += 2;
1485
1486 /* WR will need len16 */
1487 len16 = DIV_ROUND_UP(flits, 2);
1488 /* check how many descriptors needed */
1489 ndesc = DIV_ROUND_UP(flits, 8);
1490
1491 credits = chcr_txq_avail(&q->q) - ndesc;
1492 if (unlikely(credits < 0)) {
1493 chcr_eth_txq_stop(q);
1494 return NETDEV_TX_BUSY;
1495 }
1496
1497 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1498 chcr_eth_txq_stop(q);
1499 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1500 }
1501
1502 last_desc = q->q.pidx + ndesc - 1;
1503 if (last_desc >= q->q.size)
1504 last_desc -= q->q.size;
1505 sgl_sdesc = &q->q.sdesc[last_desc];
1506
1507 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1508 sgl_sdesc->addr) < 0)) {
1509 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1510 q->mapping_err++;
1511 return NETDEV_TX_BUSY;
1512 }
1513
1514 pos = &q->q.desc[q->q.pidx];
1515 end = (u64 *)pos + flits;
1516 /* FW_ULPTX_WR */
1517 wr = pos;
1518 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1519 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1520 wr->cookie = 0;
1521 pos += sizeof(*wr);
1522 /* ULP_TXPKT */
1523 ulptx = (struct ulp_txpkt *)(wr + 1);
1524 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1525 ULP_TXPKT_DATAMODIFY_V(0) |
1526 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1527 ULP_TXPKT_DEST_V(0) |
1528 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1529 ulptx->len = htonl(len16 - 1);
1530 /* ULPTX_IDATA sub-command */
1531 idata = (struct ulptx_idata *)(ulptx + 1);
1532 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1533 idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1534 /* CPL_TX_DATA */
1535 tx_data = (struct cpl_tx_data *)(idata + 1);
1536 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1537 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1538 TX_LENGTH_V(data_len + prior_data_len));
1539 /* set tcp seq number */
1540 tx_data->rsvd = htonl(tcp_seq);
1541 tx_data->flags = htonl(TX_BYPASS_F);
1542 if (tcp_push)
1543 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1544
1545 pos = tx_data + 1;
1546 /* apart from prior_data_len, we should set remaining part of 16 bytes
1547 * to be zero.
1548 */
1549 if (prior_data_len)
1550 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1551
1552 /* check left again, it might go beyond queue limit */
1553 left = (void *)q->q.stat - pos;
1554
1555 /* check the position again */
1556 if (!left) {
1557 left = (void *)end - (void *)q->q.stat;
1558 pos = q->q.desc;
1559 end = pos + left;
1560 }
1561 /* send the complete packet including the header */
1562 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1563 skb_offset, data_len);
1564 sgl_sdesc->skb = skb;
1565
1566 chcr_txq_advance(&q->q, ndesc);
1567 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1568 return 0;
1569 }
1570
chcr_ktls_tunnel_pkt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q)1571 static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1572 struct sk_buff *skb,
1573 struct sge_eth_txq *q)
1574 {
1575 u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1576 struct tx_sw_desc *sgl_sdesc;
1577 struct fw_eth_tx_pkt_wr *wr;
1578 struct cpl_tx_pkt_core *cpl;
1579 unsigned int flits, ndesc;
1580 int credits, last_desc;
1581 u64 cntrl1, *end;
1582 void *pos;
1583
1584 ctrl = sizeof(*cpl);
1585 flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1586
1587 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1588 len16 = DIV_ROUND_UP(flits, 2);
1589 /* check how many descriptors needed */
1590 ndesc = DIV_ROUND_UP(flits, 8);
1591
1592 credits = chcr_txq_avail(&q->q) - ndesc;
1593 if (unlikely(credits < 0)) {
1594 chcr_eth_txq_stop(q);
1595 return -ENOMEM;
1596 }
1597
1598 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1599 chcr_eth_txq_stop(q);
1600 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1601 }
1602
1603 last_desc = q->q.pidx + ndesc - 1;
1604 if (last_desc >= q->q.size)
1605 last_desc -= q->q.size;
1606 sgl_sdesc = &q->q.sdesc[last_desc];
1607
1608 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1609 sgl_sdesc->addr) < 0)) {
1610 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1611 q->mapping_err++;
1612 return -ENOMEM;
1613 }
1614
1615 iplen = skb_network_header_len(skb);
1616 maclen = skb_mac_header_len(skb);
1617
1618 pos = &q->q.desc[q->q.pidx];
1619 end = (u64 *)pos + flits;
1620 wr = pos;
1621
1622 /* Firmware work request header */
1623 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1624 FW_WR_IMMDLEN_V(ctrl));
1625
1626 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1627 wr->r3 = 0;
1628
1629 cpl = (void *)(wr + 1);
1630
1631 /* CPL header */
1632 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1633 TXPKT_INTF_V(tx_info->tx_chan) |
1634 TXPKT_PF_V(tx_info->adap->pf));
1635 cpl->pack = 0;
1636 cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1637 TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1638 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1639 TXPKT_IPHDR_LEN_V(iplen);
1640 /* checksum offload */
1641 cpl->ctrl1 = cpu_to_be64(cntrl1);
1642 cpl->len = htons(skb->len);
1643
1644 pos = cpl + 1;
1645
1646 cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1647 sgl_sdesc->skb = skb;
1648 chcr_txq_advance(&q->q, ndesc);
1649 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1650 return 0;
1651 }
1652
1653 /*
1654 * chcr_ktls_copy_record_in_skb
1655 * @nskb - new skb where the frags to be added.
1656 * @skb - old skb, to copy socket and destructor details.
1657 * @record - specific record which has complete 16k record in frags.
1658 */
chcr_ktls_copy_record_in_skb(struct sk_buff * nskb,struct sk_buff * skb,struct tls_record_info * record)1659 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1660 struct sk_buff *skb,
1661 struct tls_record_info *record)
1662 {
1663 int i = 0;
1664
1665 for (i = 0; i < record->num_frags; i++) {
1666 skb_shinfo(nskb)->frags[i] = record->frags[i];
1667 /* increase the frag ref count */
1668 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1669 }
1670
1671 skb_shinfo(nskb)->nr_frags = record->num_frags;
1672 nskb->data_len = record->len;
1673 nskb->len += record->len;
1674 nskb->truesize += record->len;
1675 nskb->sk = skb->sk;
1676 nskb->destructor = skb->destructor;
1677 refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1678 }
1679
1680 /*
1681 * chcr_end_part_handler: This handler will handle the record which
1682 * is complete or if record's end part is received. T6 adapter has a issue that
1683 * it can't send out TAG with partial record so if its an end part then we have
1684 * to send TAG as well and for which we need to fetch the complete record and
1685 * send it to crypto module.
1686 * @tx_info - driver specific tls info.
1687 * @skb - skb contains partial record.
1688 * @record - complete record of 16K size.
1689 * @tcp_seq
1690 * @mss - segment size in which TP needs to chop a packet.
1691 * @tcp_push_no_fin - tcp push if fin is not set.
1692 * @q - TX queue.
1693 * @tls_end_offset - offset from end of the record.
1694 * @last wr : check if this is the last part of the skb going out.
1695 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1696 */
chcr_end_part_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,struct sge_eth_txq * q,u32 skb_offset,u32 tls_end_offset,bool last_wr)1697 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1698 struct sk_buff *skb,
1699 struct tls_record_info *record,
1700 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1701 struct sge_eth_txq *q, u32 skb_offset,
1702 u32 tls_end_offset, bool last_wr)
1703 {
1704 bool free_skb_if_tx_fails = false;
1705 struct sk_buff *nskb = NULL;
1706
1707 /* check if it is a complete record */
1708 if (tls_end_offset == record->len) {
1709 nskb = skb;
1710 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1711 } else {
1712 nskb = alloc_skb(0, GFP_ATOMIC);
1713 if (!nskb) {
1714 dev_kfree_skb_any(skb);
1715 return NETDEV_TX_BUSY;
1716 }
1717
1718 /* copy complete record in skb */
1719 chcr_ktls_copy_record_in_skb(nskb, skb, record);
1720 /* packet is being sent from the beginning, update the tcp_seq
1721 * accordingly.
1722 */
1723 tcp_seq = tls_record_start_seq(record);
1724 /* reset skb offset */
1725 skb_offset = 0;
1726
1727 if (last_wr)
1728 dev_kfree_skb_any(skb);
1729 else
1730 free_skb_if_tx_fails = true;
1731
1732 last_wr = true;
1733
1734 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1735 }
1736
1737 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1738 last_wr, record->len, skb_offset,
1739 record->num_frags,
1740 (last_wr && tcp_push_no_fin),
1741 mss)) {
1742 if (free_skb_if_tx_fails)
1743 dev_kfree_skb_any(skb);
1744 goto out;
1745 }
1746 tx_info->prev_seq = record->end_seq;
1747 return 0;
1748 out:
1749 dev_kfree_skb_any(nskb);
1750 return NETDEV_TX_BUSY;
1751 }
1752
1753 /*
1754 * chcr_short_record_handler: This handler will take care of the records which
1755 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1756 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1757 * This partial record might be the first part of the record, or the middle
1758 * part. In case of middle record we should fetch the prior data to make it 16
1759 * byte aligned. If it has a partial tls header or iv then get to the start of
1760 * tls header. And if it has partial TAG, then remove the complete TAG and send
1761 * only the payload.
1762 * There is one more possibility that it gets a partial header, send that
1763 * portion as a plaintext.
1764 * @tx_info - driver specific tls info.
1765 * @skb - skb contains partial record..
1766 * @record - complete record of 16K size.
1767 * @tcp_seq
1768 * @mss - segment size in which TP needs to chop a packet.
1769 * @tcp_push_no_fin - tcp push if fin is not set.
1770 * @q - TX queue.
1771 * @tls_end_offset - offset from end of the record.
1772 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1773 */
chcr_short_record_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,u32 data_len,u32 skb_offset,struct sge_eth_txq * q,u32 tls_end_offset)1774 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1775 struct sk_buff *skb,
1776 struct tls_record_info *record,
1777 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1778 u32 data_len, u32 skb_offset,
1779 struct sge_eth_txq *q, u32 tls_end_offset)
1780 {
1781 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1782 u8 prior_data[16] = {0};
1783 u32 prior_data_len = 0;
1784
1785 /* check if the skb is ending in middle of tag/HASH, its a big
1786 * trouble, send the packet before the HASH.
1787 */
1788 int remaining_record = tls_end_offset - data_len;
1789
1790 if (remaining_record > 0 &&
1791 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1792 int trimmed_len = 0;
1793
1794 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1795 trimmed_len = data_len -
1796 (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1797 remaining_record);
1798 if (!trimmed_len)
1799 return FALLBACK;
1800
1801 WARN_ON(trimmed_len > data_len);
1802
1803 data_len = trimmed_len;
1804 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1805 }
1806
1807 /* check if it is only the header part. */
1808 if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1809 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1810 tcp_push_no_fin, q,
1811 tx_info->port_id, prior_data,
1812 data_len, skb_offset, prior_data_len))
1813 goto out;
1814
1815 tx_info->prev_seq = tcp_seq + data_len;
1816 return 0;
1817 }
1818
1819 /* check if the middle record's start point is 16 byte aligned. CTR
1820 * needs 16 byte aligned start point to start encryption.
1821 */
1822 if (tls_rec_offset) {
1823 /* there is an offset from start, means its a middle record */
1824 int remaining = 0;
1825
1826 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1827 prior_data_len = tls_rec_offset;
1828 tls_rec_offset = 0;
1829 remaining = 0;
1830 } else {
1831 prior_data_len =
1832 (tls_rec_offset -
1833 (TLS_HEADER_SIZE + tx_info->iv_size))
1834 % AES_BLOCK_LEN;
1835 remaining = tls_rec_offset - prior_data_len;
1836 }
1837
1838 /* if prior_data_len is not zero, means we need to fetch prior
1839 * data to make this record 16 byte aligned, or we need to reach
1840 * to start offset.
1841 */
1842 if (prior_data_len) {
1843 int i = 0;
1844 u8 *data = NULL;
1845 skb_frag_t *f;
1846 u8 *vaddr;
1847 int frag_size = 0, frag_delta = 0;
1848
1849 while (remaining > 0) {
1850 frag_size = skb_frag_size(&record->frags[i]);
1851 if (remaining < frag_size)
1852 break;
1853
1854 remaining -= frag_size;
1855 i++;
1856 }
1857 f = &record->frags[i];
1858 vaddr = kmap_atomic(skb_frag_page(f));
1859
1860 data = vaddr + skb_frag_off(f) + remaining;
1861 frag_delta = skb_frag_size(f) - remaining;
1862
1863 if (frag_delta >= prior_data_len) {
1864 memcpy(prior_data, data, prior_data_len);
1865 kunmap_atomic(vaddr);
1866 } else {
1867 memcpy(prior_data, data, frag_delta);
1868 kunmap_atomic(vaddr);
1869 /* get the next page */
1870 f = &record->frags[i + 1];
1871 vaddr = kmap_atomic(skb_frag_page(f));
1872 data = vaddr + skb_frag_off(f);
1873 memcpy(prior_data + frag_delta,
1874 data, (prior_data_len - frag_delta));
1875 kunmap_atomic(vaddr);
1876 }
1877 /* reset tcp_seq as per the prior_data_required len */
1878 tcp_seq -= prior_data_len;
1879 }
1880 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1881 } else {
1882 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1883 }
1884
1885 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1886 mss, tls_rec_offset, prior_data,
1887 prior_data_len, data_len, skb_offset)) {
1888 goto out;
1889 }
1890
1891 tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1892 return 0;
1893 out:
1894 dev_kfree_skb_any(skb);
1895 return NETDEV_TX_BUSY;
1896 }
1897
chcr_ktls_sw_fallback(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q)1898 static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1899 struct chcr_ktls_info *tx_info,
1900 struct sge_eth_txq *q)
1901 {
1902 u32 data_len, skb_offset;
1903 struct sk_buff *nskb;
1904 struct tcphdr *th;
1905
1906 nskb = tls_encrypt_skb(skb);
1907
1908 if (!nskb)
1909 return 0;
1910
1911 th = tcp_hdr(nskb);
1912 skb_offset = skb_transport_offset(nskb) + tcp_hdrlen(nskb);
1913 data_len = nskb->len - skb_offset;
1914 skb_tx_timestamp(nskb);
1915
1916 if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1917 goto out;
1918
1919 tx_info->prev_seq = ntohl(th->seq) + data_len;
1920 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1921 return 0;
1922 out:
1923 dev_kfree_skb_any(nskb);
1924 return 0;
1925 }
1926 /* nic tls TX handler */
chcr_ktls_xmit(struct sk_buff * skb,struct net_device * dev)1927 static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1928 {
1929 u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1930 struct ch_ktls_port_stats_debug *port_stats;
1931 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1932 struct ch_ktls_stats_debug *stats;
1933 struct tcphdr *th = tcp_hdr(skb);
1934 int data_len, qidx, ret = 0, mss;
1935 struct tls_record_info *record;
1936 struct chcr_ktls_info *tx_info;
1937 struct tls_context *tls_ctx;
1938 struct sge_eth_txq *q;
1939 struct adapter *adap;
1940 unsigned long flags;
1941
1942 tcp_seq = ntohl(th->seq);
1943 skb_offset = skb_transport_offset(skb) + tcp_hdrlen(skb);
1944 skb_data_len = skb->len - skb_offset;
1945 data_len = skb_data_len;
1946
1947 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1948
1949 tls_ctx = tls_get_ctx(skb->sk);
1950 if (unlikely(tls_ctx->netdev != dev))
1951 goto out;
1952
1953 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1954 tx_info = tx_ctx->chcr_info;
1955
1956 if (unlikely(!tx_info))
1957 goto out;
1958
1959 adap = tx_info->adap;
1960 stats = &adap->ch_ktls_stats;
1961 port_stats = &stats->ktls_port[tx_info->port_id];
1962
1963 qidx = skb->queue_mapping;
1964 q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1965 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1966 /* if tcp options are set but finish is not send the options first */
1967 if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1968 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1969 tx_info->tx_chan);
1970 if (ret)
1971 return NETDEV_TX_BUSY;
1972 }
1973
1974 /* TCP segments can be in received either complete or partial.
1975 * chcr_end_part_handler will handle cases if complete record or end
1976 * part of the record is received. Incase of partial end part of record,
1977 * we will send the complete record again.
1978 */
1979
1980 spin_lock_irqsave(&tx_ctx->base.lock, flags);
1981
1982 do {
1983
1984 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1985 /* fetch the tls record */
1986 record = tls_get_record(&tx_ctx->base, tcp_seq,
1987 &tx_info->record_no);
1988 /* By the time packet reached to us, ACK is received, and record
1989 * won't be found in that case, handle it gracefully.
1990 */
1991 if (unlikely(!record)) {
1992 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1993 atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
1994 goto out;
1995 }
1996
1997 tls_end_offset = record->end_seq - tcp_seq;
1998
1999 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
2000 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
2001 /* update tcb for the skb */
2002 if (skb_data_len == data_len) {
2003 u32 tx_max = tcp_seq;
2004
2005 if (!tls_record_is_start_marker(record) &&
2006 tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
2007 tx_max = record->end_seq -
2008 TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2009
2010 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
2011 ntohl(th->ack_seq),
2012 ntohs(th->window),
2013 tls_end_offset !=
2014 record->len);
2015 if (ret) {
2016 spin_unlock_irqrestore(&tx_ctx->base.lock,
2017 flags);
2018 goto out;
2019 }
2020
2021 if (th->fin)
2022 skb_get(skb);
2023 }
2024
2025 if (unlikely(tls_record_is_start_marker(record))) {
2026 atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
2027 /* If tls_end_offset < data_len, means there is some
2028 * data after start marker, which needs encryption, send
2029 * plaintext first and take skb refcount. else send out
2030 * complete pkt as plaintext.
2031 */
2032 if (tls_end_offset < data_len)
2033 skb_get(skb);
2034 else
2035 tls_end_offset = data_len;
2036
2037 ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2038 (!th->fin && th->psh), q,
2039 tx_info->port_id, NULL,
2040 tls_end_offset, skb_offset,
2041 0);
2042
2043 if (ret) {
2044 /* free the refcount taken earlier */
2045 if (tls_end_offset < data_len)
2046 dev_kfree_skb_any(skb);
2047 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2048 goto out;
2049 }
2050
2051 data_len -= tls_end_offset;
2052 tcp_seq = record->end_seq;
2053 skb_offset += tls_end_offset;
2054 continue;
2055 }
2056
2057 /* if a tls record is finishing in this SKB */
2058 if (tls_end_offset <= data_len) {
2059 ret = chcr_end_part_handler(tx_info, skb, record,
2060 tcp_seq, mss,
2061 (!th->fin && th->psh), q,
2062 skb_offset,
2063 tls_end_offset,
2064 skb_offset +
2065 tls_end_offset == skb->len);
2066
2067 data_len -= tls_end_offset;
2068 /* tcp_seq increment is required to handle next record.
2069 */
2070 tcp_seq += tls_end_offset;
2071 skb_offset += tls_end_offset;
2072 } else {
2073 ret = chcr_short_record_handler(tx_info, skb,
2074 record, tcp_seq, mss,
2075 (!th->fin && th->psh),
2076 data_len, skb_offset,
2077 q, tls_end_offset);
2078 data_len = 0;
2079 }
2080
2081 /* if any failure, come out from the loop. */
2082 if (ret) {
2083 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2084 if (th->fin)
2085 dev_kfree_skb_any(skb);
2086
2087 if (ret == FALLBACK)
2088 return chcr_ktls_sw_fallback(skb, tx_info, q);
2089
2090 return NETDEV_TX_OK;
2091 }
2092
2093 /* length should never be less than 0 */
2094 WARN_ON(data_len < 0);
2095
2096 } while (data_len > 0);
2097
2098 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2099 atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2100 atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2101
2102 /* tcp finish is set, send a separate tcp msg including all the options
2103 * as well.
2104 */
2105 if (th->fin) {
2106 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2107 dev_kfree_skb_any(skb);
2108 }
2109
2110 return NETDEV_TX_OK;
2111 out:
2112 dev_kfree_skb_any(skb);
2113 return NETDEV_TX_OK;
2114 }
2115
chcr_ktls_uld_add(const struct cxgb4_lld_info * lldi)2116 static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2117 {
2118 struct chcr_ktls_uld_ctx *u_ctx;
2119
2120 pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2121 CHCR_KTLS_DRV_VERSION);
2122 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2123 if (!u_ctx) {
2124 u_ctx = ERR_PTR(-ENOMEM);
2125 goto out;
2126 }
2127 u_ctx->lldi = *lldi;
2128 u_ctx->detach = false;
2129 xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH);
2130 out:
2131 return u_ctx;
2132 }
2133
2134 static const struct tlsdev_ops chcr_ktls_ops = {
2135 .tls_dev_add = chcr_ktls_dev_add,
2136 .tls_dev_del = chcr_ktls_dev_del,
2137 };
2138
2139 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2140 [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2141 [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2142 };
2143
chcr_ktls_uld_rx_handler(void * handle,const __be64 * rsp,const struct pkt_gl * pgl)2144 static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2145 const struct pkt_gl *pgl)
2146 {
2147 const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2148 struct chcr_ktls_uld_ctx *u_ctx = handle;
2149 u8 opcode = rpl->ot.opcode;
2150 struct adapter *adap;
2151
2152 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2153
2154 if (!work_handlers[opcode]) {
2155 pr_err("Unsupported opcode %d received\n", opcode);
2156 return 0;
2157 }
2158
2159 work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2160 return 0;
2161 }
2162
clear_conn_resources(struct chcr_ktls_info * tx_info)2163 static void clear_conn_resources(struct chcr_ktls_info *tx_info)
2164 {
2165 /* clear l2t entry */
2166 if (tx_info->l2te)
2167 cxgb4_l2t_release(tx_info->l2te);
2168
2169 #if IS_ENABLED(CONFIG_IPV6)
2170 /* clear clip entry */
2171 if (tx_info->ip_family == AF_INET6)
2172 cxgb4_clip_release(tx_info->netdev, (const u32 *)
2173 &tx_info->sk->sk_v6_rcv_saddr,
2174 1);
2175 #endif
2176
2177 /* clear tid */
2178 if (tx_info->tid != -1)
2179 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
2180 tx_info->tid, tx_info->ip_family);
2181 }
2182
ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx * u_ctx)2183 static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx)
2184 {
2185 struct ch_ktls_port_stats_debug *port_stats;
2186 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
2187 struct chcr_ktls_info *tx_info;
2188 unsigned long index;
2189
2190 xa_for_each(&u_ctx->tid_list, index, tx_ctx) {
2191 tx_info = tx_ctx->chcr_info;
2192 clear_conn_resources(tx_info);
2193 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
2194 atomic64_inc(&port_stats->ktls_tx_connection_close);
2195 kvfree(tx_info);
2196 tx_ctx->chcr_info = NULL;
2197 /* release module refcount */
2198 module_put(THIS_MODULE);
2199 }
2200 }
2201
chcr_ktls_uld_state_change(void * handle,enum cxgb4_state new_state)2202 static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2203 {
2204 struct chcr_ktls_uld_ctx *u_ctx = handle;
2205
2206 switch (new_state) {
2207 case CXGB4_STATE_UP:
2208 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2209 mutex_lock(&dev_mutex);
2210 list_add_tail(&u_ctx->entry, &uld_ctx_list);
2211 mutex_unlock(&dev_mutex);
2212 break;
2213 case CXGB4_STATE_START_RECOVERY:
2214 case CXGB4_STATE_DOWN:
2215 case CXGB4_STATE_DETACH:
2216 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2217 mutex_lock(&dev_mutex);
2218 u_ctx->detach = true;
2219 list_del(&u_ctx->entry);
2220 ch_ktls_reset_all_conn(u_ctx);
2221 xa_destroy(&u_ctx->tid_list);
2222 mutex_unlock(&dev_mutex);
2223 break;
2224 default:
2225 break;
2226 }
2227
2228 return 0;
2229 }
2230
2231 static struct cxgb4_uld_info chcr_ktls_uld_info = {
2232 .name = CHCR_KTLS_DRV_MODULE_NAME,
2233 .nrxq = 1,
2234 .rxq_size = 1024,
2235 .add = chcr_ktls_uld_add,
2236 .tx_handler = chcr_ktls_xmit,
2237 .rx_handler = chcr_ktls_uld_rx_handler,
2238 .state_change = chcr_ktls_uld_state_change,
2239 .tlsdev_ops = &chcr_ktls_ops,
2240 };
2241
chcr_ktls_init(void)2242 static int __init chcr_ktls_init(void)
2243 {
2244 cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2245 return 0;
2246 }
2247
chcr_ktls_exit(void)2248 static void __exit chcr_ktls_exit(void)
2249 {
2250 struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2251 struct adapter *adap;
2252
2253 pr_info("driver unloaded\n");
2254
2255 mutex_lock(&dev_mutex);
2256 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2257 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2258 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2259 list_del(&u_ctx->entry);
2260 xa_destroy(&u_ctx->tid_list);
2261 kfree(u_ctx);
2262 }
2263 mutex_unlock(&dev_mutex);
2264 cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2265 }
2266
2267 module_init(chcr_ktls_init);
2268 module_exit(chcr_ktls_exit);
2269
2270 MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2271 MODULE_LICENSE("GPL");
2272 MODULE_AUTHOR("Chelsio Communications");
2273 MODULE_VERSION(CHCR_KTLS_DRV_VERSION);
2274