1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
3
4 #include <net/inet6_hashtables.h>
5 #include "en_accel/en_accel.h"
6 #include "en_accel/tls.h"
7 #include "en_accel/ktls_txrx.h"
8 #include "en_accel/ktls_utils.h"
9 #include "en_accel/fs_tcp.h"
10
11 struct accel_rule {
12 struct work_struct work;
13 struct mlx5e_priv *priv;
14 struct mlx5_flow_handle *rule;
15 };
16
17 #define PROGRESS_PARAMS_WRITE_UNIT 64
18 #define PROGRESS_PARAMS_PADDED_SIZE \
19 (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \
20 PROGRESS_PARAMS_WRITE_UNIT))
21
22 struct mlx5e_ktls_rx_resync_buf {
23 union {
24 struct mlx5_wqe_tls_progress_params_seg progress;
25 u8 pad[PROGRESS_PARAMS_PADDED_SIZE];
26 } ____cacheline_aligned_in_smp;
27 dma_addr_t dma_addr;
28 struct mlx5e_ktls_offload_context_rx *priv_rx;
29 };
30
31 enum {
32 MLX5E_PRIV_RX_FLAG_DELETING,
33 MLX5E_NUM_PRIV_RX_FLAGS,
34 };
35
36 struct mlx5e_ktls_rx_resync_ctx {
37 struct tls_offload_resync_async core;
38 struct work_struct work;
39 struct mlx5e_priv *priv;
40 refcount_t refcnt;
41 __be64 sw_rcd_sn_be;
42 u32 seq;
43 };
44
45 struct mlx5e_ktls_offload_context_rx {
46 struct tls12_crypto_info_aes_gcm_128 crypto_info;
47 struct accel_rule rule;
48 struct sock *sk;
49 struct mlx5e_rq_stats *stats;
50 struct completion add_ctx;
51 u32 tirn;
52 u32 key_id;
53 u32 rxq;
54 DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);
55
56 /* resync */
57 struct mlx5e_ktls_rx_resync_ctx resync;
58 };
59
mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx * priv_rx)60 static bool mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx *priv_rx)
61 {
62 if (!refcount_dec_and_test(&priv_rx->resync.refcnt))
63 return false;
64
65 kfree(priv_rx);
66 return true;
67 }
68
mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx * priv_rx)69 static void mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx *priv_rx)
70 {
71 refcount_inc(&priv_rx->resync.refcnt);
72 }
73
mlx5e_ktls_create_tir(struct mlx5_core_dev * mdev,u32 * tirn,u32 rqtn)74 static int mlx5e_ktls_create_tir(struct mlx5_core_dev *mdev, u32 *tirn, u32 rqtn)
75 {
76 int err, inlen;
77 void *tirc;
78 u32 *in;
79
80 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
81 in = kvzalloc(inlen, GFP_KERNEL);
82 if (!in)
83 return -ENOMEM;
84
85 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
86
87 MLX5_SET(tirc, tirc, transport_domain, mdev->mlx5e_res.td.tdn);
88 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
89 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
90 MLX5_SET(tirc, tirc, indirect_table, rqtn);
91 MLX5_SET(tirc, tirc, tls_en, 1);
92 MLX5_SET(tirc, tirc, self_lb_block,
93 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST |
94 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST);
95
96 err = mlx5_core_create_tir(mdev, in, tirn);
97
98 kvfree(in);
99 return err;
100 }
101
accel_rule_handle_work(struct work_struct * work)102 static void accel_rule_handle_work(struct work_struct *work)
103 {
104 struct mlx5e_ktls_offload_context_rx *priv_rx;
105 struct accel_rule *accel_rule;
106 struct mlx5_flow_handle *rule;
107
108 accel_rule = container_of(work, struct accel_rule, work);
109 priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule);
110 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
111 goto out;
112
113 rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk,
114 priv_rx->tirn, MLX5_FS_DEFAULT_FLOW_TAG);
115 if (!IS_ERR_OR_NULL(rule))
116 accel_rule->rule = rule;
117 out:
118 complete(&priv_rx->add_ctx);
119 }
120
accel_rule_init(struct accel_rule * rule,struct mlx5e_priv * priv,struct sock * sk)121 static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv,
122 struct sock *sk)
123 {
124 INIT_WORK(&rule->work, accel_rule_handle_work);
125 rule->priv = priv;
126 }
127
icosq_fill_wi(struct mlx5e_icosq * sq,u16 pi,struct mlx5e_icosq_wqe_info * wi)128 static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi,
129 struct mlx5e_icosq_wqe_info *wi)
130 {
131 sq->db.wqe_info[pi] = *wi;
132 }
133
134 static struct mlx5_wqe_ctrl_seg *
post_static_params(struct mlx5e_icosq * sq,struct mlx5e_ktls_offload_context_rx * priv_rx)135 post_static_params(struct mlx5e_icosq *sq,
136 struct mlx5e_ktls_offload_context_rx *priv_rx)
137 {
138 struct mlx5e_set_tls_static_params_wqe *wqe;
139 struct mlx5e_icosq_wqe_info wi;
140 u16 pi, num_wqebbs, room;
141
142 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
143 room = mlx5e_stop_room_for_wqe(num_wqebbs);
144 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room)))
145 return ERR_PTR(-ENOSPC);
146
147 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
148 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
149 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info,
150 priv_rx->tirn, priv_rx->key_id,
151 priv_rx->resync.seq, false,
152 TLS_OFFLOAD_CTX_DIR_RX);
153 wi = (struct mlx5e_icosq_wqe_info) {
154 .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS,
155 .num_wqebbs = num_wqebbs,
156 .tls_set_params.priv_rx = priv_rx,
157 };
158 icosq_fill_wi(sq, pi, &wi);
159 sq->pc += num_wqebbs;
160
161 return &wqe->ctrl;
162 }
163
164 static struct mlx5_wqe_ctrl_seg *
post_progress_params(struct mlx5e_icosq * sq,struct mlx5e_ktls_offload_context_rx * priv_rx,u32 next_record_tcp_sn)165 post_progress_params(struct mlx5e_icosq *sq,
166 struct mlx5e_ktls_offload_context_rx *priv_rx,
167 u32 next_record_tcp_sn)
168 {
169 struct mlx5e_set_tls_progress_params_wqe *wqe;
170 struct mlx5e_icosq_wqe_info wi;
171 u16 pi, num_wqebbs, room;
172
173 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
174 room = mlx5e_stop_room_for_wqe(num_wqebbs);
175 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room)))
176 return ERR_PTR(-ENOSPC);
177
178 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
179 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
180 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_rx->tirn, false,
181 next_record_tcp_sn,
182 TLS_OFFLOAD_CTX_DIR_RX);
183 wi = (struct mlx5e_icosq_wqe_info) {
184 .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS,
185 .num_wqebbs = num_wqebbs,
186 .tls_set_params.priv_rx = priv_rx,
187 };
188
189 icosq_fill_wi(sq, pi, &wi);
190 sq->pc += num_wqebbs;
191
192 return &wqe->ctrl;
193 }
194
post_rx_param_wqes(struct mlx5e_channel * c,struct mlx5e_ktls_offload_context_rx * priv_rx,u32 next_record_tcp_sn)195 static int post_rx_param_wqes(struct mlx5e_channel *c,
196 struct mlx5e_ktls_offload_context_rx *priv_rx,
197 u32 next_record_tcp_sn)
198 {
199 struct mlx5_wqe_ctrl_seg *cseg;
200 struct mlx5e_icosq *sq;
201 int err;
202
203 err = 0;
204 sq = &c->async_icosq;
205 spin_lock_bh(&c->async_icosq_lock);
206
207 cseg = post_static_params(sq, priv_rx);
208 if (IS_ERR(cseg))
209 goto err_out;
210 cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn);
211 if (IS_ERR(cseg))
212 goto err_out;
213
214 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
215 unlock:
216 spin_unlock_bh(&c->async_icosq_lock);
217
218 return err;
219
220 err_out:
221 priv_rx->stats->tls_resync_req_skip++;
222 err = PTR_ERR(cseg);
223 complete(&priv_rx->add_ctx);
224 goto unlock;
225 }
226
227 static void
mlx5e_set_ktls_rx_priv_ctx(struct tls_context * tls_ctx,struct mlx5e_ktls_offload_context_rx * priv_rx)228 mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx,
229 struct mlx5e_ktls_offload_context_rx *priv_rx)
230 {
231 struct mlx5e_ktls_offload_context_rx **ctx =
232 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
233
234 BUILD_BUG_ON(sizeof(priv_rx) > TLS_DRIVER_STATE_SIZE_RX);
235
236 *ctx = priv_rx;
237 }
238
239 static struct mlx5e_ktls_offload_context_rx *
mlx5e_get_ktls_rx_priv_ctx(struct tls_context * tls_ctx)240 mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx)
241 {
242 struct mlx5e_ktls_offload_context_rx **ctx =
243 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
244
245 return *ctx;
246 }
247
248 /* Re-sync */
249 /* Runs in work context */
250 static int
resync_post_get_progress_params(struct mlx5e_icosq * sq,struct mlx5e_ktls_offload_context_rx * priv_rx)251 resync_post_get_progress_params(struct mlx5e_icosq *sq,
252 struct mlx5e_ktls_offload_context_rx *priv_rx)
253 {
254 struct mlx5e_get_tls_progress_params_wqe *wqe;
255 struct mlx5e_ktls_rx_resync_buf *buf;
256 struct mlx5e_icosq_wqe_info wi;
257 struct mlx5_wqe_ctrl_seg *cseg;
258 struct mlx5_seg_get_psv *psv;
259 struct device *pdev;
260 int err;
261 u16 pi;
262
263 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
264 if (unlikely(!buf)) {
265 err = -ENOMEM;
266 goto err_out;
267 }
268
269 pdev = mlx5_core_dma_dev(sq->channel->priv->mdev);
270 buf->dma_addr = dma_map_single(pdev, &buf->progress,
271 PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
272 if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) {
273 err = -ENOMEM;
274 goto err_free;
275 }
276
277 buf->priv_rx = priv_rx;
278
279 BUILD_BUG_ON(MLX5E_KTLS_GET_PROGRESS_WQEBBS != 1);
280
281 spin_lock_bh(&sq->channel->async_icosq_lock);
282
283 if (unlikely(!mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, 1))) {
284 spin_unlock_bh(&sq->channel->async_icosq_lock);
285 err = -ENOSPC;
286 goto err_dma_unmap;
287 }
288
289 pi = mlx5e_icosq_get_next_pi(sq, 1);
290 wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi);
291
292 #define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS))
293
294 cseg = &wqe->ctrl;
295 cseg->opmod_idx_opcode =
296 cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV |
297 (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24));
298 cseg->qpn_ds =
299 cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT);
300
301 psv = &wqe->psv;
302 psv->num_psv = 1 << 4;
303 psv->l_key = sq->channel->mkey_be;
304 psv->psv_index[0] = cpu_to_be32(priv_rx->tirn);
305 psv->va = cpu_to_be64(buf->dma_addr);
306
307 wi = (struct mlx5e_icosq_wqe_info) {
308 .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS,
309 .num_wqebbs = 1,
310 .tls_get_params.buf = buf,
311 };
312 icosq_fill_wi(sq, pi, &wi);
313 sq->pc++;
314 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
315 spin_unlock_bh(&sq->channel->async_icosq_lock);
316
317 return 0;
318
319 err_dma_unmap:
320 dma_unmap_single(pdev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
321 err_free:
322 kfree(buf);
323 err_out:
324 priv_rx->stats->tls_resync_req_skip++;
325 return err;
326 }
327
328 /* Function is called with elevated refcount.
329 * It decreases it only if no WQE is posted.
330 */
resync_handle_work(struct work_struct * work)331 static void resync_handle_work(struct work_struct *work)
332 {
333 struct mlx5e_ktls_offload_context_rx *priv_rx;
334 struct mlx5e_ktls_rx_resync_ctx *resync;
335 struct mlx5e_channel *c;
336 struct mlx5e_icosq *sq;
337
338 resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work);
339 priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync);
340
341 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
342 mlx5e_ktls_priv_rx_put(priv_rx);
343 return;
344 }
345
346 c = resync->priv->channels.c[priv_rx->rxq];
347 sq = &c->async_icosq;
348
349 if (resync_post_get_progress_params(sq, priv_rx))
350 mlx5e_ktls_priv_rx_put(priv_rx);
351 }
352
resync_init(struct mlx5e_ktls_rx_resync_ctx * resync,struct mlx5e_priv * priv)353 static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync,
354 struct mlx5e_priv *priv)
355 {
356 INIT_WORK(&resync->work, resync_handle_work);
357 resync->priv = priv;
358 refcount_set(&resync->refcnt, 1);
359 }
360
361 /* Function can be called with the refcount being either elevated or not.
362 * It does not affect the refcount.
363 */
resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx * priv_rx,struct mlx5e_channel * c)364 static int resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx,
365 struct mlx5e_channel *c)
366 {
367 struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info;
368 struct mlx5_wqe_ctrl_seg *cseg;
369 struct mlx5e_icosq *sq;
370 int err;
371
372 memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq));
373 err = 0;
374
375 sq = &c->async_icosq;
376 spin_lock_bh(&c->async_icosq_lock);
377
378 cseg = post_static_params(sq, priv_rx);
379 if (IS_ERR(cseg)) {
380 priv_rx->stats->tls_resync_res_skip++;
381 err = PTR_ERR(cseg);
382 goto unlock;
383 }
384 /* Do not increment priv_rx refcnt, CQE handling is empty */
385 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
386 priv_rx->stats->tls_resync_res_ok++;
387 unlock:
388 spin_unlock_bh(&c->async_icosq_lock);
389
390 return err;
391 }
392
393 /* Function can be called with the refcount being either elevated or not.
394 * It decreases the refcount and may free the kTLS priv context.
395 * Refcount is not elevated only if tls_dev_del has been called, but GET_PSV was
396 * already in flight.
397 */
mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info * wi,struct mlx5e_icosq * sq)398 void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi,
399 struct mlx5e_icosq *sq)
400 {
401 struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf;
402 struct mlx5e_ktls_offload_context_rx *priv_rx;
403 struct mlx5e_ktls_rx_resync_ctx *resync;
404 u8 tracker_state, auth_state, *ctx;
405 struct device *dev;
406 u32 hw_seq;
407
408 priv_rx = buf->priv_rx;
409 resync = &priv_rx->resync;
410 dev = mlx5_core_dma_dev(resync->priv->mdev);
411 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
412 goto out;
413
414 dma_sync_single_for_cpu(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE,
415 DMA_FROM_DEVICE);
416
417 ctx = buf->progress.ctx;
418 tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state);
419 auth_state = MLX5_GET(tls_progress_params, ctx, auth_state);
420 if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING ||
421 auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) {
422 priv_rx->stats->tls_resync_req_skip++;
423 goto out;
424 }
425
426 hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn);
427 tls_offload_rx_resync_async_request_end(priv_rx->sk, cpu_to_be32(hw_seq));
428 priv_rx->stats->tls_resync_req_end++;
429 out:
430 mlx5e_ktls_priv_rx_put(priv_rx);
431 dma_unmap_single(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
432 kfree(buf);
433 }
434
435 /* Runs in NAPI.
436 * Function elevates the refcount, unless no work is queued.
437 */
resync_queue_get_psv(struct sock * sk)438 static bool resync_queue_get_psv(struct sock *sk)
439 {
440 struct mlx5e_ktls_offload_context_rx *priv_rx;
441 struct mlx5e_ktls_rx_resync_ctx *resync;
442
443 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
444 if (unlikely(!priv_rx))
445 return false;
446
447 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
448 return false;
449
450 resync = &priv_rx->resync;
451 mlx5e_ktls_priv_rx_get(priv_rx);
452 if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work)))
453 mlx5e_ktls_priv_rx_put(priv_rx);
454
455 return true;
456 }
457
458 /* Runs in NAPI */
resync_update_sn(struct mlx5e_rq * rq,struct sk_buff * skb)459 static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
460 {
461 struct ethhdr *eth = (struct ethhdr *)(skb->data);
462 struct net_device *netdev = rq->netdev;
463 struct sock *sk = NULL;
464 unsigned int datalen;
465 struct iphdr *iph;
466 struct tcphdr *th;
467 __be32 seq;
468 int depth = 0;
469
470 __vlan_get_protocol(skb, eth->h_proto, &depth);
471 iph = (struct iphdr *)(skb->data + depth);
472
473 if (iph->version == 4) {
474 depth += sizeof(struct iphdr);
475 th = (void *)iph + sizeof(struct iphdr);
476
477 sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,
478 iph->saddr, th->source, iph->daddr,
479 th->dest, netdev->ifindex);
480 #if IS_ENABLED(CONFIG_IPV6)
481 } else {
482 struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph;
483
484 depth += sizeof(struct ipv6hdr);
485 th = (void *)ipv6h + sizeof(struct ipv6hdr);
486
487 sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,
488 &ipv6h->saddr, th->source,
489 &ipv6h->daddr, ntohs(th->dest),
490 netdev->ifindex, 0);
491 #endif
492 }
493
494 depth += sizeof(struct tcphdr);
495
496 if (unlikely(!sk))
497 return;
498
499 if (unlikely(sk->sk_state == TCP_TIME_WAIT))
500 goto unref;
501
502 if (unlikely(!resync_queue_get_psv(sk)))
503 goto unref;
504
505 seq = th->seq;
506 datalen = skb->len - depth;
507 tls_offload_rx_resync_async_request_start(sk, seq, datalen);
508 rq->stats->tls_resync_req_start++;
509
510 unref:
511 sock_gen_put(sk);
512 }
513
mlx5e_ktls_rx_resync(struct net_device * netdev,struct sock * sk,u32 seq,u8 * rcd_sn)514 void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk,
515 u32 seq, u8 *rcd_sn)
516 {
517 struct mlx5e_ktls_offload_context_rx *priv_rx;
518 struct mlx5e_ktls_rx_resync_ctx *resync;
519 struct mlx5e_priv *priv;
520 struct mlx5e_channel *c;
521
522 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
523 if (unlikely(!priv_rx))
524 return;
525
526 resync = &priv_rx->resync;
527 resync->sw_rcd_sn_be = *(__be64 *)rcd_sn;
528 resync->seq = seq;
529
530 priv = netdev_priv(netdev);
531 c = priv->channels.c[priv_rx->rxq];
532
533 resync_handle_seq_match(priv_rx, c);
534 }
535
536 /* End of resync section */
537
mlx5e_ktls_handle_rx_skb(struct mlx5e_rq * rq,struct sk_buff * skb,struct mlx5_cqe64 * cqe,u32 * cqe_bcnt)538 void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb,
539 struct mlx5_cqe64 *cqe, u32 *cqe_bcnt)
540 {
541 struct mlx5e_rq_stats *stats = rq->stats;
542
543 switch (get_cqe_tls_offload(cqe)) {
544 case CQE_TLS_OFFLOAD_DECRYPTED:
545 skb->decrypted = 1;
546 stats->tls_decrypted_packets++;
547 stats->tls_decrypted_bytes += *cqe_bcnt;
548 break;
549 case CQE_TLS_OFFLOAD_RESYNC:
550 stats->tls_resync_req_pkt++;
551 resync_update_sn(rq, skb);
552 break;
553 default: /* CQE_TLS_OFFLOAD_ERROR: */
554 stats->tls_err++;
555 break;
556 }
557 }
558
mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info * wi)559 void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi)
560 {
561 struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx;
562 struct accel_rule *rule = &priv_rx->rule;
563
564 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
565 complete(&priv_rx->add_ctx);
566 return;
567 }
568 queue_work(rule->priv->tls->rx_wq, &rule->work);
569 }
570
mlx5e_ktls_sk_get_rxq(struct sock * sk)571 static int mlx5e_ktls_sk_get_rxq(struct sock *sk)
572 {
573 int rxq = sk_rx_queue_get(sk);
574
575 if (unlikely(rxq == -1))
576 rxq = 0;
577
578 return rxq;
579 }
580
mlx5e_ktls_add_rx(struct net_device * netdev,struct sock * sk,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)581 int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
582 struct tls_crypto_info *crypto_info,
583 u32 start_offload_tcp_sn)
584 {
585 struct mlx5e_ktls_offload_context_rx *priv_rx;
586 struct mlx5e_ktls_rx_resync_ctx *resync;
587 struct tls_context *tls_ctx;
588 struct mlx5_core_dev *mdev;
589 struct mlx5e_priv *priv;
590 int rxq, err;
591 u32 rqtn;
592
593 tls_ctx = tls_get_ctx(sk);
594 priv = netdev_priv(netdev);
595 mdev = priv->mdev;
596 priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL);
597 if (unlikely(!priv_rx))
598 return -ENOMEM;
599
600 err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id);
601 if (err)
602 goto err_create_key;
603
604 priv_rx->crypto_info =
605 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
606
607 rxq = mlx5e_ktls_sk_get_rxq(sk);
608 priv_rx->rxq = rxq;
609 priv_rx->sk = sk;
610
611 priv_rx->stats = &priv->channel_stats[rxq].rq;
612 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx);
613
614 rqtn = priv->direct_tir[rxq].rqt.rqtn;
615
616 err = mlx5e_ktls_create_tir(mdev, &priv_rx->tirn, rqtn);
617 if (err)
618 goto err_create_tir;
619
620 init_completion(&priv_rx->add_ctx);
621
622 accel_rule_init(&priv_rx->rule, priv, sk);
623 resync = &priv_rx->resync;
624 resync_init(resync, priv);
625 tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core;
626 tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC);
627
628 err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn);
629 if (err)
630 goto err_post_wqes;
631
632 priv_rx->stats->tls_ctx++;
633
634 return 0;
635
636 err_post_wqes:
637 mlx5_core_destroy_tir(mdev, priv_rx->tirn);
638 err_create_tir:
639 mlx5_ktls_destroy_key(mdev, priv_rx->key_id);
640 err_create_key:
641 kfree(priv_rx);
642 return err;
643 }
644
mlx5e_ktls_del_rx(struct net_device * netdev,struct tls_context * tls_ctx)645 void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
646 {
647 struct mlx5e_ktls_offload_context_rx *priv_rx;
648 struct mlx5e_ktls_rx_resync_ctx *resync;
649 struct mlx5_core_dev *mdev;
650 struct mlx5e_priv *priv;
651
652 priv = netdev_priv(netdev);
653 mdev = priv->mdev;
654
655 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx);
656 set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags);
657 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL);
658 synchronize_net(); /* Sync with NAPI */
659 if (!cancel_work_sync(&priv_rx->rule.work))
660 /* completion is needed, as the priv_rx in the add flow
661 * is maintained on the wqe info (wi), not on the socket.
662 */
663 wait_for_completion(&priv_rx->add_ctx);
664 resync = &priv_rx->resync;
665 if (cancel_work_sync(&resync->work))
666 mlx5e_ktls_priv_rx_put(priv_rx);
667
668 priv_rx->stats->tls_del++;
669 if (priv_rx->rule.rule)
670 mlx5e_accel_fs_del_sk(priv_rx->rule.rule);
671
672 mlx5_core_destroy_tir(mdev, priv_rx->tirn);
673 mlx5_ktls_destroy_key(mdev, priv_rx->key_id);
674 /* priv_rx should normally be freed here, but if there is an outstanding
675 * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is
676 * processed.
677 */
678 mlx5e_ktls_priv_rx_put(priv_rx);
679 }
680