Lines Matching refs:aead
202 struct tipc_aead __rcu *aead[KEY_MAX + 1]; member
236 struct tipc_aead *aead; member
243 struct tipc_aead *aead; member
247 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
248 static inline void tipc_aead_put(struct tipc_aead *aead);
250 static int tipc_aead_users(struct tipc_aead __rcu *aead);
251 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
252 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
253 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
254 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
255 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
262 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
267 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
271 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
279 struct tipc_aead *aead, u8 pos,
292 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
377 static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead) in tipc_aead_get() argument
382 tmp = rcu_dereference(aead); in tipc_aead_get()
390 static inline void tipc_aead_put(struct tipc_aead *aead) in tipc_aead_put() argument
392 if (aead && refcount_dec_and_test(&aead->refcnt)) in tipc_aead_put()
393 call_rcu(&aead->rcu, tipc_aead_free); in tipc_aead_put()
402 struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu); in tipc_aead_free() local
405 if (aead->cloned) { in tipc_aead_free()
406 tipc_aead_put(aead->cloned); in tipc_aead_free()
408 head = *get_cpu_ptr(aead->tfm_entry); in tipc_aead_free()
409 put_cpu_ptr(aead->tfm_entry); in tipc_aead_free()
420 free_percpu(aead->tfm_entry); in tipc_aead_free()
421 kfree_sensitive(aead->key); in tipc_aead_free()
422 kfree(aead); in tipc_aead_free()
425 static int tipc_aead_users(struct tipc_aead __rcu *aead) in tipc_aead_users() argument
431 tmp = rcu_dereference(aead); in tipc_aead_users()
439 static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim) in tipc_aead_users_inc() argument
444 tmp = rcu_dereference(aead); in tipc_aead_users_inc()
450 static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim) in tipc_aead_users_dec() argument
455 tmp = rcu_dereference(aead); in tipc_aead_users_dec()
457 atomic_add_unless(&rcu_dereference(aead)->users, -1, lim); in tipc_aead_users_dec()
461 static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val) in tipc_aead_users_set() argument
467 tmp = rcu_dereference(aead); in tipc_aead_users_set()
481 static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead) in tipc_aead_tfm_next() argument
486 tfm_entry = get_cpu_ptr(aead->tfm_entry); in tipc_aead_tfm_next()
507 static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey, in tipc_aead_init() argument
516 if (unlikely(*aead)) in tipc_aead_init()
602 *aead = tmp; in tipc_aead_init()
621 struct tipc_aead *aead; in tipc_aead_clone() local
633 aead = kzalloc(sizeof(*aead), GFP_ATOMIC); in tipc_aead_clone()
634 if (unlikely(!aead)) in tipc_aead_clone()
637 aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC); in tipc_aead_clone()
638 if (unlikely(!aead->tfm_entry)) { in tipc_aead_clone()
639 kfree_sensitive(aead); in tipc_aead_clone()
644 *per_cpu_ptr(aead->tfm_entry, cpu) = in tipc_aead_clone()
648 memcpy(aead->hint, src->hint, sizeof(src->hint)); in tipc_aead_clone()
649 aead->mode = src->mode; in tipc_aead_clone()
650 aead->salt = src->salt; in tipc_aead_clone()
651 aead->authsize = src->authsize; in tipc_aead_clone()
652 atomic_set(&aead->users, 0); in tipc_aead_clone()
653 atomic64_set(&aead->seqno, 0); in tipc_aead_clone()
654 refcount_set(&aead->refcnt, 1); in tipc_aead_clone()
657 aead->cloned = src; in tipc_aead_clone()
659 *dst = aead; in tipc_aead_clone()
725 static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb, in tipc_aead_encrypt() argument
730 struct crypto_aead *tfm = tipc_aead_tfm_next(aead); in tipc_aead_encrypt()
743 tailen = len - skb->len + aead->authsize; in tipc_aead_encrypt()
796 salt = aead->salt; in tipc_aead_encrypt()
797 if (aead->mode == CLUSTER_KEY) in tipc_aead_encrypt()
814 tx_ctx->aead = aead; in tipc_aead_encrypt()
842 struct tipc_aead *aead = tx_ctx->aead; in tipc_aead_encrypt_done() local
843 struct tipc_crypto *tx = aead->crypto; in tipc_aead_encrypt_done()
866 tipc_aead_put(aead); in tipc_aead_encrypt_done()
881 static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead, in tipc_aead_decrypt() argument
895 if (unlikely(!aead)) in tipc_aead_decrypt()
905 tfm = tipc_aead_tfm_next(aead); in tipc_aead_decrypt()
921 salt = aead->salt; in tipc_aead_decrypt()
922 if (aead->mode == CLUSTER_KEY) in tipc_aead_decrypt()
939 rx_ctx->aead = aead; in tipc_aead_decrypt()
966 struct tipc_aead *aead = rx_ctx->aead; in tipc_aead_decrypt_done() local
967 struct tipc_crypto_stats __percpu *stats = aead->crypto->stats; in tipc_aead_decrypt_done()
968 struct net *net = aead->crypto->net; in tipc_aead_decrypt_done()
982 tipc_crypto_rcv_complete(net, aead, b, &skb, err); in tipc_aead_decrypt_done()
1034 static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead, in tipc_ehdr_build() argument
1053 if (!__rx || aead->mode == CLUSTER_KEY) in tipc_ehdr_build()
1054 seqno = atomic64_inc_return(&aead->seqno); in tipc_ehdr_build()
1073 ehdr->master_key = aead->crypto->key_master; in tipc_ehdr_build()
1126 struct tipc_aead *aead = NULL; in tipc_crypto_key_init() local
1130 rc = tipc_aead_init(&aead, ukey, mode); in tipc_crypto_key_init()
1134 rc = tipc_crypto_key_attach(c, aead, 0, master_key); in tipc_crypto_key_init()
1136 tipc_aead_free(&aead->rcu); in tipc_crypto_key_init()
1152 struct tipc_aead *aead, u8 pos, in tipc_crypto_key_attach() argument
1168 if (tipc_aead_users(c->aead[key.pending]) > 0) in tipc_crypto_key_attach()
1190 aead->crypto = c; in tipc_crypto_key_attach()
1191 aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen; in tipc_crypto_key_attach()
1192 tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock); in tipc_crypto_key_attach()
1225 tipc_aead_users_dec(tx->aead[k], 0); in tipc_crypto_key_flush()
1234 tipc_crypto_key_detach(c->aead[k], &c->lock); in tipc_crypto_key_flush()
1270 if (tipc_aead_users(rx->aead[key.pending]) > 0) in tipc_crypto_key_try_align()
1274 tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock); in tipc_crypto_key_try_align()
1277 rcu_assign_pointer(rx->aead[key.pending], NULL); in tipc_crypto_key_try_align()
1281 tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock)); in tipc_crypto_key_try_align()
1288 rcu_assign_pointer(rx->aead[new_pending], tmp1); in tipc_crypto_key_try_align()
1290 rcu_assign_pointer(rx->aead[new_passive], tmp2); in tipc_crypto_key_try_align()
1320 struct tipc_aead *aead = NULL; in tipc_crypto_key_pick_tx() local
1337 aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock); in tipc_crypto_key_pick_tx()
1345 aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock); in tipc_crypto_key_pick_tx()
1346 if (!aead) in tipc_crypto_key_pick_tx()
1348 if (aead->mode != CLUSTER_KEY || in tipc_crypto_key_pick_tx()
1349 aead == skb_cb->tx_clone_ctx.last) { in tipc_crypto_key_pick_tx()
1350 aead = NULL; in tipc_crypto_key_pick_tx()
1354 skb_cb->tx_clone_ctx.last = aead; in tipc_crypto_key_pick_tx()
1363 if (likely(aead)) in tipc_crypto_key_pick_tx()
1364 WARN_ON(!refcount_inc_not_zero(&aead->refcnt)); in tipc_crypto_key_pick_tx()
1367 return aead; in tipc_crypto_key_pick_tx()
1430 tipc_aead_users_inc(tx->aead[new], INT_MAX); in tipc_crypto_key_synch()
1432 tipc_aead_users_dec(tx->aead[cur], 0); in tipc_crypto_key_synch()
1454 tipc_crypto_key_detach(tx->aead[key.active], &tx->lock); in tipc_crypto_key_revoke()
1535 tipc_aead_put(rcu_dereference(c->aead[k])); in tipc_crypto_stop()
1556 if (key.active && tipc_aead_users(tx->aead[key.active]) > 0) in tipc_crypto_timeout()
1558 if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0) in tipc_crypto_timeout()
1565 tipc_crypto_key_detach(tx->aead[key.active], &tx->lock); in tipc_crypto_timeout()
1575 if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0) in tipc_crypto_timeout()
1589 if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10) in tipc_crypto_timeout()
1593 tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock); in tipc_crypto_timeout()
1602 tipc_aead_users(rx->aead[key.active]) > 0) in tipc_crypto_timeout()
1611 tipc_aead_users_set(rx->aead[key.pending], 0); in tipc_crypto_timeout()
1620 tipc_aead_users(rx->aead[key.passive]) > -10) in tipc_crypto_timeout()
1624 tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock); in tipc_crypto_timeout()
1693 struct tipc_aead *aead = NULL; in tipc_crypto_xmit() local
1755 aead = tipc_aead_get(tx->aead[tx_key]); in tipc_crypto_xmit()
1756 if (unlikely(!aead)) in tipc_crypto_xmit()
1758 rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx); in tipc_crypto_xmit()
1760 rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode); in tipc_crypto_xmit()
1783 tipc_aead_put(aead); in tipc_crypto_xmit()
1814 struct tipc_aead *aead = NULL; in tipc_crypto_rcv() local
1839 aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key); in tipc_crypto_rcv()
1840 if (aead) in tipc_crypto_rcv()
1846 if (!aead) in tipc_crypto_rcv()
1847 aead = tipc_aead_get(rx->aead[tx_key]); in tipc_crypto_rcv()
1848 rc = tipc_aead_decrypt(net, aead, *skb, b); in tipc_crypto_rcv()
1874 rcu_access_pointer(rx->aead[n])); in tipc_crypto_rcv()
1888 tipc_crypto_rcv_complete(net, aead, b, skb, rc); in tipc_crypto_rcv()
1892 static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead, in tipc_crypto_rcv_complete() argument
1897 struct tipc_crypto *rx = aead->crypto; in tipc_crypto_rcv_complete()
1903 if (unlikely(is_tx(aead->crypto))) { in tipc_crypto_rcv_complete()
1906 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead, in tipc_crypto_rcv_complete()
1910 aead->crypto->aead[1], aead->crypto->aead[2], in tipc_crypto_rcv_complete()
1911 aead->crypto->aead[3]); in tipc_crypto_rcv_complete()
1935 if (tipc_aead_clone(&tmp, aead) < 0) in tipc_crypto_rcv_complete()
1942 tipc_aead_put(aead); in tipc_crypto_rcv_complete()
1943 aead = tmp; in tipc_crypto_rcv_complete()
1947 tipc_aead_users_dec(aead, INT_MIN); in tipc_crypto_rcv_complete()
1952 tipc_aead_users_set(aead, 1); in tipc_crypto_rcv_complete()
1967 pskb_trim(*skb, (*skb)->len - aead->authsize); in tipc_crypto_rcv_complete()
1996 tipc_aead_put(aead); in tipc_crypto_rcv_complete()
2077 struct tipc_aead *aead; in tipc_crypto_key_dump() local
2103 aead = rcu_dereference(c->aead[k]); in tipc_crypto_key_dump()
2104 if (aead) in tipc_crypto_key_dump()
2107 aead->hint, in tipc_crypto_key_dump()
2108 (aead->mode == CLUSTER_KEY) ? "c" : "p", in tipc_crypto_key_dump()
2109 atomic_read(&aead->users), in tipc_crypto_key_dump()
2110 refcount_read(&aead->refcnt)); in tipc_crypto_key_dump()
2197 struct tipc_aead *aead; in tipc_crypto_key_distr() local
2206 aead = tipc_aead_get(tx->aead[key]); in tipc_crypto_key_distr()
2207 if (likely(aead)) { in tipc_crypto_key_distr()
2208 rc = tipc_crypto_key_xmit(tx->net, aead->key, in tipc_crypto_key_distr()
2209 aead->gen, aead->mode, in tipc_crypto_key_distr()
2211 tipc_aead_put(aead); in tipc_crypto_key_distr()
2444 struct tipc_aead *aead; in tipc_crypto_work_tx() local
2452 aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]); in tipc_crypto_work_tx()
2453 if (unlikely(!aead)) { in tipc_crypto_work_tx()
2460 skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC); in tipc_crypto_work_tx()