49 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
52 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
56 				  u64 const h[][2], u64 dg[], u8 ctr[],
59 				 u64 const h[][2], u64 dg[], u8 ctr[],
71 static void ghash_do_update(int blocks, u64 dg[], const char *src,  in ghash_do_update()  argument
74 	be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };  in ghash_do_update()
91 	dg[0] = be64_to_cpu(dst.b);  in ghash_do_update()
92 	dg[1] = be64_to_cpu(dst.a);  in ghash_do_update()
96 void ghash_do_simd_update(int blocks, u64 dg[], const char *src,  in ghash_do_simd_update()  argument
98 			  void (*simd_update)(int blocks, u64 dg[],  in ghash_do_simd_update()
105 		simd_update(blocks, dg, src, key->h, head);  in ghash_do_simd_update()
108 		ghash_do_update(blocks, dg, src, key, head);  in ghash_do_simd_update()
274 static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],  in gcm_update_mac()  argument
290 		ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,  in gcm_update_mac()
305 static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])  in gcm_calculate_auth_mac()  argument
326 		gcm_update_mac(dg, p, n, buf, &buf_count, ctx);  in gcm_calculate_auth_mac()
336 		ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,  in gcm_calculate_auth_mac()
349 	u64 dg[2] = {};  in gcm_encrypt()  local
358 		gcm_calculate_auth_mac(req, dg);  in gcm_encrypt()
383 					  dg, iv, ctx->aes_key.key_enc, nrounds,  in gcm_encrypt()
412 			ghash_do_update(blocks, dg, walk.dst.virt.addr,  in gcm_encrypt()
431 		ghash_do_update(1, dg, tag, &ctx->ghash_key,  in gcm_encrypt()
437 		put_unaligned_be64(dg[1], tag);  in gcm_encrypt()
438 		put_unaligned_be64(dg[0], tag + 8);  in gcm_encrypt()
464 	u64 dg[2] = {};  in gcm_decrypt()  local
473 		gcm_calculate_auth_mac(req, dg);  in gcm_decrypt()
505 						dg, iv, ctx->aes_key.key_enc,  in gcm_decrypt()
529 			ghash_do_update(blocks, dg, walk.src.virt.addr,  in gcm_decrypt()
552 		ghash_do_update(1, dg, tag, &ctx->ghash_key,  in gcm_decrypt()
567 		put_unaligned_be64(dg[1], tag);  in gcm_decrypt()
568 		put_unaligned_be64(dg[0], tag + 8);  in gcm_decrypt()