44 	struct blk_keyslot_manager *ksm;  member
47 static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm)  in blk_ksm_hw_enter()  argument
54 	if (ksm->dev)  in blk_ksm_hw_enter()
55 		pm_runtime_get_sync(ksm->dev);  in blk_ksm_hw_enter()
56 	down_write(&ksm->lock);  in blk_ksm_hw_enter()
59 static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm)  in blk_ksm_hw_exit()  argument
61 	up_write(&ksm->lock);  in blk_ksm_hw_exit()
62 	if (ksm->dev)  in blk_ksm_hw_exit()
63 		pm_runtime_put_sync(ksm->dev);  in blk_ksm_hw_exit()
66 static inline bool blk_ksm_is_passthrough(struct blk_keyslot_manager *ksm)  in blk_ksm_is_passthrough()  argument
68 	return ksm->num_slots == 0;  in blk_ksm_is_passthrough()
81 int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots)  in blk_ksm_init()  argument
87 	memset(ksm, 0, sizeof(*ksm));  in blk_ksm_init()
92 	ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL);  in blk_ksm_init()
93 	if (!ksm->slots)  in blk_ksm_init()
96 	ksm->num_slots = num_slots;  in blk_ksm_init()
98 	init_rwsem(&ksm->lock);  in blk_ksm_init()
100 	init_waitqueue_head(&ksm->idle_slots_wait_queue);  in blk_ksm_init()
101 	INIT_LIST_HEAD(&ksm->idle_slots);  in blk_ksm_init()
104 		ksm->slots[slot].ksm = ksm;  in blk_ksm_init()
105 		list_add_tail(&ksm->slots[slot].idle_slot_node,  in blk_ksm_init()
106 			      &ksm->idle_slots);  in blk_ksm_init()
109 	spin_lock_init(&ksm->idle_slots_lock);  in blk_ksm_init()
119 	ksm->log_slot_ht_size = ilog2(slot_hashtable_size);  in blk_ksm_init()
120 	ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size,  in blk_ksm_init()
121 					     sizeof(ksm->slot_hashtable[0]),  in blk_ksm_init()
123 	if (!ksm->slot_hashtable)  in blk_ksm_init()
126 		INIT_HLIST_HEAD(&ksm->slot_hashtable[i]);  in blk_ksm_init()
131 	blk_ksm_destroy(ksm);  in blk_ksm_init()
136 static void blk_ksm_destroy_callback(void *ksm)  in blk_ksm_destroy_callback()  argument
138 	blk_ksm_destroy(ksm);  in blk_ksm_destroy_callback()
152 int devm_blk_ksm_init(struct device *dev, struct blk_keyslot_manager *ksm,  in devm_blk_ksm_init()  argument
155 	int err = blk_ksm_init(ksm, num_slots);  in devm_blk_ksm_init()
160 	return devm_add_action_or_reset(dev, blk_ksm_destroy_callback, ksm);  in devm_blk_ksm_init()
165 blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm,  in blk_ksm_hash_bucket_for_key()  argument
168 	return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)];  in blk_ksm_hash_bucket_for_key()
173 	struct blk_keyslot_manager *ksm = slot->ksm;  in blk_ksm_remove_slot_from_lru_list()  local
176 	spin_lock_irqsave(&ksm->idle_slots_lock, flags);  in blk_ksm_remove_slot_from_lru_list()
178 	spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);  in blk_ksm_remove_slot_from_lru_list()
182 					struct blk_keyslot_manager *ksm,  in blk_ksm_find_keyslot()  argument
185 	const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key);  in blk_ksm_find_keyslot()
196 					struct blk_keyslot_manager *ksm,  in blk_ksm_find_and_grab_keyslot()  argument
201 	slot = blk_ksm_find_keyslot(ksm, key);  in blk_ksm_find_and_grab_keyslot()
213 	return slot - slot->ksm->slots;  in blk_ksm_get_slot_idx()
233 blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm,  in blk_ksm_get_slot_for_key()  argument
243 	if (blk_ksm_is_passthrough(ksm))  in blk_ksm_get_slot_for_key()
246 	down_read(&ksm->lock);  in blk_ksm_get_slot_for_key()
247 	slot = blk_ksm_find_and_grab_keyslot(ksm, key);  in blk_ksm_get_slot_for_key()
248 	up_read(&ksm->lock);  in blk_ksm_get_slot_for_key()
253 		blk_ksm_hw_enter(ksm);  in blk_ksm_get_slot_for_key()
254 		slot = blk_ksm_find_and_grab_keyslot(ksm, key);  in blk_ksm_get_slot_for_key()
256 			blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
264 		if (!list_empty(&ksm->idle_slots))  in blk_ksm_get_slot_for_key()
267 		blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
268 		wait_event(ksm->idle_slots_wait_queue,  in blk_ksm_get_slot_for_key()
269 			   !list_empty(&ksm->idle_slots));  in blk_ksm_get_slot_for_key()
272 	slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot,  in blk_ksm_get_slot_for_key()
276 	err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx);  in blk_ksm_get_slot_for_key()
278 		wake_up(&ksm->idle_slots_wait_queue);  in blk_ksm_get_slot_for_key()
279 		blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
287 	hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key));  in blk_ksm_get_slot_for_key()
293 	blk_ksm_hw_exit(ksm);  in blk_ksm_get_slot_for_key()
307 	struct blk_keyslot_manager *ksm;  in blk_ksm_put_slot()  local
313 	ksm = slot->ksm;  in blk_ksm_put_slot()
316 					&ksm->idle_slots_lock, flags)) {  in blk_ksm_put_slot()
317 		list_add_tail(&slot->idle_slot_node, &ksm->idle_slots);  in blk_ksm_put_slot()
318 		spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);  in blk_ksm_put_slot()
319 		wake_up(&ksm->idle_slots_wait_queue);  in blk_ksm_put_slot()
333 bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm,  in blk_ksm_crypto_cfg_supported()  argument
336 	if (!ksm)  in blk_ksm_crypto_cfg_supported()
338 	if (!(ksm->crypto_modes_supported[cfg->crypto_mode] &  in blk_ksm_crypto_cfg_supported()
341 	if (ksm->max_dun_bytes_supported < cfg->dun_bytes)  in blk_ksm_crypto_cfg_supported()
344 		if (!(ksm->features & BLK_CRYPTO_FEATURE_WRAPPED_KEYS))  in blk_ksm_crypto_cfg_supported()
347 		if (!(ksm->features & BLK_CRYPTO_FEATURE_STANDARD_KEYS))  in blk_ksm_crypto_cfg_supported()
367 int blk_ksm_evict_key(struct blk_keyslot_manager *ksm,  in blk_ksm_evict_key()  argument
373 	if (blk_ksm_is_passthrough(ksm)) {  in blk_ksm_evict_key()
374 		if (ksm->ksm_ll_ops.keyslot_evict) {  in blk_ksm_evict_key()
375 			blk_ksm_hw_enter(ksm);  in blk_ksm_evict_key()
376 			err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, -1);  in blk_ksm_evict_key()
377 			blk_ksm_hw_exit(ksm);  in blk_ksm_evict_key()
383 	blk_ksm_hw_enter(ksm);  in blk_ksm_evict_key()
384 	slot = blk_ksm_find_keyslot(ksm, key);  in blk_ksm_evict_key()
392 	err = ksm->ksm_ll_ops.keyslot_evict(ksm, key,  in blk_ksm_evict_key()
401 	blk_ksm_hw_exit(ksm);  in blk_ksm_evict_key()
414 void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm)  in blk_ksm_reprogram_all_keys()  argument
418 	if (blk_ksm_is_passthrough(ksm))  in blk_ksm_reprogram_all_keys()
422 	down_write(&ksm->lock);  in blk_ksm_reprogram_all_keys()
423 	for (slot = 0; slot < ksm->num_slots; slot++) {  in blk_ksm_reprogram_all_keys()
424 		const struct blk_crypto_key *key = ksm->slots[slot].key;  in blk_ksm_reprogram_all_keys()
430 		err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot);  in blk_ksm_reprogram_all_keys()
433 	up_write(&ksm->lock);  in blk_ksm_reprogram_all_keys()
437 void blk_ksm_destroy(struct blk_keyslot_manager *ksm)  in blk_ksm_destroy()  argument
439 	if (!ksm)  in blk_ksm_destroy()
441 	kvfree(ksm->slot_hashtable);  in blk_ksm_destroy()
442 	kvfree_sensitive(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots);  in blk_ksm_destroy()
443 	memzero_explicit(ksm, sizeof(*ksm));  in blk_ksm_destroy()
447 bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q)  in blk_ksm_register()  argument
453 	q->ksm = ksm;  in blk_ksm_register()
460 	q->ksm = NULL;  in blk_ksm_unregister()
480 int blk_ksm_derive_raw_secret(struct blk_keyslot_manager *ksm,  in blk_ksm_derive_raw_secret()  argument
487 	if (ksm->ksm_ll_ops.derive_raw_secret) {  in blk_ksm_derive_raw_secret()
488 		blk_ksm_hw_enter(ksm);  in blk_ksm_derive_raw_secret()
489 		err = ksm->ksm_ll_ops.derive_raw_secret(ksm, wrapped_key,  in blk_ksm_derive_raw_secret()
492 		blk_ksm_hw_exit(ksm);  in blk_ksm_derive_raw_secret()
625 void blk_ksm_init_passthrough(struct blk_keyslot_manager *ksm)  in blk_ksm_init_passthrough()  argument
627 	memset(ksm, 0, sizeof(*ksm));  in blk_ksm_init_passthrough()
628 	init_rwsem(&ksm->lock);  in blk_ksm_init_passthrough()