70 bool __generic_fsdax_supported(struct dax_device *dax_dev,  in __generic_fsdax_supported()  argument
89 	if (!dax_dev) {  in __generic_fsdax_supported()
111 	len = dax_direct_access(dax_dev, pgoff, 1, &kaddr, &pfn);  in __generic_fsdax_supported()
112 	len2 = dax_direct_access(dax_dev, pgoff_end, 1, &end_kaddr, &end_pfn);  in __generic_fsdax_supported()
170 	struct dax_device *dax_dev;  in __bdev_dax_supported()  local
183 	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);  in __bdev_dax_supported()
184 	if (!dax_dev) {  in __bdev_dax_supported()
191 	ret = dax_supported(dax_dev, bdev, blocksize, 0,  in __bdev_dax_supported()
195 	put_dax(dax_dev);  in __bdev_dax_supported()
232 	struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));  in write_cache_show()  local
235 	WARN_ON_ONCE(!dax_dev);  in write_cache_show()
236 	if (!dax_dev)  in write_cache_show()
239 	rc = sprintf(buf, "%d\n", !!dax_write_cache_enabled(dax_dev));  in write_cache_show()
240 	put_dax(dax_dev);  in write_cache_show()
249 	struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));  in write_cache_store()  local
251 	WARN_ON_ONCE(!dax_dev);  in write_cache_store()
252 	if (!dax_dev)  in write_cache_store()
258 		dax_write_cache(dax_dev, write_cache);  in write_cache_store()
260 	put_dax(dax_dev);  in write_cache_store()
268 	struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));  in dax_visible()  local
270 	WARN_ON_ONCE(!dax_dev);  in dax_visible()
271 	if (!dax_dev)  in dax_visible()
304 long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,  in dax_direct_access()  argument
309 	if (!dax_dev)  in dax_direct_access()
312 	if (!dax_alive(dax_dev))  in dax_direct_access()
318 	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,  in dax_direct_access()
326 bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev,  in dax_supported()  argument
329 	if (!dax_dev)  in dax_supported()
332 	if (!dax_alive(dax_dev))  in dax_supported()
335 	return dax_dev->ops->dax_supported(dax_dev, bdev, blocksize, start, len);  in dax_supported()
339 size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,  in dax_copy_from_iter()  argument
342 	if (!dax_alive(dax_dev))  in dax_copy_from_iter()
345 	return dax_dev->ops->copy_from_iter(dax_dev, pgoff, addr, bytes, i);  in dax_copy_from_iter()
349 size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,  in dax_copy_to_iter()  argument
352 	if (!dax_alive(dax_dev))  in dax_copy_to_iter()
355 	return dax_dev->ops->copy_to_iter(dax_dev, pgoff, addr, bytes, i);  in dax_copy_to_iter()
359 int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,  in dax_zero_page_range()  argument
362 	if (!dax_alive(dax_dev))  in dax_zero_page_range()
372 	return dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);  in dax_zero_page_range()
378 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)  in dax_flush()  argument
380 	if (unlikely(!dax_write_cache_enabled(dax_dev)))  in dax_flush()
386 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)  in dax_flush()  argument
392 void dax_write_cache(struct dax_device *dax_dev, bool wc)  in dax_write_cache()  argument
395 		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
397 		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
401 bool dax_write_cache_enabled(struct dax_device *dax_dev)  in dax_write_cache_enabled()  argument
403 	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache_enabled()
407 bool __dax_synchronous(struct dax_device *dax_dev)  in __dax_synchronous()  argument
409 	return test_bit(DAXDEV_SYNC, &dax_dev->flags);  in __dax_synchronous()
413 void __set_dax_synchronous(struct dax_device *dax_dev)  in __set_dax_synchronous()  argument
415 	set_bit(DAXDEV_SYNC, &dax_dev->flags);  in __set_dax_synchronous()
419 bool dax_alive(struct dax_device *dax_dev)  in dax_alive()  argument
422 	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_alive()
437 void kill_dax(struct dax_device *dax_dev)  in kill_dax()  argument
439 	if (!dax_dev)  in kill_dax()
442 	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);  in kill_dax()
447 	hlist_del_init(&dax_dev->list);  in kill_dax()
452 void run_dax(struct dax_device *dax_dev)  in run_dax()  argument
454 	set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in run_dax()
460 	struct dax_device *dax_dev;  in dax_alloc_inode()  local
463 	dax_dev = kmem_cache_alloc(dax_cache, GFP_KERNEL);  in dax_alloc_inode()
464 	if (!dax_dev)  in dax_alloc_inode()
467 	inode = &dax_dev->inode;  in dax_alloc_inode()
479 	struct dax_device *dax_dev = to_dax_dev(inode);  in dax_free_inode()  local
480 	kfree(dax_dev->host);  in dax_free_inode()
481 	dax_dev->host = NULL;  in dax_free_inode()
484 	kmem_cache_free(dax_cache, dax_dev);  in dax_free_inode()
489 	struct dax_device *dax_dev = to_dax_dev(inode);  in dax_destroy_inode()  local
490 	WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),  in dax_destroy_inode()
534 	struct dax_device *dax_dev;  in dax_dev_get()  local
543 	dax_dev = to_dax_dev(inode);  in dax_dev_get()
545 		set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_dev_get()
546 		inode->i_cdev = &dax_dev->cdev;  in dax_dev_get()
553 	return dax_dev;  in dax_dev_get()
556 static void dax_add_host(struct dax_device *dax_dev, const char *host)  in dax_add_host()  argument
564 	INIT_HLIST_NODE(&dax_dev->list);  in dax_add_host()
565 	dax_dev->host = host;  in dax_add_host()
571 	hlist_add_head(&dax_dev->list, &dax_host_list[hash]);  in dax_add_host()
578 	struct dax_device *dax_dev;  in alloc_dax()  local
599 	dax_dev = dax_dev_get(devt);  in alloc_dax()
600 	if (!dax_dev)  in alloc_dax()
603 	dax_add_host(dax_dev, host);  in alloc_dax()
604 	dax_dev->ops = ops;  in alloc_dax()
605 	dax_dev->private = private;  in alloc_dax()
607 		set_dax_synchronous(dax_dev);  in alloc_dax()
609 	return dax_dev;  in alloc_dax()
619 void put_dax(struct dax_device *dax_dev)  in put_dax()  argument
621 	if (!dax_dev)  in put_dax()
623 	iput(&dax_dev->inode);  in put_dax()
633 	struct dax_device *dax_dev, *found = NULL;  in dax_get_by_host()  local
643 	hlist_for_each_entry(dax_dev, &dax_host_list[hash], list) {  in dax_get_by_host()
644 		if (!dax_alive(dax_dev)  in dax_get_by_host()
645 				|| strcmp(host, dax_dev->host) != 0)  in dax_get_by_host()
648 		if (igrab(&dax_dev->inode))  in dax_get_by_host()
649 			found = dax_dev;  in dax_get_by_host()
674 struct inode *dax_inode(struct dax_device *dax_dev)  in dax_inode()  argument
676 	return &dax_dev->inode;  in dax_inode()
680 void *dax_get_private(struct dax_device *dax_dev)  in dax_get_private()  argument
682 	if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))  in dax_get_private()
684 	return dax_dev->private;  in dax_get_private()
690 	struct dax_device *dax_dev = _dax_dev;  in init_once()  local
691 	struct inode *inode = &dax_dev->inode;  in init_once()
693 	memset(dax_dev, 0, sizeof(*dax_dev));  in init_once()