44 	struct afu *afu = cmd->parent;  in process_cmd_err()  local
45 	struct cxlflash_cfg *cfg = afu->parent;  in process_cmd_err()
156 	struct afu *afu = cmd->parent;  in cmd_complete()  local
157 	struct cxlflash_cfg *cfg = afu->parent;  in cmd_complete()
159 	struct hwq *hwq = get_hwq(afu, cmd->hwq_index);  in cmd_complete()
193 	struct cxlflash_cfg *cfg = hwq->afu->parent;  in flush_pending_cmds()
237 	struct cxlflash_cfg *cfg = hwq->afu->parent;  in context_reset()
300 static int send_cmd_ioarrin(struct afu *afu, struct afu_cmd *cmd)  in send_cmd_ioarrin()  argument
302 	struct cxlflash_cfg *cfg = afu->parent;  in send_cmd_ioarrin()
304 	struct hwq *hwq = get_hwq(afu, cmd->hwq_index);  in send_cmd_ioarrin()
344 static int send_cmd_sq(struct afu *afu, struct afu_cmd *cmd)  in send_cmd_sq()  argument
346 	struct cxlflash_cfg *cfg = afu->parent;  in send_cmd_sq()
348 	struct hwq *hwq = get_hwq(afu, cmd->hwq_index);  in send_cmd_sq()
389 static int wait_resp(struct afu *afu, struct afu_cmd *cmd)  in wait_resp()  argument
391 	struct cxlflash_cfg *cfg = afu->parent;  in wait_resp()
423 			     struct afu *afu)  in cmd_to_target_hwq()  argument
428 	if (afu->num_hwqs == 1)  in cmd_to_target_hwq()
431 	switch (afu->hwq_mode) {  in cmd_to_target_hwq()
433 		hwq = afu->hwq_rr_count++ % afu->num_hwqs;  in cmd_to_target_hwq()
440 		hwq = smp_processor_id() % afu->num_hwqs;  in cmd_to_target_hwq()
461 	struct afu *afu = cfg->afu;  in send_tmf()  local
464 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);  in send_tmf()
490 	cmd->parent = afu;  in send_tmf()
503 	rc = afu->send_cmd(afu, cmd);  in send_tmf()
563 	struct afu *afu = cfg->afu;  in cxlflash_queuecommand()  local
567 	int hwq_index = cmd_to_target_hwq(host, scp, afu);  in cxlflash_queuecommand()
568 	struct hwq *hwq = get_hwq(afu, hwq_index);  in cxlflash_queuecommand()
608 		atomic_inc(&afu->cmds_active);  in cxlflash_queuecommand()
618 	cmd->parent = afu;  in cxlflash_queuecommand()
633 	rc = afu->send_cmd(afu, cmd);  in cxlflash_queuecommand()
634 	atomic_dec(&afu->cmds_active);  in cxlflash_queuecommand()
659 	struct afu *afu = cfg->afu;  in free_mem()  local
661 	if (cfg->afu) {  in free_mem()
662 		free_pages((ulong)afu, get_order(sizeof(struct afu)));  in free_mem()
663 		cfg->afu = NULL;  in free_mem()
692 	struct afu *afu = cfg->afu;  in stop_afu()  local
700 	if (likely(afu)) {  in stop_afu()
701 		while (atomic_read(&afu->cmds_active))  in stop_afu()
704 		if (afu_is_irqpoll_enabled(afu)) {  in stop_afu()
705 			for (i = 0; i < afu->num_hwqs; i++) {  in stop_afu()
706 				hwq = get_hwq(afu, i);  in stop_afu()
712 		if (likely(afu->afu_map)) {  in stop_afu()
713 			cfg->ops->psa_unmap(afu->afu_map);  in stop_afu()
714 			afu->afu_map = NULL;  in stop_afu()
730 	struct afu *afu = cfg->afu;  in term_intr()  local
734 	if (!afu) {  in term_intr()
739 	hwq = get_hwq(afu, index);  in term_intr()
776 	struct afu *afu = cfg->afu;  in term_mc()  local
781 	if (!afu) {  in term_mc()
786 	hwq = get_hwq(afu, index);  in term_mc()
827 	for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)  in term_afu()
832 	for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)  in term_afu()
850 	struct afu *afu = cfg->afu;  in notify_shutdown()  local
861 	if (!afu || !afu->afu_map) {  in notify_shutdown()
1011 	cfg->afu = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in alloc_mem()
1012 					    get_order(sizeof(struct afu)));  in alloc_mem()
1013 	if (unlikely(!cfg->afu)) {  in alloc_mem()
1015 			__func__, get_order(sizeof(struct afu)));  in alloc_mem()
1019 	cfg->afu->parent = cfg;  in alloc_mem()
1020 	cfg->afu->desired_hwqs = CXLFLASH_DEF_HWQS;  in alloc_mem()
1021 	cfg->afu->afu_map = NULL;  in alloc_mem()
1188 static void afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs,  in afu_set_wwpn()  argument
1191 	struct cxlflash_cfg *cfg = afu->parent;  in afu_set_wwpn()
1224 static void afu_link_reset(struct afu *afu, int port, __be64 __iomem *fc_regs)  in afu_link_reset()  argument
1226 	struct cxlflash_cfg *cfg = afu->parent;  in afu_link_reset()
1231 	port_sel = readq_be(&afu->afu_map->global.regs.afu_port_sel);  in afu_link_reset()
1233 	writeq_be(port_sel, &afu->afu_map->global.regs.afu_port_sel);  in afu_link_reset()
1234 	cxlflash_afu_sync(afu, 0, 0, AFU_GSYNC);  in afu_link_reset()
1250 	writeq_be(port_sel, &afu->afu_map->global.regs.afu_port_sel);  in afu_link_reset()
1251 	cxlflash_afu_sync(afu, 0, 0, AFU_GSYNC);  in afu_link_reset()
1260 static void afu_err_intr_init(struct afu *afu)  in afu_err_intr_init()  argument
1262 	struct cxlflash_cfg *cfg = afu->parent;  in afu_err_intr_init()
1265 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);  in afu_err_intr_init()
1276 	writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_mask);  in afu_err_intr_init()
1280 	if (afu->internal_lun)  in afu_err_intr_init()
1282 	writeq_be(reg, &afu->afu_map->global.regs.afu_ctrl);  in afu_err_intr_init()
1284 	writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_clear);  in afu_err_intr_init()
1287 	writeq_be(SISL_ASTATUS_MASK, &afu->afu_map->global.regs.aintr_mask);  in afu_err_intr_init()
1290 	writeq_be(-1ULL, &afu->afu_map->global.regs.aintr_clear);  in afu_err_intr_init()
1296 	if (afu->internal_lun)  in afu_err_intr_init()
1297 		reg |= ((u64)(afu->internal_lun - 1) << SISL_FC_INTERNAL_SHIFT);  in afu_err_intr_init()
1314 	for (i = 0; i < afu->num_hwqs; i++) {  in afu_err_intr_init()
1315 		hwq = get_hwq(afu, i);  in afu_err_intr_init()
1335 	struct cxlflash_cfg *cfg = hwq->afu->parent;  in cxlflash_sync_err_irq()
1370 	struct afu *afu = hwq->afu;  in process_hrrq()  local
1390 		if (afu_is_sq_cmd_mode(afu)) {  in process_hrrq()
1473 	struct afu *afu = hwq->afu;  in cxlflash_rrq_irq()  local
1486 	if (afu_is_irqpoll_enabled(afu)) {  in cxlflash_rrq_irq()
1541 	struct afu *afu = hwq->afu;  in cxlflash_async_err_irq()  local
1542 	struct cxlflash_cfg *cfg = afu->parent;  in cxlflash_async_err_irq()
1545 	struct sisl_global_map __iomem *global = &afu->afu_map->global;  in cxlflash_async_err_irq()
1732 	struct afu *afu = cfg->afu;  in init_pcr()  local
1739 		ctrl_map = &afu->afu_map->ctrls[i].ctrl;  in init_pcr()
1748 	for (i = 0; i < afu->num_hwqs; i++) {  in init_pcr()
1749 		hwq = get_hwq(afu, i);  in init_pcr()
1753 		hwq->host_map = &afu->afu_map->hosts[hwq->ctx_hndl].host;  in init_pcr()
1754 		hwq->ctrl_map = &afu->afu_map->ctrls[hwq->ctx_hndl].ctrl;  in init_pcr()
1767 	struct afu *afu = cfg->afu;  in init_global()  local
1786 	for (i = 0; i < afu->num_hwqs; i++) {  in init_global()
1787 		hwq = get_hwq(afu, i);  in init_global()
1794 		if (afu_is_sq_cmd_mode(afu)) {  in init_global()
1801 	reg = readq_be(&afu->afu_map->global.regs.afu_config);  in init_global()
1807 	writeq_be(reg, &afu->afu_map->global.regs.afu_config);  in init_global()
1810 	if (afu->internal_lun) {  in init_global()
1812 		writeq_be(PORT0, &afu->afu_map->global.regs.afu_port_sel);  in init_global()
1816 			  &afu->afu_map->global.regs.afu_port_sel);  in init_global()
1831 			afu_set_wwpn(afu, i, &fc_port_regs[0], wwpn[i]);  in init_global()
1838 	if (afu_is_ocxl_lisn(afu)) {  in init_global()
1840 		for (i = 0; i < afu->num_hwqs; i++) {  in init_global()
1841 			hwq = get_hwq(afu, i);  in init_global()
1860 	for (i = 0; i < afu->num_hwqs; i++) {  in init_global()
1861 		hwq = get_hwq(afu, i);  in init_global()
1876 	hwq = get_hwq(afu, PRIMARY_HWQ);  in init_global()
1882 	afu->hb = readq_be(&afu->afu_map->global.regs.afu_hb);  in init_global()
1893 	struct afu *afu = cfg->afu;  in start_afu()  local
1902 	for (i = 0; i < afu->num_hwqs; i++) {  in start_afu()
1903 		hwq = get_hwq(afu, i);  in start_afu()
1919 		if (afu_is_sq_cmd_mode(afu)) {  in start_afu()
1929 		if (afu_is_irqpoll_enabled(afu))  in start_afu()
1930 			irq_poll_init(&hwq->irqpoll, afu->irqpoll_weight,  in start_afu()
2008 	struct hwq *hwq = get_hwq(cfg->afu, index);  in init_mc()
2013 	hwq->afu = cfg->afu;  in init_mc()
2089 	struct afu *afu = cfg->afu;  in get_num_afu_ports()  local
2094 	port_mask = readq_be(&afu->afu_map->global.regs.afu_port_sel);  in get_num_afu_ports()
2118 	struct afu *afu = cfg->afu;  in init_afu()  local
2125 	mutex_init(&afu->sync_active);  in init_afu()
2126 	afu->num_hwqs = afu->desired_hwqs;  in init_afu()
2127 	for (i = 0; i < afu->num_hwqs; i++) {  in init_afu()
2137 	hwq = get_hwq(afu, PRIMARY_HWQ);  in init_afu()
2138 	afu->afu_map = cfg->ops->psa_map(hwq->ctx_cookie);  in init_afu()
2139 	if (!afu->afu_map) {  in init_afu()
2146 	reg = readq(&afu->afu_map->global.regs.afu_version);  in init_afu()
2147 	memcpy(afu->version, &reg, sizeof(reg));  in init_afu()
2148 	afu->interface_version =  in init_afu()
2149 	    readq_be(&afu->afu_map->global.regs.interface_version);  in init_afu()
2150 	if ((afu->interface_version + 1) == 0) {  in init_afu()
2152 			"interface version %016llx\n", afu->version,  in init_afu()
2153 		       afu->interface_version);  in init_afu()
2158 	if (afu_is_sq_cmd_mode(afu)) {  in init_afu()
2159 		afu->send_cmd = send_cmd_sq;  in init_afu()
2160 		afu->context_reset = context_reset_sq;  in init_afu()
2162 		afu->send_cmd = send_cmd_ioarrin;  in init_afu()
2163 		afu->context_reset = context_reset_ioarrin;  in init_afu()
2167 		afu->version, afu->interface_version);  in init_afu()
2177 	afu_err_intr_init(cfg->afu);  in init_afu()
2178 	for (i = 0; i < afu->num_hwqs; i++) {  in init_afu()
2179 		hwq = get_hwq(afu, i);  in init_afu()
2191 	for (i = afu->num_hwqs - 1; i >= 0; i--) {  in init_afu()
2299 static int send_afu_cmd(struct afu *afu, struct sisl_ioarcb *rcb)  in send_afu_cmd()  argument
2301 	struct cxlflash_cfg *cfg = afu->parent;  in send_afu_cmd()
2304 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);  in send_afu_cmd()
2316 	mutex_lock(&afu->sync_active);  in send_afu_cmd()
2317 	atomic_inc(&afu->cmds_active);  in send_afu_cmd()
2332 	cmd->parent = afu;  in send_afu_cmd()
2337 		__func__, afu, cmd, cmd->rcb.cdb[0], nretry);  in send_afu_cmd()
2339 	rc = afu->send_cmd(afu, cmd);  in send_afu_cmd()
2345 	rc = wait_resp(afu, cmd);  in send_afu_cmd()
2348 		rc = afu->context_reset(hwq);  in send_afu_cmd()
2370 	atomic_dec(&afu->cmds_active);  in send_afu_cmd()
2371 	mutex_unlock(&afu->sync_active);  in send_afu_cmd()
2393 int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx, res_hndl_t res, u8 mode)  in cxlflash_afu_sync()  argument
2395 	struct cxlflash_cfg *cfg = afu->parent;  in cxlflash_afu_sync()
2400 		__func__, afu, ctx, res, mode);  in cxlflash_afu_sync()
2411 	return send_afu_cmd(afu, &rcb);  in cxlflash_afu_sync()
2431 	struct afu *afu = cfg->afu;  in cxlflash_eh_abort_handler()  local
2432 	struct hwq *hwq = get_hwq(afu, cmd->hwq_index);  in cxlflash_eh_abort_handler()
2451 	rc = afu->context_reset(hwq);  in cxlflash_eh_abort_handler()
2689 	struct afu *afu = cfg->afu;  in lun_mode_show()  local
2691 	return scnprintf(buf, PAGE_SIZE, "%u\n", afu->internal_lun);  in lun_mode_show()
2722 	struct afu *afu = cfg->afu;  in lun_mode_store()  local
2727 	if (!rc && (lun_mode < 5) && (lun_mode != afu->internal_lun)) {  in lun_mode_store()
2728 		afu->internal_lun = lun_mode;  in lun_mode_store()
2735 		if (afu->internal_lun)  in lun_mode_store()
2885 	struct afu *afu = cfg->afu;  in irqpoll_weight_show()  local
2887 	return scnprintf(buf, PAGE_SIZE, "%u\n", afu->irqpoll_weight);  in irqpoll_weight_show()
2908 	struct afu *afu = cfg->afu;  in irqpoll_weight_store()  local
2923 	if (weight == afu->irqpoll_weight) {  in irqpoll_weight_store()
2929 	if (afu_is_irqpoll_enabled(afu)) {  in irqpoll_weight_store()
2930 		for (i = 0; i < afu->num_hwqs; i++) {  in irqpoll_weight_store()
2931 			hwq = get_hwq(afu, i);  in irqpoll_weight_store()
2937 	afu->irqpoll_weight = weight;  in irqpoll_weight_store()
2940 		for (i = 0; i < afu->num_hwqs; i++) {  in irqpoll_weight_store()
2941 			hwq = get_hwq(afu, i);  in irqpoll_weight_store()
2963 	struct afu *afu = cfg->afu;  in num_hwqs_show()  local
2965 	return scnprintf(buf, PAGE_SIZE, "%u\n", afu->num_hwqs);  in num_hwqs_show()
2987 	struct afu *afu = cfg->afu;  in num_hwqs_store()  local
3002 	afu->desired_hwqs = min(num_hwqs, CXLFLASH_MAX_HWQS);  in num_hwqs_store()
3003 	WARN_ON_ONCE(afu->desired_hwqs == 0);  in num_hwqs_store()
3048 	struct afu *afu = cfg->afu;  in hwq_mode_show()  local
3050 	return scnprintf(buf, PAGE_SIZE, "%s\n", hwq_mode_name[afu->hwq_mode]);  in hwq_mode_show()
3074 	struct afu *afu = cfg->afu;  in hwq_mode_store()  local
3090 	afu->hwq_mode = mode;  in hwq_mode_store()
3219 	struct afu *afu = cfg->afu;  in cxlflash_worker_thread()  local
3243 			afu_link_reset(afu, port, fc_port_regs);  in cxlflash_worker_thread()
3304 	struct afu *afu = cfg->afu;  in cxlflash_lun_provision()  local
3317 	if (!afu_is_lun_provision(afu)) {  in cxlflash_lun_provision()
3369 	rc = send_afu_cmd(afu, &rcb);  in cxlflash_lun_provision()
3398 	struct afu *afu = cfg->afu;  in cxlflash_afu_debug()  local
3410 	if (!afu_is_afu_debug(afu)) {  in cxlflash_afu_debug()
3458 	rc = send_afu_cmd(afu, &rcb);  in cxlflash_afu_debug()