259 static void scc_mgr_set(u32 off, u32 grp, u32 val)  in scc_mgr_set()  argument
261 	writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));  in scc_mgr_set()
376 static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,  in scc_mgr_set_all_ranks()  argument
383 		scc_mgr_set(off, grp, val);  in scc_mgr_set_all_ranks()
386 			writel(grp, &sdr_scc_mgr->dqs_ena);  in scc_mgr_set_all_ranks()
1481 rw_mgr_mem_calibrate_read_test_all_ranks(const u32 grp, const u32 num_tries,  in rw_mgr_mem_calibrate_read_test_all_ranks()  argument
1486 	return rw_mgr_mem_calibrate_read_test(0, grp, num_tries, all_correct,  in rw_mgr_mem_calibrate_read_test_all_ranks()
1496 static void rw_mgr_incr_vfifo(const u32 grp)  in rw_mgr_incr_vfifo()  argument
1498 	writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);  in rw_mgr_incr_vfifo()
1507 static void rw_mgr_decr_vfifo(const u32 grp)  in rw_mgr_decr_vfifo()  argument
1512 		rw_mgr_incr_vfifo(grp);  in rw_mgr_decr_vfifo()
1521 static int find_vfifo_failing_read(const u32 grp)  in find_vfifo_failing_read()  argument
1528 		ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in find_vfifo_failing_read()
1538 		rw_mgr_incr_vfifo(grp);  in find_vfifo_failing_read()
1557 static int sdr_find_phase_delay(int working, int delay, const u32 grp,  in sdr_find_phase_delay()  argument
1566 			scc_mgr_set_dqs_en_delay_all_ranks(grp, *pd);  in sdr_find_phase_delay()
1568 			scc_mgr_set_dqs_en_phase_all_ranks(grp, *pd);  in sdr_find_phase_delay()
1570 		ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in sdr_find_phase_delay()
1594 static int sdr_find_phase(int working, const u32 grp, u32 *work,  in sdr_find_phase()  argument
1604 		ret = sdr_find_phase_delay(working, 0, grp, work,  in sdr_find_phase()
1611 			rw_mgr_incr_vfifo(grp);  in sdr_find_phase()
1630 static int sdr_working_phase(const u32 grp, u32 *work_bgn, u32 *d,  in sdr_working_phase()  argument
1641 		scc_mgr_set_dqs_en_delay_all_ranks(grp, *d);  in sdr_working_phase()
1642 		ret = sdr_find_phase(1, grp, work_bgn, i, p);  in sdr_working_phase()
1662 static void sdr_backup_phase(const u32 grp, u32 *work_bgn, u32 *p)  in sdr_backup_phase()  argument
1670 		rw_mgr_decr_vfifo(grp);  in sdr_backup_phase()
1675 	scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);  in sdr_backup_phase()
1679 		scc_mgr_set_dqs_en_delay_all_ranks(grp, d);  in sdr_backup_phase()
1681 		ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in sdr_backup_phase()
1695 		rw_mgr_incr_vfifo(grp);  in sdr_backup_phase()
1698 	scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);  in sdr_backup_phase()
1710 static int sdr_nonworking_phase(const u32 grp, u32 *work_end, u32 *p, u32 *i)  in sdr_nonworking_phase()  argument
1719 		rw_mgr_incr_vfifo(grp);  in sdr_nonworking_phase()
1722 	ret = sdr_find_phase(0, grp, work_end, i, p);  in sdr_nonworking_phase()
1740 static int sdr_find_window_center(const u32 grp, const u32 work_bgn,  in sdr_find_window_center()  argument
1773 	scc_mgr_set_dqs_en_phase_all_ranks(grp, p);  in sdr_find_window_center()
1774 	scc_mgr_set_dqs_en_delay_all_ranks(grp, d);  in sdr_find_window_center()
1782 		if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,  in sdr_find_window_center()
1792 		rw_mgr_incr_vfifo(grp);  in sdr_find_window_center()
1806 static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp)  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()  argument
1814 	debug("%s:%d %u\n", __func__, __LINE__, grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1818 	scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1819 	scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1826 	find_vfifo_failing_read(grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1830 	ret = sdr_working_phase(grp, &work_bgn, &d, &p, &i);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1846 		sdr_backup_phase(grp, &work_bgn, &p);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1852 		ret = sdr_nonworking_phase(grp, &work_end, &p, &i);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1861 			rw_mgr_decr_vfifo(grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1867 		scc_mgr_set_dqs_en_phase_all_ranks(grp, p);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1876 	sdr_find_phase_delay(0, 1, grp, &work_end,  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1908 		rw_mgr_decr_vfifo(grp);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1917 	scc_mgr_set_dqs_en_phase_all_ranks(grp, p);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1931 	found_passing_read = !sdr_find_phase_delay(1, 1, grp, NULL, 0, &d);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1937 		found_failing_read = !sdr_find_phase_delay(0, 1, grp, NULL, 0,  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()
1959 	ret = sdr_find_window_center(grp, work_bgn, work_end);  in rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase()