773 static void pctl_start(struct dram_info *dram,  in pctl_start()  argument
777 	const struct chan_info *chan_0 = &dram->chan[0];  in pctl_start()
778 	const struct chan_info *chan_1 = &dram->chan[1];  in pctl_start()
788 		writel(0x01000000, &dram->grf->ddrc0_con0);  in pctl_start()
804 		writel(0x01000100, &dram->grf->ddrc0_con0);  in pctl_start()
825 		writel(0x01000000, &dram->grf->ddrc1_con0);  in pctl_start()
840 		writel(0x01000100, &dram->grf->ddrc1_con0);  in pctl_start()
1813 static void dram_all_config(struct dram_info *dram,  in dram_all_config()  argument
1832 		ddr_msch_regs = dram->chan[channel].msch;  in dram_all_config()
1838 			setbits_le32(&dram->chan[channel].pctl->denali_ctl[276],  in dram_all_config()
1842 	writel(sys_reg2, &dram->pmugrf->os_reg2);  in dram_all_config()
1843 	writel(sys_reg3, &dram->pmugrf->os_reg3);  in dram_all_config()
1844 	rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10,  in dram_all_config()
1850 		&dram->pmucru->pmucru_rstnhold_con[1]);  in dram_all_config()
1851 	clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3);  in dram_all_config()
1854 static int switch_to_phy_index1(struct dram_info *dram,  in switch_to_phy_index1()  argument
1865 			&dram->cic->cic_ctrl0);  in switch_to_phy_index1()
1866 	while (!(readl(&dram->cic->cic_status0) & (1 << 2))) {  in switch_to_phy_index1()
1876 	writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0);  in switch_to_phy_index1()
1877 	while (!(readl(&dram->cic->cic_status0) & (1 << 0))) {  in switch_to_phy_index1()
1886 		denali_phy = dram->chan[channel].publ->denali_phy;  in switch_to_phy_index1()
1888 		ret = data_training(&dram->chan[channel], channel,  in switch_to_phy_index1()
2263 static u64 dram_detect_cap(struct dram_info *dram,  in dram_detect_cap()  argument
2267 	const struct chan_info *chan = &dram->chan[channel];  in dram_detect_cap()
2387 static int read_mr_for_detect(struct dram_info *dram, u32 channel, u32 rank,  in read_mr_for_detect()  argument
2396 		&dram->chan[channel];  in read_mr_for_detect()
2402 	stride = get_ddr_stride(dram->pmusgrf);  in read_mr_for_detect()
2444 		set_ddr_stride(dram->pmusgrf, 0x17);  in read_mr_for_detect()
2446 		set_ddr_stride(dram->pmusgrf, 0x18);  in read_mr_for_detect()
2477 	set_ddr_stride(dram->pmusgrf, stride);  in read_mr_for_detect()
2501 static void dram_copy_phy_fn(struct dram_info *dram,  in dram_copy_phy_fn()  argument
2513 	denali_ctl = dram->chan[channel].pctl->denali_ctl;  in dram_copy_phy_fn()
2514 	denali_phy = dram->chan[channel].publ->denali_phy;  in dram_copy_phy_fn()
2766 		read_mr(dram->chan[channel].pctl, 1, 5, &mr5);  in dram_copy_phy_fn()
2767 		set_ds_odt(&dram->chan[channel], f1_sdram_params, 1, 0, mr5);  in dram_copy_phy_fn()
2768 		set_ds_odt(&dram->chan[channel], f1_sdram_params, 1, 1, mr5);  in dram_copy_phy_fn()
2771 		set_lp4_dq_odt(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2773 		set_lp4_ca_odt(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2775 		set_lp4_MR3(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2777 		set_lp4_MR12(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2779 		set_lp4_MR14(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2782 		set_lp4_dq_odt(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2784 		set_lp4_ca_odt(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2786 		set_lp4_MR3(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2788 		set_lp4_MR12(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2790 		set_lp4_MR14(&dram->chan[channel], f1_sdram_params,  in dram_copy_phy_fn()
2821 static void dram_set_phy_fn(struct dram_info *dram,  in dram_set_phy_fn()  argument
2828 		dram_copy_phy_fn(dram, sdram_params, fn, f1_sdram_params,  in dram_set_phy_fn()
2832 static int dram_set_rate(struct dram_info *dram,  in dram_set_rate()  argument
2840 	writel(0x70007, &dram->grf->soc_con0);  in dram_set_rate()
2842 	setbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));  in dram_set_rate()
2844 	setbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));  in dram_set_rate()
2845 	while ((readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))  in dram_set_rate()
2851 		(fn << 4) | (1 << 2) | 1, &dram->cic->cic_ctrl0);  in dram_set_rate()
2852 	while (!(readl(&dram->cic->cic_status0) & (1 << 2)))  in dram_set_rate()
2855 	ret_clk = clk_set_rate(&dram->ddr_clk, hz);  in dram_set_rate()
2861 	writel(0x20002, &dram->cic->cic_ctrl0);  in dram_set_rate()
2862 	while (!(readl(&dram->cic->cic_status0) & (1 << 0)))  in dram_set_rate()
2866 	clrbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18));  in dram_set_rate()
2867 	while (readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18))  in dram_set_rate()
2871 	clrbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7));  in dram_set_rate()
2878 			ret[channel] = data_training(&dram->chan[channel],  in dram_set_rate()
2897 static void set_rate0(struct dram_info *dram)  in set_rate0()  argument
2904 	dram_set_phy_fn(dram, g_sdram_params, phy_fn, &dfs_configs[ctl_fn]);  in set_rate0()
2905 	dram_set_rate(dram, g_sdram_params, ctl_fn,  in set_rate0()
2911 static void set_rate1(struct dram_info *dram)  in set_rate1()  argument
2918 	dram_set_phy_fn(dram, g_sdram_params, phy_fn, &dfs_configs[ctl_fn]);  in set_rate1()
2919 	dram_set_rate(dram, g_sdram_params, ctl_fn,  in set_rate1()
2925 static int sdram_init(struct dram_info *dram,  in sdram_init()  argument
2951 					&dram->chan[channel];  in sdram_init()
2952 				struct rk3399_cru *cru = dram->cru;  in sdram_init()
2961 			pctl_start(dram, sdram_params, 3);  in sdram_init()
2968 			dram_set_cs(&dram->chan[ch], tmp, 2048,  in sdram_init()
2975 				if (!read_mr_for_detect(dram, ch, rank,  in sdram_init()
2990 				if (!(data_training(&dram->chan[ch], ch,  in sdram_init()
3001 		const struct chan_info *chan = &dram->chan[channel];  in sdram_init()
3016 			set_ddr_stride(dram->pmusgrf, 0x17);  in sdram_init()
3018 			set_ddr_stride(dram->pmusgrf, 0x18);  in sdram_init()
3020 		if (dram_detect_cap(dram, sdram_params, channel)) {  in sdram_init()
3044 	dram_all_config(dram, sdram_params);  in sdram_init()
3047 		switch_to_phy_index1(dram, sdram_params);  in sdram_init()
3051 		set_rate0(dram);  in sdram_init()
3052 		set_rate1(dram);  in sdram_init()