67 	u32 reg, phase, delay, cs, pup;  in ddr3_write_leveling_hw()  local
76 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);  in ddr3_write_leveling_hw()
77 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {  in ddr3_write_leveling_hw()
80 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));  in ddr3_write_leveling_hw()
84 	reg = 1 << REG_DRAM_TRAINING_WL_OFFS;  in ddr3_write_leveling_hw()
86 	reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);  in ddr3_write_leveling_hw()
87 	reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));  in ddr3_write_leveling_hw()
88 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */  in ddr3_write_leveling_hw()
90 	reg =  reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |  in ddr3_write_leveling_hw()
92 	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);  in ddr3_write_leveling_hw()
96 		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &  in ddr3_write_leveling_hw()
98 	} while (reg);		/* Wait for '0' */  in ddr3_write_leveling_hw()
100 	reg = reg_read(REG_DRAM_TRAINING_ADDR);  in ddr3_write_leveling_hw()
102 	if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {  in ddr3_write_leveling_hw()
115 					reg =  in ddr3_write_leveling_hw()
119 					    (reg >> REG_PHY_PHASE_OFFS) &  in ddr3_write_leveling_hw()
121 					delay = reg & PUP_DELAY_MASK;  in ddr3_write_leveling_hw()
126 					reg =  in ddr3_write_leveling_hw()
130 					    (reg & 0x3F);  in ddr3_write_leveling_hw()
163 			reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |  in ddr3_write_leveling_hw()
165 			reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);  in ddr3_write_leveling_hw()
188 	u32 tmp_count, ecc, reg;  in ddr3_wl_supplement()  local
219 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |  in ddr3_wl_supplement()
224 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_wl_supplement()
226 	reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);  in ddr3_wl_supplement()
227 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */  in ddr3_wl_supplement()
251 					reg =  in ddr3_wl_supplement()
255 					reg |=  in ddr3_wl_supplement()
260 						  reg);  in ddr3_wl_supplement()
400 					reg =  in ddr3_wl_supplement()
405 						  reg);  in ddr3_wl_supplement()
437 				reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);  in ddr3_wl_supplement()
439 				    (reg >> REG_PHY_PHASE_OFFS) &  in ddr3_wl_supplement()
451 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);  in ddr3_wl_supplement()
452 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);  in ddr3_wl_supplement()
454 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_wl_supplement()
456 	reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |  in ddr3_wl_supplement()
458 	reg_write(REG_DRAM_TRAINING_1_ADDR, reg);  in ddr3_wl_supplement()
475 	u32 reg, phase, delay, cs, pup, pup_num;  in ddr3_write_leveling_hw_reg_dimm()  local
499 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);  in ddr3_write_leveling_hw_reg_dimm()
500 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {  in ddr3_write_leveling_hw_reg_dimm()
503 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));  in ddr3_write_leveling_hw_reg_dimm()
507 	reg = (1 << REG_DRAM_TRAINING_WL_OFFS);  in ddr3_write_leveling_hw_reg_dimm()
509 	reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);  in ddr3_write_leveling_hw_reg_dimm()
510 	reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));  in ddr3_write_leveling_hw_reg_dimm()
511 	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */  in ddr3_write_leveling_hw_reg_dimm()
513 	reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |  in ddr3_write_leveling_hw_reg_dimm()
515 	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);  in ddr3_write_leveling_hw_reg_dimm()
519 		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &  in ddr3_write_leveling_hw_reg_dimm()
521 	} while (reg);		/* Wait for '0' */  in ddr3_write_leveling_hw_reg_dimm()
523 	reg = reg_read(REG_DRAM_TRAINING_ADDR);  in ddr3_write_leveling_hw_reg_dimm()
525 	if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {  in ddr3_write_leveling_hw_reg_dimm()
538 					reg =  in ddr3_write_leveling_hw_reg_dimm()
542 					    (reg >> REG_PHY_PHASE_OFFS) &  in ddr3_write_leveling_hw_reg_dimm()
544 					delay = reg & PUP_DELAY_MASK;  in ddr3_write_leveling_hw_reg_dimm()
562 					reg =  in ddr3_write_leveling_hw_reg_dimm()
566 					    (reg & 0x3F);  in ddr3_write_leveling_hw_reg_dimm()
599 			reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |  in ddr3_write_leveling_hw_reg_dimm()
601 			reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);  in ddr3_write_leveling_hw_reg_dimm()
660 	u32 reg, cs, cnt, pup, max_pup_num;  in ddr3_write_leveling_sw()  local
670 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);  in ddr3_write_leveling_sw()
671 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {  in ddr3_write_leveling_sw()
674 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));  in ddr3_write_leveling_sw()
681 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw()
683 			reg |= odt_static[dram_info->cs_ena][cs];  in ddr3_write_leveling_sw()
684 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);  in ddr3_write_leveling_sw()
687 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw()
689 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw()
696 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw()
705 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |  in ddr3_write_leveling_sw()
709 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw()
713 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &  in ddr3_write_leveling_sw()
717 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw()
729 				reg =  in ddr3_write_leveling_sw()
734 				reg_write(REG_SDRAM_OPERATION_ADDR, reg);	/* 0x1418 - SDRAM Operation Register */  in ddr3_write_leveling_sw()
737 					reg =  in ddr3_write_leveling_sw()
741 				} while (reg);	/* Wait for '0' */  in ddr3_write_leveling_sw()
746 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw()
749 			reg &= REG_DDR3_MR1_ODT_MASK;  in ddr3_write_leveling_sw()
750 			reg |= odt_static[dram_info->cs_ena][cs];  in ddr3_write_leveling_sw()
752 			reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);  in ddr3_write_leveling_sw()
754 			reg_write(REG_DDR3_MR1_ADDR, reg);	/* 0x15D4 - DDR3 MR1 Register */  in ddr3_write_leveling_sw()
756 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw()
763 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw()
784 			reg = reg_read(REG_TRAINING_WL_ADDR) |  in ddr3_write_leveling_sw()
787 			reg_write(REG_TRAINING_WL_ADDR, reg);  in ddr3_write_leveling_sw()
802 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw()
804 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);  in ddr3_write_leveling_sw()
807 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw()
809 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw()
816 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw()
824 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);  in ddr3_write_leveling_sw()
825 	reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);  in ddr3_write_leveling_sw()
827 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw()
831 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);  in ddr3_write_leveling_sw()
832 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);  in ddr3_write_leveling_sw()
834 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw()
839 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw()
841 			reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;  in ddr3_write_leveling_sw()
842 			reg |= odt_static[dram_info->cs_ena][cs];  in ddr3_write_leveling_sw()
845 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw()
847 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw()
854 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw()
863 		reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |  in ddr3_write_leveling_sw()
865 		reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);  in ddr3_write_leveling_sw()
885 	u32 reg, cs, cnt, pup;  in ddr3_write_leveling_sw_reg_dimm()  local
894 	reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);  in ddr3_write_leveling_sw_reg_dimm()
895 	if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {  in ddr3_write_leveling_sw_reg_dimm()
898 			  reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));  in ddr3_write_leveling_sw_reg_dimm()
916 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw_reg_dimm()
918 			reg |= odt_static[dram_info->cs_ena][cs];  in ddr3_write_leveling_sw_reg_dimm()
919 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);  in ddr3_write_leveling_sw_reg_dimm()
922 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
924 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw_reg_dimm()
931 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
940 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |  in ddr3_write_leveling_sw_reg_dimm()
944 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
948 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &  in ddr3_write_leveling_sw_reg_dimm()
952 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
963 				reg =  in ddr3_write_leveling_sw_reg_dimm()
968 				reg_write(REG_SDRAM_OPERATION_ADDR, reg);	/* 0x1418 - SDRAM Operation Register */  in ddr3_write_leveling_sw_reg_dimm()
971 					reg =  in ddr3_write_leveling_sw_reg_dimm()
975 				} while (reg);	/* Wait for '0' */  in ddr3_write_leveling_sw_reg_dimm()
983 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw_reg_dimm()
986 			reg &= REG_DDR3_MR1_ODT_MASK;  in ddr3_write_leveling_sw_reg_dimm()
987 			reg |= odt_static[dram_info->cs_ena][cs];  in ddr3_write_leveling_sw_reg_dimm()
989 			reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);  in ddr3_write_leveling_sw_reg_dimm()
995 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
997 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw_reg_dimm()
1004 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1019 			reg = reg_read(REG_TRAINING_WL_ADDR) |  in ddr3_write_leveling_sw_reg_dimm()
1022 			reg_write(REG_TRAINING_WL_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1034 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw_reg_dimm()
1036 			reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);  in ddr3_write_leveling_sw_reg_dimm()
1039 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1041 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw_reg_dimm()
1048 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1056 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);  in ddr3_write_leveling_sw_reg_dimm()
1057 	reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);  in ddr3_write_leveling_sw_reg_dimm()
1059 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1063 	reg = reg_read(REG_DRAM_TRAINING_2_ADDR);  in ddr3_write_leveling_sw_reg_dimm()
1064 	reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);  in ddr3_write_leveling_sw_reg_dimm()
1066 	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1071 			reg = reg_read(REG_DDR3_MR1_ADDR) &  in ddr3_write_leveling_sw_reg_dimm()
1073 			reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;  in ddr3_write_leveling_sw_reg_dimm()
1074 			reg |= odt_static[dram_info->cs_ena][cs];  in ddr3_write_leveling_sw_reg_dimm()
1077 			reg_write(REG_DDR3_MR1_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1079 			reg = REG_SDRAM_OPERATION_CMD_MR1 &  in ddr3_write_leveling_sw_reg_dimm()
1086 			reg_write(REG_SDRAM_OPERATION_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1095 		reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |  in ddr3_write_leveling_sw_reg_dimm()
1097 		reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);  in ddr3_write_leveling_sw_reg_dimm()
1128 	u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,  in ddr3_write_leveling_single_cs()  local
1165 	reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &  in ddr3_write_leveling_single_cs()
1167 	reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));  in ddr3_write_leveling_single_cs()
1170 	reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);  in ddr3_write_leveling_single_cs()
1179 	reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;  in ddr3_write_leveling_single_cs()
1180 	reg |= (1 << REG_TRAINING_WL_UPD_OFFS);	/* [2] - trnWLCsUpd */  in ddr3_write_leveling_single_cs()
1182 	reg_write(REG_TRAINING_WL_ADDR, reg);  in ddr3_write_leveling_single_cs()
1193 		reg = (reg_read(REG_TRAINING_WL_ADDR) &  in ddr3_write_leveling_single_cs()
1196 		reg = (reg_read(REG_TRAINING_WL_ADDR) &  in ddr3_write_leveling_single_cs()
1200 	reg_write(REG_TRAINING_WL_ADDR, reg);  in ddr3_write_leveling_single_cs()
1205 		reg = (reg_read(REG_TRAINING_WL_ADDR)) &  in ddr3_write_leveling_single_cs()
1207 	} while (reg == 0x0);	/* Wait for '1' */  in ddr3_write_leveling_single_cs()
1210 	reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &  in ddr3_write_leveling_single_cs()
1238 				reg = (reg_read(REG_TRAINING_WL_ADDR) &  in ddr3_write_leveling_single_cs()
1242 				reg = (reg_read(REG_TRAINING_WL_ADDR) &  in ddr3_write_leveling_single_cs()
1246 			reg_write(REG_TRAINING_WL_ADDR, reg);	/* 0x16AC  */  in ddr3_write_leveling_single_cs()
1250 				reg = (reg_read(REG_TRAINING_WL_ADDR)) &  in ddr3_write_leveling_single_cs()
1252 			} while (reg == 0x0);	/* [29] Wait for '1' */  in ddr3_write_leveling_single_cs()
1255 			reg = reg_read(REG_TRAINING_WL_ADDR);  in ddr3_write_leveling_single_cs()
1256 			reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &  in ddr3_write_leveling_single_cs()
1260 					(u32) reg, 3);  in ddr3_write_leveling_single_cs()
1274 				if (((reg >> pup_num) & 0x1) == 0)  in ddr3_write_leveling_single_cs()
1277 				if (((reg >> pup_num) & 0x1)  in ddr3_write_leveling_single_cs()
1328 	reg =  reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &  in ddr3_write_leveling_single_cs()
1331 	reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);  in ddr3_write_leveling_single_cs()
1341 	u32 reg = 0;  in ddr3_write_ctrl_pup_reg()  local
1344 	reg = (data & 0xFFFF);  in ddr3_write_ctrl_pup_reg()
1347 	reg |= (1 << REG_PHY_CNTRL_OFFS);  in ddr3_write_ctrl_pup_reg()
1351 		reg |= (1 << REG_PHY_BC_OFFS);  in ddr3_write_ctrl_pup_reg()
1353 		reg |= (pup << REG_PHY_PUP_OFFS);  in ddr3_write_ctrl_pup_reg()
1356 	reg |= (reg_addr << REG_PHY_CS_OFFS);  in ddr3_write_ctrl_pup_reg()
1358 	reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);	/* 0x16A0 */  in ddr3_write_ctrl_pup_reg()
1359 	reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;  in ddr3_write_ctrl_pup_reg()
1360 	reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);	/* 0x16A0 */  in ddr3_write_ctrl_pup_reg()
1363 		reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &  in ddr3_write_ctrl_pup_reg()
1365 	} while (reg);		/* Wait for '0' to mark the end of the transaction */  in ddr3_write_ctrl_pup_reg()