67 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay);
97 	u32 pup, dq, pups, cur_max_pup, valid_pup, reg;  in ddr3_pbs_tx()  local
125 		for (pup = 0; pup < pups; pup++) {  in ddr3_pbs_tx()
127 				skew_sum_array[pup][dq] = 0;  in ddr3_pbs_tx()
173 				for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_tx()
176 							pup + ecc *  in ddr3_pbs_tx()
204 							pbs_dq_mapping[pup *  in ddr3_pbs_tx()
245 				for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_tx()
251 					DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1);  in ddr3_pbs_tx()
266 							    [((pup) * DQ_NUM) +  in ddr3_pbs_tx()
277 				for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_tx()
279 						skew_sum_array[pup + (ecc * (max_pup - 1))]  in ddr3_pbs_tx()
281 							[((pup) * DQ_NUM) + dq];  in ddr3_pbs_tx()
294 		for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_tx()
300 			DEBUG_PBS_D(pup, 1);  in ddr3_pbs_tx()
309 				DEBUG_PBS_D(skew_sum_array[pup][dq] /  in ddr3_pbs_tx()
323 		for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_tx()
330 				pattern_skew_array[pup][dq] +=  in ddr3_pbs_tx()
331 					(skew_sum_array[pup][dq] /  in ddr3_pbs_tx()
338 	for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_tx()
340 			skew_array[((pup) * DQ_NUM) + dq] =  in ddr3_pbs_tx()
341 				pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;  in ddr3_pbs_tx()
345 	for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_tx()
351 		DEBUG_PBS_D(pup, 1);  in ddr3_pbs_tx()
364 			DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);  in ddr3_pbs_tx()
371 	for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_tx()
372 		if (pup == (max_pup - 1) && dram_info->ecc_ena)  in ddr3_pbs_tx()
373 			pup = ECC_PUP;  in ddr3_pbs_tx()
374 		ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY);  in ddr3_pbs_tx()
412 	u32 cur_max_pup, pup;  in ddr3_tx_shift_dqs_adll_step_before_fail()  local
452 		for (pup = 0; pup < cur_max_pup; pup++)  in ddr3_tx_shift_dqs_adll_step_before_fail()
454 						   pup * (1 - ecc) +  in ddr3_tx_shift_dqs_adll_step_before_fail()
483 			for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_tx_shift_dqs_adll_step_before_fail()
484 				if (((new_lockup_pup >> pup) & 0x1) &&  in ddr3_tx_shift_dqs_adll_step_before_fail()
485 				    dqs_dly_set[pup] == 0)  in ddr3_tx_shift_dqs_adll_step_before_fail()
486 					dqs_dly_set[pup] = adll_val - 1;  in ddr3_tx_shift_dqs_adll_step_before_fail()
494 		for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_tx_shift_dqs_adll_step_before_fail()
495 			if (((unlock_pup >> pup) & 0x1) &&  in ddr3_tx_shift_dqs_adll_step_before_fail()
496 			    dqs_dly_set[pup] == 0)  in ddr3_tx_shift_dqs_adll_step_before_fail()
497 				dqs_dly_set[pup] = adll_val - 1;  in ddr3_tx_shift_dqs_adll_step_before_fail()
505 	for (pup = 0; pup < cur_max_pup; pup++)  in ddr3_tx_shift_dqs_adll_step_before_fail()
506 		ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc,  in ddr3_tx_shift_dqs_adll_step_before_fail()
507 					   dqs_dly_set[pup]);  in ddr3_tx_shift_dqs_adll_step_before_fail()
539 	u32 pup, dq, pups, cur_max_pup, valid_pup, reg;  in ddr3_pbs_rx()  local
568 		for (pup = 0; pup < pups; pup++) {  in ddr3_pbs_rx()
570 				skew_sum_array[pup][dq] = 0;  in ddr3_pbs_rx()
615 				for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_rx()
618 							pup + ecc * (max_pup - 1)][dq] =  in ddr3_pbs_rx()
640 					for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_rx()
645 								pup * (1 - ecc)  in ddr3_pbs_rx()
648 								pup + ecc * ECC_PUP,  in ddr3_pbs_rx()
653 					for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_rx()
656 								   pup +  in ddr3_pbs_rx()
699 						for (pup = 0; pup < max_pup;  in ddr3_pbs_rx()
700 						     pup++) {  in ddr3_pbs_rx()
703 							     pup +  in ddr3_pbs_rx()
709 						for (pup = 0; pup < cur_max_pup;  in ddr3_pbs_rx()
710 						     pup++) {  in ddr3_pbs_rx()
716 								     [pup * (1 - ecc) +  in ddr3_pbs_rx()
719 								     pup + ecc * ECC_PUP,  in ddr3_pbs_rx()
748 				for (pup = 0; pup < cur_max_pup; pup++)  in ddr3_pbs_rx()
750 							   pup + ecc * ECC_PUP,  in ddr3_pbs_rx()
755 				for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_rx()
761 					DEBUG_PBS_FULL_D((pup +  in ddr3_pbs_rx()
777 								 [((pup) *  in ddr3_pbs_rx()
789 				for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_pbs_rx()
792 							[pup + (ecc * (max_pup - 1))]  in ddr3_pbs_rx()
794 							skew_array[((pup) * DQ_NUM) + dq];  in ddr3_pbs_rx()
811 		for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_rx()
818 				pattern_skew_array[pup][dq] +=  in ddr3_pbs_rx()
819 					(skew_sum_array[pup][dq] /  in ddr3_pbs_rx()
826 		for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_rx()
832 			DEBUG_PBS_D(pup, 1);  in ddr3_pbs_rx()
845 				DEBUG_PBS_D(skew_sum_array[pup][dq] /  in ddr3_pbs_rx()
854 	for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_rx()
856 			skew_array[((pup) * DQ_NUM) + dq] =  in ddr3_pbs_rx()
857 				pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN;  in ddr3_pbs_rx()
861 	for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_rx()
867 		DEBUG_PBS_D(pup, 1);  in ddr3_pbs_rx()
880 			DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2);  in ddr3_pbs_rx()
924 	u32 cur_max_pup, pup, pass_pup;  in ddr3_rx_shift_dqs_to_first_fail()  local
958 	for (pup = 0; pup < cur_max_pup; pup++)  in ddr3_rx_shift_dqs_to_first_fail()
959 		ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0,  in ddr3_rx_shift_dqs_to_first_fail()
984 			ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP,  in ddr3_rx_shift_dqs_to_first_fail()
1009 		for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_rx_shift_dqs_to_first_fail()
1010 			if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {  in ddr3_rx_shift_dqs_to_first_fail()
1012 						   pup + ecc * ECC_PUP, 0,  in ddr3_rx_shift_dqs_to_first_fail()
1045 		for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_rx_shift_dqs_to_first_fail()
1046 			if (IS_PUP_ACTIVE(pass_pup, pup) == 1) {  in ddr3_rx_shift_dqs_to_first_fail()
1048 						   pup + ecc * ECC_PUP, 0,  in ddr3_rx_shift_dqs_to_first_fail()
1071 		for (pup = 0; pup < cur_max_pup; pup++) {  in ddr3_rx_shift_dqs_to_first_fail()
1072 			if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) {  in ddr3_rx_shift_dqs_to_first_fail()
1074 						   pup + ecc * ECC_PUP, 0,  in ddr3_rx_shift_dqs_to_first_fail()
1087 static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array,  in lock_pups()  argument
1095 	DEBUG_PBS_FULL_D(pup, 1);  in lock_pups()
1098 	idx = pup * (1 - ecc) + ecc * ECC_PUP;  in lock_pups()
1099 	*pup_locked &= ~(1 << pup);  in lock_pups()
1102 		if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) {  in lock_pups()
1106 			unlock_pup_dq_array[dq] &= ~(1 << pup);  in lock_pups()
1107 			skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val;  in lock_pups()
1149 	u32 pup, dq;  in ddr3_pbs_per_bit()  local
1211 			for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_per_bit()
1214 				idx = pup * (1 - ecc) + ecc * ECC_PUP;  in ddr3_pbs_per_bit()
1216 				if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup)  in ddr3_pbs_per_bit()
1249 			for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_per_bit()
1252 							   pup) == 1)  in ddr3_pbs_per_bit()
1255 					      pup) == 0)) {  in ddr3_pbs_per_bit()
1259 						DEBUG_PBS_FULL_D(pup, 1);  in ddr3_pbs_per_bit()
1309 			for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_per_bit()
1311 				if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {  in ddr3_pbs_per_bit()
1313 					if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==  in ddr3_pbs_per_bit()
1316 								 pup, 1);  in ddr3_pbs_per_bit()
1319 					    if (IS_PUP_ACTIVE(sum_pup_fail, pup)  in ddr3_pbs_per_bit()
1322 								 pup, 1);  in ddr3_pbs_per_bit()
1328 					DEBUG_PBS_FULL_D(pup, 1);  in ddr3_pbs_per_bit()
1334 						     pup) == 1) {  in ddr3_pbs_per_bit()
1338 							skew_array[((pup) *  in ddr3_pbs_per_bit()
1351 						*pcur_pup &= ~(1 << pup);  in ddr3_pbs_per_bit()
1356 					DEBUG_PBS_FULL_D(pup, 1);  in ddr3_pbs_per_bit()
1371 		for (pup = 0; pup < max_pup; pup++) {  in ddr3_pbs_per_bit()
1372 			if (IS_PUP_ACTIVE(pup_locked, pup) == 1) {  in ddr3_pbs_per_bit()
1374 				if (first_failed[pup] == 0) {  in ddr3_pbs_per_bit()
1376 					if (IS_PUP_ACTIVE(sum_pup_fail, pup) ==  in ddr3_pbs_per_bit()
1380 								 pup, 1);  in ddr3_pbs_per_bit()
1381 						first_failed[pup] = 1;  in ddr3_pbs_per_bit()
1382 						first_fail[pup] = pbs_curr_val;  in ddr3_pbs_per_bit()
1389 							first_fail[pup];  in ddr3_pbs_per_bit()
1392 						calc_pbs_diff = first_fail[pup] -  in ddr3_pbs_per_bit()
1397 						lock_pups(pup, &pup_locked,  in ddr3_pbs_per_bit()
1419 	u32 pup, phys_pup, dq;  in ddr3_set_pbs_results()  local
1429 	for (pup = 0; pup < max_pup; pup++) {  in ddr3_set_pbs_results()
1430 		if (pup == (max_pup - 1) && dram_info->ecc_ena)  in ddr3_set_pbs_results()
1433 			phys_pup = pup;  in ddr3_set_pbs_results()
1442 			if (pbs_min > skew_array[(pup * DQ_NUM) + dq])  in ddr3_set_pbs_results()
1443 				pbs_min = skew_array[(pup * DQ_NUM) + dq];  in ddr3_set_pbs_results()
1445 			if (pbs_max < skew_array[(pup * DQ_NUM) + dq])  in ddr3_set_pbs_results()
1446 				pbs_max = skew_array[(pup * DQ_NUM) + dq];  in ddr3_set_pbs_results()
1457 		val[pup] = 0;  in ddr3_set_pbs_results()
1473 			DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] -  in ddr3_set_pbs_results()
1477 			idx = (pup * DQ_NUM) + dq;  in ddr3_set_pbs_results()
1489 				val[pup] += skew_array[idx] - pbs_min;  in ddr3_set_pbs_results()
1499 					   val[pup] / 8);  in ddr3_set_pbs_results()
1508 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay)  in ddr3_pbs_write_pup_dqs_reg()  argument
1512 	reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF);  in ddr3_pbs_write_pup_dqs_reg()
1516 	reg |= (pup << REG_PHY_PUP_OFFS);  in ddr3_pbs_write_pup_dqs_reg()