292 #define LMC_DEBUGFS_ENT(_field)	DEBUGFS_FIELD_ATTR(lmc, _field)
307 	struct thunderx_lmc *lmc = file->private_data;  in thunderx_lmc_inject_int_write()  local
315 		writeq(val, lmc->regs + LMC_INT_W1S);  in thunderx_lmc_inject_int_write()
326 	struct thunderx_lmc *lmc = file->private_data;  in thunderx_lmc_int_read()  local
328 	u64 lmc_int = readq(lmc->regs + LMC_INT);  in thunderx_lmc_int_read()
338 	struct thunderx_lmc *lmc = arg;  in inject_ecc_fn()  local
344 	addr = (uintptr_t)page_address(lmc->mem);  in inject_ecc_fn()
345 	phys = (uintptr_t)page_to_phys(lmc->mem);  in inject_ecc_fn()
348 	lmc->parity_test &= ~(7ULL << 8);  in inject_ecc_fn()
349 	lmc->parity_test |= (cl_idx << 8);  in inject_ecc_fn()
351 	writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);  in inject_ecc_fn()
352 	writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);  in inject_ecc_fn()
353 	writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);  in inject_ecc_fn()
355 	readq(lmc->regs + LMC_CHAR_MASK0);  in inject_ecc_fn()
356 	readq(lmc->regs + LMC_CHAR_MASK2);  in inject_ecc_fn()
357 	readq(lmc->regs + LMC_ECC_PARITY_TEST);  in inject_ecc_fn()
410 	struct thunderx_lmc *lmc = file->private_data;  in thunderx_lmc_inject_ecc_write()  local
416 	atomic_set(&lmc->ecc_int, 0);  in thunderx_lmc_inject_ecc_write()
418 	lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);  in thunderx_lmc_inject_ecc_write()
419 	if (!lmc->mem)  in thunderx_lmc_inject_ecc_write()
424 		__free_pages(lmc->mem, 0);  in thunderx_lmc_inject_ecc_write()
428 	addr = page_address(lmc->mem);  in thunderx_lmc_inject_ecc_write()
430 	while (!atomic_read(&lmc->ecc_int) && timeout--) {  in thunderx_lmc_inject_ecc_write()
431 		stop_machine(inject_ecc_fn, lmc, NULL);  in thunderx_lmc_inject_ecc_write()
444 	__free_pages(lmc->mem, 0);  in thunderx_lmc_inject_ecc_write()
491 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)  in thunderx_faddr_to_phys()  argument
496 	addr |= lmc->node << 40;  in thunderx_faddr_to_phys()
497 	addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;  in thunderx_faddr_to_phys()
498 	addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;  in thunderx_faddr_to_phys()
499 	addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;  in thunderx_faddr_to_phys()
500 	addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;  in thunderx_faddr_to_phys()
502 	bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;  in thunderx_faddr_to_phys()
504 	if (lmc->xor_bank)  in thunderx_faddr_to_phys()
505 		bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);  in thunderx_faddr_to_phys()
507 	addr |= bank << lmc->bank_lsb;  in thunderx_faddr_to_phys()
509 	xbits = PCI_FUNC(lmc->pdev->devfn);  in thunderx_faddr_to_phys()
511 	if (lmc->l2c_alias)  in thunderx_faddr_to_phys()
512 		xbits ^= get_bits(addr, 20, lmc->xbits) ^  in thunderx_faddr_to_phys()
513 			 get_bits(addr, 12, lmc->xbits);  in thunderx_faddr_to_phys()
548 	struct thunderx_lmc *lmc = mci->pvt_info;  in thunderx_lmc_err_isr()  local
550 	unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));  in thunderx_lmc_err_isr()
551 	struct lmc_err_ctx *ctx = &lmc->err_ctx[head];  in thunderx_lmc_err_isr()
553 	writeq(0, lmc->regs + LMC_CHAR_MASK0);  in thunderx_lmc_err_isr()
554 	writeq(0, lmc->regs + LMC_CHAR_MASK2);  in thunderx_lmc_err_isr()
555 	writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);  in thunderx_lmc_err_isr()
557 	ctx->reg_int = readq(lmc->regs + LMC_INT);  in thunderx_lmc_err_isr()
558 	ctx->reg_fadr = readq(lmc->regs + LMC_FADR);  in thunderx_lmc_err_isr()
559 	ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);  in thunderx_lmc_err_isr()
560 	ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);  in thunderx_lmc_err_isr()
561 	ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);  in thunderx_lmc_err_isr()
563 	lmc->ring_head++;  in thunderx_lmc_err_isr()
565 	atomic_set(&lmc->ecc_int, 1);  in thunderx_lmc_err_isr()
568 	writeq(ctx->reg_int, lmc->regs + LMC_INT);  in thunderx_lmc_err_isr()
576 	struct thunderx_lmc *lmc = mci->pvt_info;  in thunderx_lmc_threaded_isr()  local
593 	while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,  in thunderx_lmc_threaded_isr()
594 		ARRAY_SIZE(lmc->err_ctx))) {  in thunderx_lmc_threaded_isr()
595 		tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));  in thunderx_lmc_threaded_isr()
597 		ctx = &lmc->err_ctx[tail];  in thunderx_lmc_threaded_isr()
599 		dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",  in thunderx_lmc_threaded_isr()
601 		dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",  in thunderx_lmc_threaded_isr()
603 		dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",  in thunderx_lmc_threaded_isr()
605 		dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",  in thunderx_lmc_threaded_isr()
607 		dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",  in thunderx_lmc_threaded_isr()
621 		phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);  in thunderx_lmc_threaded_isr()
634 		lmc->ring_tail++;  in thunderx_lmc_threaded_isr()
664 	struct thunderx_lmc *lmc;  in thunderx_lmc_probe()  local
694 	lmc = mci->pvt_info;  in thunderx_lmc_probe()
698 	lmc->regs = pcim_iomap_table(pdev)[0];  in thunderx_lmc_probe()
700 	lmc_control = readq(lmc->regs + LMC_CONTROL);  in thunderx_lmc_probe()
701 	lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);  in thunderx_lmc_probe()
702 	lmc_config = readq(lmc->regs + LMC_CONFIG);  in thunderx_lmc_probe()
722 	lmc->pdev = pdev;  in thunderx_lmc_probe()
723 	lmc->msix_ent.entry = 0;  in thunderx_lmc_probe()
725 	lmc->ring_head = 0;  in thunderx_lmc_probe()
726 	lmc->ring_tail = 0;  in thunderx_lmc_probe()
728 	ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);  in thunderx_lmc_probe()
734 	ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,  in thunderx_lmc_probe()
743 	lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));  in thunderx_lmc_probe()
745 	lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;  in thunderx_lmc_probe()
746 	lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&  in thunderx_lmc_probe()
749 	lmc->pbank_lsb = (lmc_config >> 5) & 0xf;  in thunderx_lmc_probe()
750 	lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;  in thunderx_lmc_probe()
751 	lmc->rank_lsb = lmc->dimm_lsb;  in thunderx_lmc_probe()
752 	lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;  in thunderx_lmc_probe()
753 	lmc->bank_lsb = 7 + lmc->xbits;  in thunderx_lmc_probe()
754 	lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;  in thunderx_lmc_probe()
756 	lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;  in thunderx_lmc_probe()
758 	lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;  in thunderx_lmc_probe()
760 	l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);  in thunderx_lmc_probe()
767 	lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);  in thunderx_lmc_probe()
777 	lmc_int = readq(lmc->regs + LMC_INT);  in thunderx_lmc_probe()
778 	writeq(lmc_int, lmc->regs + LMC_INT);  in thunderx_lmc_probe()
780 	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);  in thunderx_lmc_probe()
785 						    lmc,  in thunderx_lmc_probe()
806 	struct thunderx_lmc *lmc = mci->pvt_info;  in thunderx_lmc_remove()  local
808 	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);  in thunderx_lmc_remove()