[PATCH 078/109] drm/amdgpu/ras: use flip-bit mask to generalize bad page address derivation

From: YiPeng Chai <[email protected]>

Use flip-bit mask to generalize bad page address derivation in a row.

Signed-off-by: YiPeng Chai <[email protected]>
Reviewed-by: Tao Zhou <[email protected]>
Signed-off-by: Alex Deucher <[email protected]>
---
 drivers/gpu/drm/amd/ras/core/eeprom.h        |   1 +
 drivers/gpu/drm/amd/ras/core/ras_umc.c       |  77 +++++++++++--
 drivers/gpu/drm/amd/ras/core/ras_umc.h       |   3 -
 drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c | 110 ++++++++++++++++---
 drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c |  49 ++++-----
 5 files changed, 181 insertions(+), 59 deletions(-)

diff --git a/drivers/gpu/drm/amd/ras/core/eeprom.h b/drivers/gpu/drm/amd/ras/core/eeprom.h
index ae22bb3f83187..bf721c2a11f1e 100644
--- a/drivers/gpu/drm/amd/ras/core/eeprom.h
+++ b/drivers/gpu/drm/amd/ras/core/eeprom.h
@@ -167,6 +167,7 @@ struct eeprom_umc_record {
 	 * used with cur_nps_retired_row_pfn to derive all bad page addresses
 	 */
 	uint64_t cur_nps_pa_flip_mask;
+	uint32_t cur_nps_valid_page_num;
 	uint32_t cur_nps_bank;
 	uint32_t cur_nps;
 	uint64_t ipid;
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc.c b/drivers/gpu/drm/amd/ras/core/ras_umc.c
index cef5770a75a60..c30d11942a72b 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc.c
@@ -177,19 +177,61 @@ int ras_umc_clear_logged_ecc(struct ras_core_context *ras_core)
 	return 0;
 }
 
+static int ras_umc_expand_row_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *page_pfns, uint32_t nr_page_pfns)
+{
+	uint64_t retired_addr = RAS_PFN_TO_ADDR(record->cur_nps_retired_row_pfn);
+	uint64_t flip_mask = record->cur_nps_pa_flip_mask;
+	uint64_t subset;
+	uint64_t row_pa, addr;
+	uint32_t count = 0;
+
+	if (!retired_addr || !flip_mask || !page_pfns || !nr_page_pfns)
+		return -ENOEXEC;
+
+	row_pa = retired_addr & ~(flip_mask);
+
+	if (count < nr_page_pfns)
+		page_pfns[count++] = RAS_ADDR_TO_PFN(row_pa);
+
+	subset = flip_mask;
+	while (subset) {
+		addr = row_pa ^ subset;
+
+		if (count >= nr_page_pfns)
+			break;
+
+		page_pfns[count++] = RAS_ADDR_TO_PFN(addr);
+
+		subset = (subset - 1) & flip_mask;
+	};
+
+	return count;
+}
+
 int ras_umc_convert_record_to_nps_pages(struct ras_core_context *ras_core,
 		struct eeprom_umc_record *record, uint32_t nps,
-		uint64_t *page_pfn, uint32_t max_pages)
+		uint64_t *page_pfns, uint32_t nr_page_pfns)
 {
-	int count = 0;
-	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	uint32_t new_nps;
+	int ret, count = 0;
 
-	if (!page_pfn || !max_pages)
+	if (!page_pfns || !nr_page_pfns || !record ||
+	    (record->cur_nps > UMC_MEMORY_PARTITION_MODE_NPS8))
 		return -EINVAL;
 
-	if (ras_umc->ip_func && ras_umc->ip_func->eeprom_record_to_nps_pages)
-		count = ras_umc->ip_func->eeprom_record_to_nps_pages(ras_core,
-					record, nps, page_pfn, max_pages);
+	if (!record->cur_nps || !record->cur_nps_retired_row_pfn ||
+	    !record->cur_nps_pa_flip_mask) {
+		new_nps = record->cur_nps ?
+			record->cur_nps : ras_core_get_curr_nps_mode(ras_core);
+		ret = ras_umc_record_to_nps_record(ras_core, record, new_nps);
+		if (ret)
+			return ret;
+	}
+
+	count = ras_umc_expand_row_pages(ras_core, record, page_pfns, nr_page_pfns);
+	if (count > 0)
+		record->cur_nps_valid_page_num = count;
 
 	return count;
 }
@@ -496,7 +538,7 @@ static bool ras_umc_check_retired_record(struct ras_core_context *ras_core,
 	int ret;
 
 	nps = ras_core_get_curr_nps_mode(ras_core);
-	ret = ras_umc_eeprom_rec2nps_rec(ras_core, record, nps);
+	ret = ras_umc_record_to_nps_record(ras_core, record, nps);
 	if (ret) {
 		RAS_DEV_ERR(ras_core->dev, "Failed to translate nps record! ret:%d\n", ret);
 		return true;
@@ -993,11 +1035,28 @@ int ras_umc_bank_to_umc_record(struct ras_core_context *ras_core,
 int ras_umc_record_to_nps_record(struct ras_core_context *ras_core,
 		struct eeprom_umc_record *record,  uint32_t nps)
 {
+	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	uint64_t ch_idx_v2;
+	uint32_t save_nps;
+
 	if (!record || !nps ||
 		(nps >= UMC_MEMORY_PARTITION_MODE_UNKNOWN))
 		return -EINVAL;
 
-	return ras_umc_eeprom_rec2nps_rec(ras_core, record, nps);
+	/* Avoid redundant conversion for the same NPS mode */
+	if ((record->cur_nps == nps) && record->cur_nps_retired_row_pfn &&
+	    record->cur_nps_pa_flip_mask)
+		return 0;
+
+	save_nps = EEPROM_RECORD_UMC_NPS_MODE(record);
+	ch_idx_v2 = record->retired_row_pfn & UMC_CHANNEL_IDX_V2;
+	if (!save_nps && !ch_idx_v2)
+		return ras_umc_eeprom_rec2nps_rec(ras_core, record, nps);
+
+	if (!ras_umc->ip_func || !ras_umc->ip_func->eeprom_record_to_nps_record)
+		return -EOPNOTSUPP;
+
+	return ras_umc->ip_func->eeprom_record_to_nps_record(ras_core, record, nps);
 }
 
 int ras_umc_dump_fw_records(struct ras_core_context *ras_core)
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc.h b/drivers/gpu/drm/amd/ras/core/ras_umc.h
index a64b5106d0051..b52480ba886b8 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc.h
@@ -128,9 +128,6 @@ struct umc_bank_addr {
 struct ras_umc_ip_func {
 	int (*bank_to_eeprom_record)(struct ras_core_context *ras_core,
 			struct ras_bank_ecc *bank, struct eeprom_umc_record *record);
-	int (*eeprom_record_to_nps_pages)(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *record, uint32_t nps,
-			uint64_t *pfns, uint32_t num);
 	int (*eeprom_record_to_nps_record)(struct ras_core_context *ras_core,
 			struct eeprom_umc_record *record, uint32_t nps);
 	int (*bank_to_soc_pa)(struct ras_core_context *ras_core,
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c b/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c
index 438270adcfcbb..37fa24acddd02 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c
@@ -148,21 +148,31 @@ static void __get_nps_pa_flip_bits(struct ras_core_context *ras_core,
 	}
 }
 
-static uint64_t umc_v12_0_nps_pa_to_row_pa(struct ras_core_context *ras_core,
-		uint64_t pa, enum umc_memory_partition_mode nps, bool zero_pfn_ok)
+static uint64_t __get_nps_pa_flip_mask(struct ras_core_context *ras_core,
+		enum umc_memory_partition_mode nps)
 {
 	struct umc_flip_bits flip_bits = {0};
-	uint64_t row_pa;
+	uint64_t flip_mask = 0;
 	int i;
 
 	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
 
-	row_pa = pa;
-	/* clear loop bits in soc physical address */
 	for (i = 0; i < flip_bits.bit_num; i++)
-		row_pa &= ~BIT_ULL(flip_bits.flip_bits_in_pa[i]);
+		flip_mask |= BIT_ULL(flip_bits.flip_bits_in_pa[i]);
+
+	return flip_mask;
+}
+
+static uint64_t umc_v12_0_nps_pa_to_row_pa(struct ras_core_context *ras_core,
+		uint64_t pa, enum umc_memory_partition_mode nps, bool zero_pfn_ok)
+{
+	struct umc_flip_bits flip_bits = {0};
+	uint64_t row_pa;
 
-	if (!zero_pfn_ok && !RAS_ADDR_TO_PFN(row_pa))
+	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
+	row_pa = pa & ~__get_nps_pa_flip_mask(ras_core, nps);
+
+	if (!zero_pfn_ok && !RAS_ADDR_TO_PFN(row_pa) && flip_bits.bit_num > 2)
 		row_pa |= BIT_ULL(flip_bits.flip_bits_in_pa[2]);
 
 	return row_pa;
@@ -176,13 +186,15 @@ static int lookup_bad_pages_in_a_row(struct ras_core_context *ras_core,
 	uint32_t col, col_lower, row, row_lower, idx, row_high;
 	uint64_t soc_pa, row_pa, column, err_addr;
 	uint64_t retired_addr = RAS_PFN_TO_ADDR(record->cur_nps_retired_row_pfn);
+	uint64_t flip_mask = 0;
 	struct umc_flip_bits flip_bits = {0};
 	uint32_t retire_unit;
 	uint32_t i;
 
 	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
 
-	row_pa = umc_v12_0_nps_pa_to_row_pa(ras_core, retired_addr, nps, true);
+	flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+	row_pa = retired_addr & ~(flip_mask);
 
 	err_addr = record->address;
 	/* get column bit 0 and 1 in mca address */
@@ -348,7 +360,24 @@ static int umc_v12_0_ma2pa(struct ras_core_context *ras_core,
 
 	return 0;
 }
+static int convert_ma_to_pa(struct ras_core_context *ras_core,
+			struct umc_mca_addr *addr_in, struct umc_phy_addr *addr_out,
+			uint32_t nps)
+{
+	int ret;
 
+	if (ras_psp_check_supported_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS))
+		ret = ras_umc_ras_ta_translate_addr(ras_core,
+				addr_in, addr_out, nps);
+	else
+		ret = umc_v12_0_ma2pa(ras_core,
+				addr_in, addr_out, nps);
+
+	if (!addr_out->pa_flip_mask)
+		addr_out->pa_flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+
+	return ret;
+}
 static int convert_bank_to_nps_addr(struct ras_core_context *ras_core,
 			struct ras_bank_ecc *bank, struct umc_phy_addr *pa_addr, uint32_t nps)
 {
@@ -365,10 +394,10 @@ static int convert_bank_to_nps_addr(struct ras_core_context *ras_core,
 	addr_in.node_inst = ACA_IPID_2_DIE_ID(bank->ipid);
 	addr_in.socket_id = ACA_IPID_2_SOCKET_ID(bank->ipid);
 
-	ret = ras_umc_ma2pa(ras_core, &addr_in, &addr_out, nps);
+	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
 	if (!ret) {
-		pa_addr->pa =
-			umc_v12_0_nps_pa_to_row_pa(ras_core, addr_out.pa, nps, false);
+		pa_addr->pa_flip_mask = addr_out.pa_flip_mask;
+		pa_addr->pa = addr_out.pa | pa_addr->pa_flip_mask;
 		pa_addr->channel_idx = addr_out.channel_idx;
 		pa_addr->bank = addr_out.bank;
 	}
@@ -408,12 +437,59 @@ static int umc_v12_0_bank_to_eeprom_record(struct ras_core_context *ras_core,
 	return 0;
 }
 
-static int umc_v12_0_eeprom_record_to_nps_pages(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *record, uint32_t nps,
-			uint64_t *pfns, uint32_t num)
+static int convert_eeprom_record_to_nps_addr(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *pa, uint64_t *pa_flip_mask, uint32_t nps)
 {
-	return lookup_bad_pages_in_a_row(ras_core,
-				record, nps, pfns, num, 0, false);
+	struct device_system_info dev_info = {0};
+	struct umc_mca_addr addr_in;
+	struct umc_phy_addr addr_out;
+	int ret;
+
+	memset(&addr_in, 0, sizeof(addr_in));
+	memset(&addr_out, 0, sizeof(addr_out));
+
+	ras_core_get_device_system_info(ras_core, &dev_info);
+
+	addr_in.err_addr = record->address;
+	addr_in.ch_inst = record->mem_channel;
+	addr_in.umc_inst = record->mcumc_id;
+	addr_in.node_inst = UMC_INV_AID_NODE;
+	addr_in.socket_id = dev_info.socket_id;
+
+	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
+	if (ret)
+		return ret;
+
+	*pa_flip_mask = addr_out.pa_flip_mask;
+	*pa = addr_out.pa | addr_out.pa_flip_mask;
+
+	return 0;
+}
+
+static int umc_v12_0_eeprom_record_to_nps_record(struct ras_core_context *ras_core,
+				struct eeprom_umc_record *record, uint32_t nps)
+{
+	uint64_t pa = 0, flip_mask = 0;
+	uint64_t row_pfn;
+	int ret = 0;
+
+	if (nps == EEPROM_RECORD_UMC_NPS_MODE(record)) {
+		record->cur_nps_pa_flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+		row_pfn = EEPROM_RECORD_UMC_ADDR_PFN(record);
+		record->cur_nps_retired_row_pfn =
+			RAS_ADDR_TO_PFN(RAS_PFN_TO_ADDR(row_pfn) | record->cur_nps_pa_flip_mask);
+	} else {
+		ret = convert_eeprom_record_to_nps_addr(ras_core,
+				record, &pa, &flip_mask, nps);
+		if (!ret) {
+			record->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(pa);
+			record->cur_nps_pa_flip_mask = flip_mask;
+		}
+	}
+
+	record->cur_nps = nps;
+
+	return ret;
 }
 
 static int umc_12_0_soc_pa_to_bank(struct ras_core_context *ras_core,
@@ -527,7 +603,7 @@ static uint32_t umc_v12_0_get_die_id(uint64_t mca_addr, uint64_t pa)
 
 const struct ras_umc_ip_func ras_umc_func_v12_0 = {
 	.bank_to_eeprom_record = umc_v12_0_bank_to_eeprom_record,
-	.eeprom_record_to_nps_pages = umc_v12_0_eeprom_record_to_nps_pages,
+	.eeprom_record_to_nps_record = umc_v12_0_eeprom_record_to_nps_record,
 	.bank_to_soc_pa = umc_12_0_bank_to_soc_pa,
 	.soc_pa_to_bank = umc_12_0_soc_pa_to_bank,
 	.mca_ipid_parse = umc_v12_0_mca_ipid_parse,
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c b/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c
index 39e95a5515f83..8f3bba2621f89 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c
@@ -33,27 +33,19 @@ static void __get_nps_pa_flip_bits(struct ras_core_context *ras_core,
 
 }
 
-static uint64_t  convert_nps_pa_to_row_pa(struct ras_core_context *ras_core,
-	struct umc_phy_addr *pa, enum umc_memory_partition_mode nps, bool zero_pfn_ok)
+static uint64_t  __get_nps_pa_flip_mask(struct ras_core_context *ras_core,
+		enum umc_memory_partition_mode nps)
 {
 	struct umc_flip_bits flip_bits = {0};
-	uint64_t row_pa;
+	uint64_t flip_mask = 0;
 	int i;
 
-	if (pa->pa_flip_mask)
-		return zero_pfn_ok ? (pa->pa & ~pa->pa_flip_mask) : pa->pa;
-
 	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
 
-	row_pa = pa->pa;
-	/* clear loop bits in soc physical address */
 	for (i = 0; i < flip_bits.bit_num; i++)
-		row_pa &= ~BIT_ULL(flip_bits.flip_bits_in_pa[i]);
-
-	if (!zero_pfn_ok && !RAS_ADDR_TO_PFN(row_pa))
-		row_pa |= BIT_ULL(flip_bits.flip_bits_in_pa[2]);
+		flip_mask |= BIT_ULL(flip_bits.flip_bits_in_pa[i]);
 
-	return row_pa;
+	return flip_mask;
 }
 
 static int lookup_bad_pages_in_a_row(struct ras_core_context *ras_core,
@@ -113,9 +105,8 @@ static int convert_bank_to_nps_addr(struct ras_core_context *ras_core,
 
 	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
 	if (!ret) {
-		pa_addr->pa =
-			convert_nps_pa_to_row_pa(ras_core, &addr_out, nps, false);
 		pa_addr->pa_flip_mask = addr_out.pa_flip_mask;
+		pa_addr->pa = addr_out.pa | pa_addr->pa_flip_mask;
 		pa_addr->channel_idx = addr_out.channel_idx;
 		pa_addr->bank = addr_out.bank;
 	}
@@ -153,7 +144,7 @@ static int umc_v15_0_bank_to_eeprom_record(struct ras_core_context *ras_core,
 }
 
 static int convert_eeprom_record_to_nps_addr(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *record, uint64_t *pa, uint32_t nps)
+	struct eeprom_umc_record *record, uint64_t *pa, uint64_t *pa_flip_mask, uint32_t nps)
 {
 	struct device_system_info dev_info = {0};
 	struct umc_mca_addr addr_in;
@@ -175,7 +166,8 @@ static int convert_eeprom_record_to_nps_addr(struct ras_core_context *ras_core,
 	if (ret)
 		return ret;
 
-	*pa = convert_nps_pa_to_row_pa(ras_core, &addr_out, nps, false);
+	*pa_flip_mask = addr_out.pa_flip_mask;
+	*pa = addr_out.pa | addr_out.pa_flip_mask;
 
 	return 0;
 }
@@ -183,16 +175,22 @@ static int convert_eeprom_record_to_nps_addr(struct ras_core_context *ras_core,
 static int umc_v15_0_eeprom_record_to_nps_record(struct ras_core_context *ras_core,
 				struct eeprom_umc_record *record, uint32_t nps)
 {
-	uint64_t pa = 0;
+	uint64_t pa = 0, flip_mask = 0;
+	uint64_t row_pfn;
 	int ret = 0;
 
 	if (nps == EEPROM_RECORD_UMC_NPS_MODE(record)) {
-		record->cur_nps_retired_row_pfn = EEPROM_RECORD_UMC_ADDR_PFN(record);
+		record->cur_nps_pa_flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+		row_pfn = EEPROM_RECORD_UMC_ADDR_PFN(record);
+		record->cur_nps_retired_row_pfn =
+			RAS_ADDR_TO_PFN(RAS_PFN_TO_ADDR(row_pfn) | record->cur_nps_pa_flip_mask);
 	} else {
 		ret = convert_eeprom_record_to_nps_addr(ras_core,
-				record, &pa, nps);
-		if (!ret)
+				record, &pa, &flip_mask, nps);
+		if (!ret) {
 			record->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(pa);
+			record->cur_nps_pa_flip_mask = flip_mask;
+		}
 	}
 
 	record->cur_nps = nps;
@@ -200,14 +198,6 @@ static int umc_v15_0_eeprom_record_to_nps_record(struct ras_core_context *ras_co
 	return ret;
 }
 
-static int umc_v15_0_eeprom_record_to_nps_pages(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *record, uint32_t nps,
-			uint64_t *pfns, uint32_t num)
-{
-	return lookup_bad_pages_in_a_row(ras_core,
-				record, nps, pfns, num, 0, false);
-}
-
 static int umc_v15_0_soc_pa_to_bank(struct ras_core_context *ras_core,
 			uint64_t soc_pa,
 			struct umc_bank_addr *bank_addr)
@@ -225,7 +215,6 @@ static int umc_v15_0_bank_to_soc_pa(struct ras_core_context *ras_core,
 const struct ras_umc_ip_func ras_umc_func_v15_0 = {
 	.bank_to_eeprom_record = umc_v15_0_bank_to_eeprom_record,
 	.eeprom_record_to_nps_record = umc_v15_0_eeprom_record_to_nps_record,
-	.eeprom_record_to_nps_pages = umc_v15_0_eeprom_record_to_nps_pages,
 	.bank_to_soc_pa = umc_v15_0_bank_to_soc_pa,
 	.soc_pa_to_bank = umc_v15_0_soc_pa_to_bank,
 };
-- 
2.55.0