[PATCH 10/15] PCI: endpoint: pci-epf-vntb: Support DMA export and shared BAR layouts

Teach pci-epf-vntb to publish a versioned control layout and an optional
exported DMA slice.

When the configuration still matches the historical one-MW-per-BAR
layout and DMA export is disabled, keep emitting the legacy control
block. Otherwise emit control-layout v1, which adds per-MW offset/size
tuples and a DMA locator for the exported slice. Add configfs knobs for
mwN_offset, dma_bar, dma_offset, and dma_num_chans, and use
pci_epf_alloc_dma() to prepare the DMA export.

Also add BAR planning code so memory windows and the DMA slice can share
a physical BAR. Shared layouts are programmed in two stages: install a
temporary whole-BAR mapping at bind time, then switch to the final
subrange map on the first host command once BAR addresses are valid.

Signed-off-by: Koichiro Den <[email protected]>
---
 drivers/pci/endpoint/functions/pci-epf-vntb.c | 760 ++++++++++++++++--
 1 file changed, 707 insertions(+), 53 deletions(-)

diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index 16656659a9ce..d493d64dca72 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -41,7 +41,9 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/sort.h>
 
+#include <linux/pci-ep-dma.h>
 #include <linux/pci-ep-msi.h>
 #include <linux/pci-epc.h>
 #include <linux/pci-epf.h>
@@ -69,6 +71,9 @@ static struct workqueue_struct *kpcintb_workqueue;
 #define MAX_DB_COUNT			32
 #define MAX_MW				4
 
+#define NTB_EPF_CTRL_VERSION_LEGACY	0
+#define NTB_EPF_CTRL_VERSION_V1		1
+
 /* Limit per-work execution to avoid monopolizing kworker on doorbell storms. */
 #define VNTB_PEER_DB_WORK_BUDGET	5
 
@@ -79,6 +84,7 @@ enum epf_ntb_bar {
 	BAR_MW2,
 	BAR_MW3,
 	BAR_MW4,
+	BAR_DMA,
 	VNTB_BAR_NUM,
 };
 
@@ -116,22 +122,41 @@ struct epf_ntb_ctrl {
 	u32 argument;
 	u16 command_status;
 	u16 link_status;
-	u32 topology;
+	u32 ctrl_version;
 	u64 addr;
 	u64 size;
 	u32 num_mws;
-	u32 reserved;
+	u32 mw1_offset;
 	u32 spad_offset;
 	u32 spad_count;
 	u32 db_entry_size;
 	u32 db_data[MAX_DB_COUNT];
 	u32 db_offset[MAX_DB_COUNT];
+	u32 mw_offset[MAX_MW];
+	u32 mw_size[MAX_MW];
+	u32 dma_abi;
+	u32 dma_bar;
+	u32 dma_offset;
+	u32 dma_size;
+	u32 dma_num_chans;
 } __packed;
 
 struct epf_ntb_mw {
 	u64 size;
+	u32 offset;
 	phys_addr_t vpci_mw_phys;
 	void __iomem *vpci_mw_addr;
+	dma_addr_t bar_phys;
+};
+
+struct epf_ntb_bar_plan {
+	bool staged;
+	bool deferred;
+	bool active;
+	unsigned int region_count;
+	size_t size;
+	phys_addr_t stage_phys;
+	void __iomem *stage_addr;
 };
 
 struct epf_ntb {
@@ -143,10 +168,14 @@ struct epf_ntb {
 	u32 db_count;
 	u32 spad_count;
 	struct epf_ntb_mw mw[MAX_MW];
+	struct epf_ntb_bar_plan bar_plan[PCI_STD_NUM_BARS];
 	atomic64_t db;
 	atomic64_t peer_db_pending;
 	struct work_struct peer_db_work;
 	u32 vbus_number;
+	u32 dma_offset;
+	u32 dma_num_chans;
+	const struct pci_epc_features *epc_features;
 	u16 vntb_pid;
 	u16 vntb_vid;
 
@@ -162,6 +191,7 @@ struct epf_ntb {
 	enum pci_barno epf_ntb_bar[VNTB_BAR_NUM];
 
 	struct epf_ntb_ctrl *reg;
+	struct pci_ep_dma *dma;
 
 	u32 *epf_db;
 
@@ -260,6 +290,258 @@ static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
 			   ntb->mw[mw].vpci_mw_phys);
 }
 
+static bool epf_ntb_bar_valid(enum pci_barno barno)
+{
+	return barno >= BAR_0 && barno <= BAR_5;
+}
+
+/*
+ * Legacy layout has no per-MW size array and no DMA locator. Only emit it
+ * when each MW owns a distinct BAR, all MWs after MW0 start at offset 0,
+ * and no DMA slice is exported, so older ntb_hw_epf can keep inferring
+ * the layout from BAR assignments alone.
+ */
+static bool epf_ntb_ctrl_layout_is_legacy(const struct epf_ntb *ntb)
+{
+	enum pci_barno barno;
+	u8 used_mw_bars = 0;
+	int i;
+
+	if (ntb->dma)
+		return false;
+
+	for (i = 0; i < ntb->num_mws; i++) {
+		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
+		if (epf_ntb_bar_valid(barno)) {
+			if (used_mw_bars & BIT(barno))
+				return false;
+			used_mw_bars |= BIT(barno);
+		}
+
+		if (i > 0 && ntb->mw[i].offset)
+			return false;
+	}
+
+	return true;
+}
+
+static int epf_ntb_validate_ctrl_layout_v1(const struct epf_ntb *ntb)
+{
+	struct device *dev = &ntb->epf->dev;
+	int i;
+
+	if (epf_ntb_ctrl_layout_is_legacy(ntb))
+		return 0;
+
+	for (i = 0; i < ntb->num_mws; i++) {
+		if (ntb->mw[i].size > U32_MAX) {
+			dev_err(dev,
+				"MW%d size %#llx exceeds control ABI v1 limit\n",
+				i + 1,
+				(unsigned long long)ntb->mw[i].size);
+			return -E2BIG;
+		}
+	}
+
+	return 0;
+}
+
+static dma_addr_t epf_ntb_mw_active_phys(const struct epf_ntb *ntb, int idx)
+{
+	if (ntb->mw[idx].bar_phys)
+		return ntb->mw[idx].bar_phys;
+
+	return ntb->mw[idx].vpci_mw_phys;
+}
+
+struct epf_ntb_bar_region {
+	u32 offset;
+	dma_addr_t phys_addr;
+	size_t size;
+};
+
+static int epf_ntb_bar_region_cmp(const void *a, const void *b)
+{
+	const struct epf_ntb_bar_region *ra = a;
+	const struct epf_ntb_bar_region *rb = b;
+
+	if (ra->offset < rb->offset)
+		return -1;
+
+	return ra->offset > rb->offset;
+}
+
+static int
+epf_ntb_add_bar_region(struct epf_ntb_bar_region *regions,
+		       unsigned int *count, u32 offset, dma_addr_t phys_addr,
+		       size_t size, bool needs_phys_addr)
+{
+	if (needs_phys_addr && !phys_addr)
+		return -EINVAL;
+
+	regions[*count] = (struct epf_ntb_bar_region) {
+		.offset = offset,
+		.phys_addr = phys_addr,
+		.size = size,
+	};
+	(*count)++;
+
+	return 0;
+}
+
+static int
+epf_ntb_collect_bar_regions(struct epf_ntb *ntb, enum pci_barno barno,
+			    struct pci_epf_bar_submap *submap,
+			    unsigned int *nregions, size_t *total)
+{
+	struct epf_ntb_bar_region regions[MAX_MW + PCI_EP_DMA_MAX_REGIONS];
+	const struct pci_ep_dma_region *dma_regions;
+	struct device *dev = &ntb->epf->dev;
+	bool needs_phys_addr = !!submap;
+	unsigned int count = 0;
+	size_t size_total = 0;
+	int i, ret;
+
+	if (ntb->dma && ntb->epf_ntb_bar[BAR_DMA] == barno) {
+		dma_regions = pci_epf_get_dma_regions(ntb->dma);
+		for (i = 0; i < pci_epf_get_dma_region_count(ntb->dma); i++) {
+			ret = epf_ntb_add_bar_region(regions, &count,
+						     dma_regions[i].offset,
+						     dma_regions[i].phys_addr,
+						     dma_regions[i].size,
+						     needs_phys_addr);
+			if (ret)
+				return ret;
+		}
+	}
+
+	for (i = 0; i < ntb->num_mws; i++) {
+		if (ntb->epf_ntb_bar[BAR_MW1 + i] != barno)
+			continue;
+
+		ret = epf_ntb_add_bar_region(regions, &count, ntb->mw[i].offset,
+					     needs_phys_addr ?
+					     epf_ntb_mw_active_phys(ntb, i) : 0,
+					     ntb->mw[i].size, needs_phys_addr);
+		if (ret)
+			return ret;
+	}
+
+	if (!count)
+		return -EINVAL;
+
+	sort(regions, count, sizeof(regions[0]), epf_ntb_bar_region_cmp, NULL);
+
+	for (i = 0; i < count; i++) {
+		if (!regions[i].size || regions[i].offset != size_total)
+			return -EINVAL;
+		if (submap) {
+			submap[i].phys_addr = regions[i].phys_addr;
+			submap[i].size = regions[i].size;
+		}
+		size_total += regions[i].size;
+	}
+
+	if (!is_power_of_2(size_total)) {
+		dev_err(dev, "Invalid total size: %#lx\n", size_total);
+		return -EINVAL;
+	}
+
+	*nregions = count;
+	*total = size_total;
+
+	return 0;
+}
+
+static int epf_ntb_bar_activate(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	struct epf_ntb_bar_plan *plan = &ntb->bar_plan[barno];
+	struct pci_epf_bar_submap *submap = NULL, *old_submap;
+	struct pci_epf_bar *epf_bar = &ntb->epf->bar[barno];
+	unsigned int nregions, old_nsub;
+	phys_addr_t old_phys;
+	size_t total;
+	int ret;
+
+	if (!plan->staged || !plan->deferred)
+		return 0;
+
+	submap = kcalloc(plan->region_count, sizeof(*submap), GFP_KERNEL);
+	if (!submap)
+		return -ENOMEM;
+
+	ret = epf_ntb_collect_bar_regions(ntb, barno, submap, &nregions, &total);
+	if (ret)
+		goto err_free;
+	if (nregions != plan->region_count || total != plan->size) {
+		ret = -EINVAL;
+		goto err_free;
+	}
+
+	old_phys = epf_bar->phys_addr;
+	old_submap = epf_bar->submap;
+	old_nsub = epf_bar->num_submap;
+
+	epf_bar->phys_addr = 0;
+	epf_bar->submap = submap;
+	epf_bar->num_submap = nregions;
+	epf_bar->size = total;
+
+	ret = pci_epc_set_bar(ntb->epf->epc,
+			      ntb->epf->func_no,
+			      ntb->epf->vfunc_no,
+			      epf_bar);
+	if (ret) {
+		epf_bar->phys_addr = old_phys;
+		epf_bar->submap = old_submap;
+		epf_bar->num_submap = old_nsub;
+		goto err_free;
+	}
+
+	if (plan->stage_addr) {
+		pci_epc_mem_free_addr(ntb->epf->epc, plan->stage_phys,
+				      plan->stage_addr, plan->size);
+		plan->stage_addr = NULL;
+		plan->stage_phys = 0;
+	}
+	kfree(old_submap);
+	plan->active = true;
+	return 0;
+
+err_free:
+	kfree(submap);
+	return ret;
+}
+
+static int epf_ntb_bar_activate_deferred(struct epf_ntb *ntb)
+{
+	bool done[PCI_STD_NUM_BARS] = { };
+	enum pci_barno barno;
+	int i, ret;
+
+	for (i = 0; i < ntb->num_mws; i++) {
+		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
+		if (!epf_ntb_bar_valid(barno) || done[barno])
+			continue;
+		if (!ntb->bar_plan[barno].active) {
+			ret = epf_ntb_bar_activate(ntb, barno);
+			if (ret)
+				return ret;
+		}
+		done[barno] = true;
+	}
+
+	barno = ntb->epf_ntb_bar[BAR_DMA];
+	if (epf_ntb_bar_valid(barno) && !done[barno] &&
+	    !ntb->bar_plan[barno].active) {
+		ret = epf_ntb_bar_activate(ntb, barno);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 /**
  * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
  * @work: work_struct for the epf_ntb_epc
@@ -309,6 +591,11 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
 		ctrl->command_status = COMMAND_STATUS_OK;
 		break;
 	case COMMAND_CONFIGURE_MW:
+		ret = epf_ntb_bar_activate_deferred(ntb);
+		if (ret < 0) {
+			ctrl->command_status = COMMAND_STATUS_ERROR;
+			break;
+		}
 		ret = epf_ntb_configure_mw(ntb, argument);
 		if (ret < 0)
 			ctrl->command_status = COMMAND_STATUS_ERROR;
@@ -320,6 +607,11 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
 		ctrl->command_status = COMMAND_STATUS_OK;
 		break;
 	case COMMAND_LINK_UP:
+		ret = epf_ntb_bar_activate_deferred(ntb);
+		if (ret < 0) {
+			ctrl->command_status = COMMAND_STATUS_ERROR;
+			goto reset_handler;
+		}
 		ntb->linkup = true;
 		ret = epf_ntb_link_up(ntb, true);
 		if (ret < 0)
@@ -476,8 +768,35 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
 	ctrl = ntb->reg;
 	ctrl->spad_offset = ctrl_size;
 
+	ctrl->ctrl_version = epf_ntb_ctrl_layout_is_legacy(ntb) ?
+		NTB_EPF_CTRL_VERSION_LEGACY : NTB_EPF_CTRL_VERSION_V1;
 	ctrl->spad_count = spad_count;
 	ctrl->num_mws = ntb->num_mws;
+	ctrl->mw1_offset = ntb->num_mws ? ntb->mw[0].offset : 0;
+
+	if (ctrl->ctrl_version >= NTB_EPF_CTRL_VERSION_V1) {
+		for (i = 0; i < ntb->num_mws; i++) {
+			ctrl->mw_offset[i] = ntb->mw[i].offset;
+			ctrl->mw_size[i] = ntb->mw[i].size;
+		}
+
+		if (ntb->dma) {
+			const struct pci_ep_dma_locator *loc =
+				pci_epf_get_dma_locator(ntb->dma);
+
+			ctrl->dma_abi = loc->abi;
+			ctrl->dma_bar = loc->bar;
+			ctrl->dma_offset = loc->offset;
+			ctrl->dma_size = loc->size;
+			ctrl->dma_num_chans = ntb->dma->num_chans;
+		} else {
+			ctrl->dma_abi = 0;
+			ctrl->dma_bar = NO_BAR;
+			ctrl->dma_offset = 0;
+			ctrl->dma_size = 0;
+			ctrl->dma_num_chans = 0;
+		}
+	}
 	ntb->spad_size = spad_size;
 
 	ctrl->db_entry_size = sizeof(u32);
@@ -509,6 +828,7 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
 	dev = &ntb->epf->dev;
 
 	epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
+	ntb->epc_features = epc_features;
 
 	if (!(epc_features->msix_capable || epc_features->msi_capable)) {
 		dev_err(dev, "MSI or MSI-X is required for doorbell\n");
@@ -721,6 +1041,35 @@ static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
 
 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
 
+static void epf_ntb_dma_cleanup(struct epf_ntb *ntb)
+{
+	if (!ntb->dma)
+		return;
+
+	pci_epf_free_dma(ntb->dma);
+	ntb->dma = NULL;
+}
+
+static int epf_ntb_dma_prepare(struct epf_ntb *ntb)
+{
+	enum pci_barno barno;
+
+	barno = ntb->epf_ntb_bar[BAR_DMA];
+	if (barno == NO_BAR)
+		return 0;
+
+	ntb->dma = pci_epf_alloc_dma(ntb->epf, barno, ntb->dma_offset,
+				     ntb->dma_num_chans ?: 1);
+	if (IS_ERR(ntb->dma)) {
+		int ret = PTR_ERR(ntb->dma);
+
+		ntb->dma = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
 /**
  * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
  *   allocated in peer's outbound address space
@@ -751,6 +1100,169 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
 			  &ntb->epf->bar[barno]);
 }
 
+static int
+epf_ntb_validate_one_bar_layout(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	unsigned int count;
+	size_t total;
+	int ret;
+
+	ret = epf_ntb_collect_bar_regions(ntb, barno, NULL, &count, &total);
+	if (ret)
+		return ret;
+
+	if (count <= 1 ||
+	    (ntb->epc_features->subrange_mapping &&
+	     ntb->epc_features->dynamic_inbound_mapping))
+		return 0;
+
+	dev_err(&ntb->epf->dev,
+		"BAR%d requires %u regions but subrange mapping unsupported\n",
+		barno, count);
+
+	return -EOPNOTSUPP;
+}
+
+static int epf_ntb_validate_bar_layout(struct epf_ntb *ntb)
+{
+	bool checked[PCI_STD_NUM_BARS] = { };
+	enum pci_barno barno;
+	int i, ret;
+
+	for (i = 0; i < ntb->num_mws; i++) {
+		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
+		if (!epf_ntb_bar_valid(barno) || checked[barno])
+			continue;
+
+		ret = epf_ntb_validate_one_bar_layout(ntb, barno);
+		if (ret)
+			return ret;
+
+		checked[barno] = true;
+	}
+
+	barno = ntb->epf_ntb_bar[BAR_DMA];
+	if (!epf_ntb_bar_valid(barno) || checked[barno])
+		return 0;
+
+	return epf_ntb_validate_one_bar_layout(ntb, barno);
+}
+
+/*
+ * Shared MW/DMA BARs are programmed in two stages. At bind time the host
+ * has not assigned BAR addresses yet, so install a temporary whole-BAR
+ * mapping first. Once the first host command arrives and BAR addresses
+ * are valid, replace it with the final subrange layout.
+ */
+static int epf_ntb_bar_stage1_program(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	struct pci_epf_bar_submap regions[MAX_MW + PCI_EP_DMA_MAX_REGIONS];
+	struct epf_ntb_bar_plan *plan = &ntb->bar_plan[barno];
+	struct pci_epf_bar *epf_bar = &ntb->epf->bar[barno];
+	unsigned int nregions;
+	size_t total;
+	int ret;
+
+	ret = epf_ntb_collect_bar_regions(ntb, barno, regions, &nregions, &total);
+	if (ret)
+		return ret;
+
+	plan->region_count = nregions;
+	plan->size = total;
+	plan->deferred = nregions > 1;
+	plan->active = !plan->deferred;
+
+	if (plan->deferred) {
+		plan->stage_addr = pci_epc_mem_alloc_addr(ntb->epf->epc,
+							  &plan->stage_phys,
+							  total);
+		if (!plan->stage_addr)
+			return -ENOMEM;
+	}
+
+	epf_bar->barno = barno;
+	epf_bar->size = total;
+	epf_bar->addr = NULL;
+	epf_bar->flags = upper_32_bits(total) ?
+		PCI_BASE_ADDRESS_MEM_TYPE_64 : PCI_BASE_ADDRESS_MEM_TYPE_32;
+	epf_bar->submap = NULL;
+	epf_bar->num_submap = 0;
+	epf_bar->phys_addr = plan->deferred ? plan->stage_phys : regions[0].phys_addr;
+
+	ret = pci_epc_set_bar(ntb->epf->epc,
+			      ntb->epf->func_no,
+			      ntb->epf->vfunc_no,
+			      epf_bar);
+	if (ret) {
+		if (plan->stage_addr) {
+			pci_epc_mem_free_addr(ntb->epf->epc, plan->stage_phys,
+					      plan->stage_addr, total);
+			plan->stage_addr = NULL;
+			plan->stage_phys = 0;
+		}
+		return ret;
+	}
+
+	plan->staged = true;
+	return 0;
+}
+
+static int epf_ntb_bar_refresh(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	struct epf_ntb_bar_plan *plan = &ntb->bar_plan[barno];
+	struct pci_epf_bar *epf_bar = &ntb->epf->bar[barno];
+	struct pci_epf_bar_submap region;
+	unsigned int nregions;
+	size_t total;
+	int ret;
+
+	if (!plan->staged)
+		return 0;
+
+	if (plan->deferred)
+		return plan->active ? epf_ntb_bar_activate(ntb, barno) : 0;
+
+	ret = epf_ntb_collect_bar_regions(ntb, barno, &region, &nregions, &total);
+	if (ret)
+		return ret;
+	if (nregions != 1 || total != plan->size)
+		return -EINVAL;
+
+	epf_bar->phys_addr = region.phys_addr;
+	epf_bar->submap = NULL;
+	epf_bar->num_submap = 0;
+	epf_bar->size = total;
+
+	return pci_epc_set_bar(ntb->epf->epc,
+			       ntb->epf->func_no,
+			       ntb->epf->vfunc_no,
+			       epf_bar);
+}
+
+static void epf_ntb_mw_bar_release(struct epf_ntb *ntb, enum pci_barno barno)
+{
+	struct epf_ntb_bar_plan *plan = &ntb->bar_plan[barno];
+	struct pci_epf_bar *epf_bar = &ntb->epf->bar[barno];
+
+	if (!plan->staged)
+		return;
+
+	pci_epc_clear_bar(ntb->epf->epc,
+			  ntb->epf->func_no,
+			  ntb->epf->vfunc_no,
+			  epf_bar);
+	kfree(epf_bar->submap);
+	epf_bar->submap = NULL;
+	epf_bar->num_submap = 0;
+	if (plan->stage_addr) {
+		pci_epc_mem_free_addr(ntb->epf->epc, plan->stage_phys,
+				      plan->stage_addr, plan->size);
+		plan->stage_addr = NULL;
+		plan->stage_phys = 0;
+	}
+	memset(plan, 0, sizeof(*plan));
+}
+
 /**
  * epf_ntb_mw_bar_init() - Configure Memory window BARs
  * @ntb: NTB device that facilitates communication between HOST and VHOST
@@ -759,54 +1271,55 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
  */
 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
 {
+	bool programmed[PCI_STD_NUM_BARS] = { };
+	struct device *dev = &ntb->epf->dev;
+	enum pci_barno barno;
 	int ret = 0;
 	int i;
-	u64 size;
-	enum pci_barno barno;
-	struct device *dev = &ntb->epf->dev;
 
 	for (i = 0; i < ntb->num_mws; i++) {
-		size = ntb->mw[i].size;
-		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
+		if (!ntb->mw[i].size)
+			return -EINVAL;
 
-		ntb->epf->bar[barno].barno = barno;
-		ntb->epf->bar[barno].size = size;
-		ntb->epf->bar[barno].addr = NULL;
-		ntb->epf->bar[barno].phys_addr = 0;
-		ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
-				PCI_BASE_ADDRESS_MEM_TYPE_64 :
-				PCI_BASE_ADDRESS_MEM_TYPE_32;
-
-		ret = pci_epc_set_bar(ntb->epf->epc,
-				      ntb->epf->func_no,
-				      ntb->epf->vfunc_no,
-				      &ntb->epf->bar[barno]);
-		if (ret) {
-			dev_err(dev, "MW set failed\n");
-			goto err_alloc_mem;
-		}
-
-		/* Allocate EPC outbound memory windows to vpci vntb device */
 		ntb->mw[i].vpci_mw_addr =
 				pci_epc_mem_alloc_addr(ntb->epf->epc,
 						       &ntb->mw[i].vpci_mw_phys,
-						       size);
+						       ntb->mw[i].size);
 		if (!ntb->mw[i].vpci_mw_addr) {
 			ret = -ENOMEM;
 			dev_err(dev, "Failed to allocate source address\n");
-			goto err_set_bar;
+			goto err_alloc;
+		}
+
+		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
+		if (!epf_ntb_bar_valid(barno)) {
+			ret = -EINVAL;
+			goto err_alloc;
+		}
+		if (programmed[barno])
+			continue;
+
+		ret = epf_ntb_bar_stage1_program(ntb, barno);
+		if (ret) {
+			dev_err(dev, "MW BAR stage1 set failed\n");
+			goto err_alloc;
 		}
+		programmed[barno] = true;
 	}
 
-	return ret;
+	barno = ntb->epf_ntb_bar[BAR_DMA];
+	if (ntb->dma && epf_ntb_bar_valid(barno) && !programmed[barno]) {
+		ret = epf_ntb_bar_stage1_program(ntb, barno);
+		if (ret) {
+			dev_err(dev, "DMA BAR stage1 set failed\n");
+			goto err_alloc;
+		}
+	}
 
-err_set_bar:
-	pci_epc_clear_bar(ntb->epf->epc,
-			  ntb->epf->func_no,
-			  ntb->epf->vfunc_no,
-			  &ntb->epf->bar[barno]);
-err_alloc_mem:
-	epf_ntb_mw_bar_clear(ntb, i);
+	return 0;
+
+err_alloc:
+	epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
 	return ret;
 }
 
@@ -817,21 +1330,32 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
  */
 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
 {
+	bool cleared[PCI_STD_NUM_BARS] = { };
 	enum pci_barno barno;
 	int i;
 
 	for (i = 0; i < num_mws; i++) {
 		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
-		pci_epc_clear_bar(ntb->epf->epc,
-				  ntb->epf->func_no,
-				  ntb->epf->vfunc_no,
-				  &ntb->epf->bar[barno]);
+		if (epf_ntb_bar_valid(barno) && !cleared[barno]) {
+			epf_ntb_mw_bar_release(ntb, barno);
+			cleared[barno] = true;
+		}
+
+		if (!ntb->mw[i].vpci_mw_addr)
+			continue;
 
 		pci_epc_mem_free_addr(ntb->epf->epc,
 				      ntb->mw[i].vpci_mw_phys,
 				      ntb->mw[i].vpci_mw_addr,
 				      ntb->mw[i].size);
+		ntb->mw[i].vpci_mw_addr = NULL;
+		ntb->mw[i].vpci_mw_phys = 0;
+		ntb->mw[i].bar_phys = 0;
 	}
+
+	barno = ntb->epf_ntb_bar[BAR_DMA];
+	if (ntb->dma && epf_ntb_bar_valid(barno) && !cleared[barno])
+		epf_ntb_mw_bar_release(ntb, barno);
 }
 
 /**
@@ -910,6 +1434,11 @@ static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
 	num_mws = ntb->num_mws;
 	dev = &ntb->epf->dev;
 	epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
+	if (!epc_features) {
+		dev_err(dev, "Failed to get EPC features\n");
+		return -ENODEV;
+	}
+	ntb->epc_features = epc_features;
 
 	/* These are required BARs which are mandatory for NTB functionality */
 	for (bar = BAR_CONFIG; bar <= BAR_MW1; bar++) {
@@ -1102,6 +1631,59 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
 	return len;							\
 }
 
+#define EPF_NTB_MW_OFF_R(_name)						\
+static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
+				      char *page)			\
+{									\
+	struct config_group *group = to_config_group(item);		\
+	struct epf_ntb *ntb = to_epf_ntb(group);			\
+	struct device *dev = &ntb->epf->dev;				\
+	int win_no, idx;						\
+									\
+	if (sscanf(#_name, "mw%d_offset", &win_no) != 1)		\
+		return -EINVAL;						\
+									\
+	idx = win_no - 1;						\
+	if (idx < 0 || idx >= ntb->num_mws) {				\
+		dev_err(dev, "MW%d out of range (num_mws=%d)\n",	\
+			win_no, ntb->num_mws);				\
+		return -ERANGE;						\
+	}								\
+	idx = array_index_nospec(idx, ntb->num_mws);			\
+									\
+	return sprintf(page, "%u\n", ntb->mw[idx].offset);		\
+}
+
+#define EPF_NTB_MW_OFF_W(_name)						\
+static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
+				       const char *page, size_t len)	\
+{									\
+	struct config_group *group = to_config_group(item);		\
+	struct epf_ntb *ntb = to_epf_ntb(group);			\
+	struct device *dev = &ntb->epf->dev;				\
+	int win_no, idx;						\
+	u32 val;							\
+	int ret;							\
+									\
+	ret = kstrtou32(page, 0, &val);					\
+	if (ret)							\
+		return ret;						\
+									\
+	if (sscanf(#_name, "mw%d_offset", &win_no) != 1)		\
+		return -EINVAL;						\
+									\
+	idx = win_no - 1;						\
+	if (idx < 0 || idx >= ntb->num_mws) {				\
+		dev_err(dev, "MW%d out of range (num_mws=%d)\n",	\
+			win_no, ntb->num_mws);				\
+		return -ERANGE;						\
+	}								\
+	idx = array_index_nospec(idx, ntb->num_mws);			\
+	ntb->mw[idx].offset = val;					\
+									\
+	return len;							\
+}
+
 #define EPF_NTB_BAR_R(_name, _id)					\
 	static ssize_t epf_ntb_##_name##_show(struct config_item *item,	\
 					      char *page)		\
@@ -1153,6 +1735,30 @@ static ssize_t epf_ntb_num_mws_store(struct config_item *item,
 	return len;
 }
 
+static ssize_t epf_ntb_dma_offset_show(struct config_item *item, char *page)
+{
+	struct config_group *group = to_config_group(item);
+	struct epf_ntb *ntb = to_epf_ntb(group);
+
+	return sprintf(page, "%u\n", ntb->dma_offset);
+}
+
+static ssize_t epf_ntb_dma_offset_store(struct config_item *item,
+					const char *page, size_t len)
+{
+	struct config_group *group = to_config_group(item);
+	struct epf_ntb *ntb = to_epf_ntb(group);
+	u32 val;
+	int ret;
+
+	ret = kstrtou32(page, 0, &val);
+	if (ret)
+		return ret;
+
+	ntb->dma_offset = val;
+	return len;
+}
+
 EPF_NTB_R(spad_count)
 EPF_NTB_W(spad_count)
 EPF_NTB_R(db_count)
@@ -1164,6 +1770,8 @@ EPF_NTB_R(vntb_pid)
 EPF_NTB_W(vntb_pid)
 EPF_NTB_R(vntb_vid)
 EPF_NTB_W(vntb_vid)
+EPF_NTB_R(dma_num_chans)
+EPF_NTB_W(dma_num_chans)
 EPF_NTB_MW_R(mw1)
 EPF_NTB_MW_W(mw1)
 EPF_NTB_MW_R(mw2)
@@ -1172,6 +1780,14 @@ EPF_NTB_MW_R(mw3)
 EPF_NTB_MW_W(mw3)
 EPF_NTB_MW_R(mw4)
 EPF_NTB_MW_W(mw4)
+EPF_NTB_MW_OFF_R(mw1_offset)
+EPF_NTB_MW_OFF_W(mw1_offset)
+EPF_NTB_MW_OFF_R(mw2_offset)
+EPF_NTB_MW_OFF_W(mw2_offset)
+EPF_NTB_MW_OFF_R(mw3_offset)
+EPF_NTB_MW_OFF_W(mw3_offset)
+EPF_NTB_MW_OFF_R(mw4_offset)
+EPF_NTB_MW_OFF_W(mw4_offset)
 EPF_NTB_BAR_R(ctrl_bar, BAR_CONFIG)
 EPF_NTB_BAR_W(ctrl_bar, BAR_CONFIG)
 EPF_NTB_BAR_R(db_bar, BAR_DB)
@@ -1184,6 +1800,8 @@ EPF_NTB_BAR_R(mw3_bar, BAR_MW3)
 EPF_NTB_BAR_W(mw3_bar, BAR_MW3)
 EPF_NTB_BAR_R(mw4_bar, BAR_MW4)
 EPF_NTB_BAR_W(mw4_bar, BAR_MW4)
+EPF_NTB_BAR_R(dma_bar, BAR_DMA)
+EPF_NTB_BAR_W(dma_bar, BAR_DMA)
 
 CONFIGFS_ATTR(epf_ntb_, spad_count);
 CONFIGFS_ATTR(epf_ntb_, db_count);
@@ -1192,15 +1810,22 @@ CONFIGFS_ATTR(epf_ntb_, mw1);
 CONFIGFS_ATTR(epf_ntb_, mw2);
 CONFIGFS_ATTR(epf_ntb_, mw3);
 CONFIGFS_ATTR(epf_ntb_, mw4);
+CONFIGFS_ATTR(epf_ntb_, mw1_offset);
+CONFIGFS_ATTR(epf_ntb_, mw2_offset);
+CONFIGFS_ATTR(epf_ntb_, mw3_offset);
+CONFIGFS_ATTR(epf_ntb_, mw4_offset);
 CONFIGFS_ATTR(epf_ntb_, vbus_number);
 CONFIGFS_ATTR(epf_ntb_, vntb_pid);
 CONFIGFS_ATTR(epf_ntb_, vntb_vid);
+CONFIGFS_ATTR(epf_ntb_, dma_num_chans);
 CONFIGFS_ATTR(epf_ntb_, ctrl_bar);
 CONFIGFS_ATTR(epf_ntb_, db_bar);
 CONFIGFS_ATTR(epf_ntb_, mw1_bar);
 CONFIGFS_ATTR(epf_ntb_, mw2_bar);
 CONFIGFS_ATTR(epf_ntb_, mw3_bar);
 CONFIGFS_ATTR(epf_ntb_, mw4_bar);
+CONFIGFS_ATTR(epf_ntb_, dma_bar);
+CONFIGFS_ATTR(epf_ntb_, dma_offset);
 
 static struct configfs_attribute *epf_ntb_attrs[] = {
 	&epf_ntb_attr_spad_count,
@@ -1210,15 +1835,22 @@ static struct configfs_attribute *epf_ntb_attrs[] = {
 	&epf_ntb_attr_mw2,
 	&epf_ntb_attr_mw3,
 	&epf_ntb_attr_mw4,
+	&epf_ntb_attr_mw1_offset,
+	&epf_ntb_attr_mw2_offset,
+	&epf_ntb_attr_mw3_offset,
+	&epf_ntb_attr_mw4_offset,
 	&epf_ntb_attr_vbus_number,
 	&epf_ntb_attr_vntb_pid,
 	&epf_ntb_attr_vntb_vid,
+	&epf_ntb_attr_dma_num_chans,
 	&epf_ntb_attr_ctrl_bar,
 	&epf_ntb_attr_db_bar,
 	&epf_ntb_attr_mw1_bar,
 	&epf_ntb_attr_mw2_bar,
 	&epf_ntb_attr_mw3_bar,
 	&epf_ntb_attr_mw4_bar,
+	&epf_ntb_attr_dma_bar,
+	&epf_ntb_attr_dma_offset,
 	NULL,
 };
 
@@ -1372,29 +2004,33 @@ static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
 		dma_addr_t addr, resource_size_t size)
 {
 	struct epf_ntb *ntb = ntb_ndev(ndev);
-	struct pci_epf_bar *epf_bar;
+	struct device *dev = &ntb->ntb.dev;
 	enum pci_barno barno;
 	int ret;
-	struct device *dev;
 
-	dev = &ntb->ntb.dev;
 	barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
-	epf_bar = &ntb->epf->bar[barno];
-	epf_bar->phys_addr = addr;
-	epf_bar->barno = barno;
-	epf_bar->size = size;
+	if (size != ntb->mw[idx].size) {
+		dev_err(dev, "unsupported MW resize for shared BAR layout\n");
+		return -EINVAL;
+	}
 
-	ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
-	if (ret) {
+	ntb->mw[idx].bar_phys = addr;
+	ret = epf_ntb_bar_refresh(ntb, barno);
+	if (ret)
 		dev_err(dev, "failure set mw trans\n");
-		return ret;
-	}
-	return 0;
+
+	return ret;
 }
 
-static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)
+static int vntb_epf_mw_clear_trans(struct ntb_dev *ndev, int pidx, int idx)
 {
-	return 0;
+	struct epf_ntb *ntb = ntb_ndev(ndev);
+	enum pci_barno barno;
+
+	barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
+
+	ntb->mw[idx].bar_phys = 0;
+	return epf_ntb_bar_refresh(ntb, barno);
 }
 
 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,
@@ -1695,6 +2331,22 @@ static int epf_ntb_bind(struct pci_epf *epf)
 		return ret;
 	}
 
+	ret = epf_ntb_dma_prepare(ntb);
+	if (ret) {
+		dev_err(dev, "Failed to prepare DMA export\n");
+		goto err_bar_alloc;
+	}
+
+	ret = epf_ntb_validate_bar_layout(ntb);
+	if (ret) {
+		dev_err(dev, "Unsupported BAR layout for this EPC\n");
+		goto err_bar_alloc;
+	}
+
+	ret = epf_ntb_validate_ctrl_layout_v1(ntb);
+	if (ret)
+		goto err_bar_alloc;
+
 	ret = epf_ntb_config_spad_bar_alloc(ntb);
 	if (ret) {
 		dev_err(dev, "Failed to allocate BAR memory\n");
@@ -1730,6 +2382,7 @@ static int epf_ntb_bind(struct pci_epf *epf)
 err_epc_cleanup:
 	epf_ntb_epc_cleanup(ntb);
 err_bar_alloc:
+	epf_ntb_dma_cleanup(ntb);
 	epf_ntb_config_spad_bar_free(ntb);
 
 	return ret;
@@ -1746,6 +2399,7 @@ static void epf_ntb_unbind(struct pci_epf *epf)
 	struct epf_ntb *ntb = epf_get_drvdata(epf);
 
 	epf_ntb_epc_cleanup(ntb);
+	epf_ntb_dma_cleanup(ntb);
 	epf_ntb_config_spad_bar_free(ntb);
 
 	pci_unregister_driver(&vntb_pci_driver);
-- 
2.51.0