[PATCH 3/6] examples/l3fwd-power: enable Rx interrupt before epoll add

event_register() added each Rx queue interrupt fd to the per-thread epoll
set before the worker loop, while the queues were still disarmed. This
works when the fd is assigned at queue setup and stays fixed, as on MSI-X
NICs.

The dpaa2 PMD delivers Rx interrupts through a per-lcore QBMan portal and
binds the fd when the interrupt is enabled, not at queue setup. Adding a
queue to the epoll set before arming it then watches an fd that is not
bound yet, so the lcore never wakes from rte_epoll_wait().

Register from rx_interrupt_wait(), after the queues are armed, and only
once: keep the epoll entry installed for the lifetime of the loop. NICs
with a fixed per-queue fd are unaffected.

Signed-off-by: Maxime Leroy <[email protected]>
---
 examples/l3fwd-power/main.c | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 70f76d202e..4874a55de1 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -945,7 +945,7 @@ static int event_register(struct lcore_conf *qconf)
 }
 
 static int
-rx_interrupt_wait(struct lcore_conf *qconf)
+rx_interrupt_wait(struct lcore_conf *qconf, int *intr_registered)
 {
 	int ret;
 
@@ -953,6 +953,16 @@ rx_interrupt_wait(struct lcore_conf *qconf)
 	if (ret != 0)
 		return ret;
 
+	/* some PMDs expose the interrupt fd only once the queue is armed */
+	if (!*intr_registered) {
+		ret = event_register(qconf);
+		if (ret != 0) {
+			rx_intr_disable_all(qconf);
+			return ret;
+		}
+		*intr_registered = 1;
+	}
+
 	sleep_until_rx_interrupt(qconf->n_rx_queue, rte_lcore_id());
 	rx_intr_disable_all(qconf);
 	return 0;
@@ -970,7 +980,8 @@ static int main_intr_loop(__rte_unused void *dummy)
 	struct lcore_rx_queue *rx_queue;
 	uint32_t lcore_rx_idle_count = 0;
 	uint32_t lcore_idle_hint = 0;
-	int intr_en = 0;
+	int intr_registered = 0;
+	int intr_en = 1;
 	int ret;
 
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
@@ -998,12 +1009,6 @@ static int main_intr_loop(__rte_unused void *dummy)
 				lcore_id, portid, queueid);
 	}
 
-	/* add into event wait list */
-	if (event_register(qconf) == 0)
-		intr_en = 1;
-	else
-		RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n");
-
 	while (!is_done()) {
 		stats[lcore_id].nb_iteration_looped++;
 
@@ -1103,7 +1108,7 @@ static int main_intr_loop(__rte_unused void *dummy)
 			else {
 				/* suspend until rx interrupt triggers */
 				if (intr_en) {
-					ret = rx_interrupt_wait(qconf);
+					ret = rx_interrupt_wait(qconf, &intr_registered);
 					if (ret == -EAGAIN)
 						goto start_rx;
 					if (ret != 0) {
@@ -1260,7 +1265,8 @@ main_legacy_loop(__rte_unused void *dummy)
 	enum freq_scale_hint_t lcore_scaleup_hint;
 	uint32_t lcore_rx_idle_count = 0;
 	uint32_t lcore_idle_hint = 0;
-	int intr_en = 0;
+	int intr_registered = 0;
+	int intr_en = 1;
 	int ret;
 
 	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
@@ -1286,12 +1292,6 @@ main_legacy_loop(__rte_unused void *dummy)
 			"rxqueueid=%" PRIu16 "\n", lcore_id, portid, queueid);
 	}
 
-	/* add into event wait list */
-	if (event_register(qconf) == 0)
-		intr_en = 1;
-	else
-		RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n");
-
 	while (!is_done()) {
 		stats[lcore_id].nb_iteration_looped++;
 
@@ -1424,7 +1424,7 @@ main_legacy_loop(__rte_unused void *dummy)
 			else {
 				/* suspend until rx interrupt triggers */
 				if (intr_en) {
-					ret = rx_interrupt_wait(qconf);
+					ret = rx_interrupt_wait(qconf, &intr_registered);
 					if (ret == -EAGAIN)
 						goto start_rx;
 					if (ret != 0) {
-- 
2.43.0