Re: PCU panic on V240

>>> "J. Hannken-Illjes" <[email protected]> wrote

> Running -current on a Sparc V240 I get a diagnostic assertion running
> the script attached:
> 
> panic: kernel diagnostic assertion "!cpu_intr_p()" failed: file "/work/build/src/sys/kern/subr_xcall.c", line 351 
:
:
> This is the high priority xcall from subr_pcu.c and
> curcpu()->ci_idepth is zero.
> 
> What is going wrong here?

There is some trick to handle ci_idepth in the case of softint.

In locore.s::softint_fastintr, decrease ci_idepth before calling
softint handlers then increase ci_idepth.  But it seems that there is
a chance to interrupt SOFTINT_SERIAL while handling other SOFTINT_*,
and ci_idepth is increased in the meantime.


locore.s:
-> 3456 	wrpr	%g0, PSTATE_INTR, %pstate	! Reenable interrupts
   3457 	jmpl	%o4, %o7		! handled = (*ih->ih_fun)(...)
   3458 	 movrz	%o0, %o2, %o0		! arg = (arg == 0) ? arg : tf
   3459 	wrpr	%g0, PSTATE_KERN, %pstate	! Disable interrupts
    :
   5236 ENTRY(softint_fastintr)
   5237 	save	%sp, -CC64FSZ, %sp
   5238 	set	CPUINFO_VA, %l0			! l0 = curcpu()
   5239 	rdpr	%pil, %l7			! l7 = splhigh()
-> 5240 	wrpr	%g0, PIL_HIGH, %pil
   5241 	ld	[%l0 + CI_IDEPTH], %l1
   5242 	LDPTR	[%l0 + CI_EINTSTACK], %l6	! l6 = ci_eintstack
   5243 	dec	%l1
   5244 	add	%sp, -CC64FSZ, %l2		! ci_eintstack = sp - CC64FSZ
   5245 	st	%l1, [%l0 + CI_IDEPTH]		! adjust ci_idepth
   5246 	STPTR	%l2, [%l0 + CI_EINTSTACK]	! save intstack for nexted intr


So, does the attached patch fix the panic?

-- Takeshi Nakayama


--- sparc64/locore.s.orig	2013-05-25 08:14:47.000000000 +0900
+++ sparc64/locore.s	2013-12-06 13:16:42.000000000 +0900
@@ -3288,12 +3288,6 @@
 	 LDPTR	[%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5
 0:
 
-	! Increment the per-cpu interrupt level
-	sethi	%hi(CPUINFO_VA+CI_IDEPTH), %g1
-	ld	[%g1 + %lo(CPUINFO_VA+CI_IDEPTH)], %g2
-	inc	%g2
-	st	%g2, [%g1 + %lo(CPUINFO_VA+CI_IDEPTH)]
-
 #ifdef TRAPSTATS
 	sethi	%hi(_C_LABEL(kintrcnt)), %g1
 	sethi	%hi(_C_LABEL(uintrcnt)), %g2
@@ -3382,6 +3376,17 @@
 	
 	wrpr	%l6, %pil
 
+#define SOFTINT_INT \
+	(1<<IPL_SOFTCLOCK|1<<IPL_SOFTBIO|1<<IPL_SOFTNET|1<<IPL_SOFTSERIAL)
+
+	! Increment the per-cpu interrupt depth in case of hardintrs
+	btst	SOFTINT_INT, %l3
+	bnz,pn	%icc, sparc_intr_retry
+	 sethi	%hi(CPUINFO_VA+CI_IDEPTH), %l1
+	ld	[%l1 + %lo(CPUINFO_VA+CI_IDEPTH)], %l2
+	inc	%l2
+	st	%l2, [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)]
+
 sparc_intr_retry:
 	wr	%l3, 0, CLEAR_SOFTINT	! (don't clear possible %tick IRQ)
 	sethi	%hi(CPUINFO_VA+CI_INTRPENDING), %l4
@@ -3481,11 +3486,14 @@
 	bnz,pn	%icc, sparc_intr_retry
 	 mov	1, %l5			! initialize intr count for next run
 
-	! Decrement this cpu's interrupt depth
-	sethi	%hi(CPUINFO_VA+CI_IDEPTH), %l4
+	! Decrement this cpu's interrupt depth in case of hardintrs
+	btst	SOFTINT_INT, %l3
+	bnz,pn	%icc, 1f
+	 sethi	%hi(CPUINFO_VA+CI_IDEPTH), %l4
 	ld	[%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5
 	dec	%l5
 	st	%l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)]
+1:
 
 #ifdef NOT_DEBUG
 	set	_C_LABEL(intrdebug), %o2
@@ -5238,11 +5246,8 @@
 	set	CPUINFO_VA, %l0			! l0 = curcpu()
 	rdpr	%pil, %l7			! l7 = splhigh()
 	wrpr	%g0, PIL_HIGH, %pil
-	ld	[%l0 + CI_IDEPTH], %l1
 	LDPTR	[%l0 + CI_EINTSTACK], %l6	! l6 = ci_eintstack
-	dec	%l1
 	add	%sp, -CC64FSZ, %l2		! ci_eintstack = sp - CC64FSZ
-	st	%l1, [%l0 + CI_IDEPTH]		! adjust ci_idepth
 	STPTR	%l2, [%l0 + CI_EINTSTACK]	! save intstack for nexted intr
 
 	mov	%i0, %o0			! o0/i0 = softint lwp
@@ -5287,10 +5292,7 @@
 
 	restore					! rewind register window
 
-	ld	[%l0 + CI_IDEPTH], %l1
 	STPTR	%l6, [%l0 + CI_EINTSTACK]	! restore ci_eintstack
-	inc	%l1
-	st	%l1, [%l0 + CI_IDEPTH]		! re-adjust ci_idepth
 	wrpr	%g0, %l7, %pil			! restore ipl
 	ret
 	 restore	%g0, 1, %o0
@@ -5314,10 +5316,7 @@
 	st	%o1, [%l0 + CI_MTX_COUNT]
 	st	%g0, [%o0 + L_CTXSWTCH]		! prev->l_ctxswtch = 0
 
-	ld	[%l0 + CI_IDEPTH], %l1
 	STPTR	%l6, [%l0 + CI_EINTSTACK]	! restore ci_eintstack
-	inc	%l1
-	st	%l1, [%l0 + CI_IDEPTH]		! re-adjust ci_idepth
 	wrpr	%g0, %l7, %pil			! restore ipl
 	ret
 	 restore	%g0, 1, %o0