Cleaned -up evidence about the PowerMac G5 multiprocessor boot hang ups with the modern VM_MAX_KERNEL_ADDRESS value
Mark Millard via freebsd-ppc <[email protected]>
| Newsgroups | gmane.os.freebsd.devel.ppc |
|---|---|
| Message-ID | <[email protected]> |
I eliminated a bunch of reports that gave information
that was the same for hang-ups and successful boots.
So this is starting over with less material to go
through.
For an unsuccessful boot:
(Values like 0x10, 0x25, 0x51 are use as recorded labels for points in the code.)
Adding CPU 0, hwref=cd38, awake=1
Waking up CPU 3 (dev=c480)
After reset 4&0 for CPU 3, hwref=c480, awake=0, *(unsigned long*)0xc0000000000000f0=0x25
After attempted wait for awake CPU 3, hwref=c480, awake=0, *(unsigned long*)0xc0000000000000f0=0x25
Waking up CPU 2 (dev=c768)
So neither pc_awake nor *0xc0000000000000f0 have machdep_ap_bootstrap
based values as seen by CPU 0 (from trying to start CPU 3).
For a successful boot:
Adding CPU 0, hwref=cd38, awake=1
Waking up CPU 3 (dev=c480)
After reset 4&0 for CPU 3, hwref=c480, awake=0, *(unsigned long*)0xc0000000000000f0=0x25
After attempted wait for awake CPU 3, hwref=c480, awake=1, *(unsigned long*)0xc0000000000000f0=0x51
Adding CPU 3, hwref=c480, awake=1
Waking up CPU 2 (dev=c768)
After reset 4&0 for CPU 2, hwref=c768, awake=0, *(unsigned long*)0xc0000000000000f0=0x51
After attempted wait for awake CPU 2, hwref=c768, awake=1, *(unsigned long*)0xc0000000000000f0=0x51
Adding CPU 2, hwref=c768, awake=1
Waking up CPU 1 (dev=ca50)
After reset 4&0 for CPU 1, hwref=ca50, awake=0, *(unsigned long*)0xc0000000000000f0=0x51
After attempted wait for awake CPU 1, hwref=ca50, awake=1, *(unsigned long*)0xc0000000000000f0=0x51
Adding CPU 1, hwref=ca50, awake=1
SMP: AP CPU #3 launched
SMP: AP CPU #1 launched
SMP: AP CPU #2 launched
Trying to mount root from ufs:/dev/ufs/FBSDG5L2rootfs [rw,noatime]...
GEOM: new disk ada1
So here waiting on cpu 0 does see pc_awake become 1 and does see
0xc0000000000000f0 updated by machdep_ap_bootstrap for CPU , for
example.
Note the awake=0 (i.e.., pc_awake) examples for each cpu even when
there is already a 0x51 label for CPU 2 and CPU 3. (The code is
shown later.)
This is based on the labeling updates to:
("labels" stored at 0xc0000000000000f0)
cpudep_ap_early_bootstrap (before-return labeled with value : 0x10)
moea64_cpu_bootstrap_native (labeled with values: 0x21, 0x22, 0x23, 0x24, 0x25 at various points)
pmap_cpu_bootstrap (before-return labeled with value : 0x20)
cpudep_ap_bootstrap (before-return labeled with value : 0x30)
cpudep_ap_setup (before-return labeled with value : 0x40)
machdep_ap_bootstrap (lableled with values: 0x5F, 0x51, and 0x50 at various points)
powermac_smp_start_cpu (reporting values after the resets for a CPU and after
the pa_awake loop)
I happen to have experimented with additional isyncs in
this variant of the code, not that they made much of
a difference.
The specifics are:
# svnlite diff /usr/src/sys/powerpc/aim/mp_cpudep.c /usr/src/sys/powerpc/powerpc/pmap_dispatch.c /usr/src/sys/powerpc/powerpc/mp_machdep.c /usr/src/sys/powerpc/powermac/platform_powermac.c /usr/src/sys/powerpc/aim/moea64_native.c | more
Index: /usr/src/sys/powerpc/aim/mp_cpudep.c
===================================================================
--- /usr/src/sys/powerpc/aim/mp_cpudep.c (revision 344018)
+++ /usr/src/sys/powerpc/aim/mp_cpudep.c (working copy)
@@ -105,6 +105,10 @@
__asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu));
powerpc_sync();
+
+ *(unsigned long*)0xc0000000000000f0 = 0x10; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
}
uintptr_t
@@ -124,6 +128,10 @@
pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
sp = pcpup->pc_curpcb->pcb_sp;
+ *(unsigned long*)0xc0000000000000f0 = 0x30; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
+
return (sp);
}
@@ -416,5 +424,9 @@
"suboptimal.\n");
break;
}
+
+ *(unsigned long*)0xc0000000000000f0 = 0x40; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
}
Index: /usr/src/sys/powerpc/aim/moea64_native.c
===================================================================
--- /usr/src/sys/powerpc/aim/moea64_native.c (revision 344018)
+++ /usr/src/sys/powerpc/aim/moea64_native.c (working copy)
@@ -384,6 +384,9 @@
mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR);
+ *(unsigned long*)0xc0000000000000f0 = 0x21; // HACK!!!
+ powerpc_sync(); // HACK!!!
+
/*
* Install kernel SLB entries
*/
@@ -393,6 +396,9 @@
__asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) :
"r"(0));
+ *(unsigned long*)0xc0000000000000f0 = 0x22; // HACK!!!
+ powerpc_sync(); // HACK!!!
+
for (i = 0; i < n_slbs; i++) {
if (!(slb[i].slbe & SLBE_VALID))
continue;
@@ -400,6 +406,9 @@
__asm __volatile ("slbmte %0, %1" ::
"r"(slb[i].slbv), "r"(slb[i].slbe));
}
+
+ *(unsigned long*)0xc0000000000000f0 = 0x23; // HACK!!!
+ powerpc_sync(); // HACK!!!
#else
for (i = 0; i < 16; i++)
mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
@@ -412,7 +421,14 @@
__asm __volatile ("ptesync; mtsdr1 %0; isync"
:: "r"(((uintptr_t)moea64_pteg_table & ~DMAP_BASE_ADDRESS)
| (uintptr_t)(flsl(moea64_pteg_mask >> 11))));
+
+ *(unsigned long*)0xc0000000000000f0 = 0x24; // HACK!!!
+ powerpc_sync(); // HACK!!!
+
tlbia();
+
+ *(unsigned long*)0xc0000000000000f0 = 0x25; // HACK!!!
+ powerpc_sync(); // HACK!!!
}
Index: /usr/src/sys/powerpc/powerpc/pmap_dispatch.c
===================================================================
--- /usr/src/sys/powerpc/powerpc/pmap_dispatch.c (revision 344018)
+++ /usr/src/sys/powerpc/powerpc/pmap_dispatch.c (working copy)
@@ -445,6 +445,10 @@
*/
return (MMU_CPU_BOOTSTRAP(mmu_obj, ap));
+
+ *(unsigned long*)0xc0000000000000f0 = 0x20; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
}
void *
Index: /usr/src/sys/powerpc/powerpc/mp_machdep.c
===================================================================
--- /usr/src/sys/powerpc/powerpc/mp_machdep.c (revision 344018)
+++ /usr/src/sys/powerpc/powerpc/mp_machdep.c (working copy)
@@ -73,12 +73,20 @@
void
machdep_ap_bootstrap(void)
{
+ *(unsigned long*)0xc0000000000000f0 = 0x5F; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
PCPU_SET(awake, 1);
__asm __volatile("msync; isync");
+ *(unsigned long*)0xc0000000000000f0 = 0x51; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
+
while (ap_letgo == 0)
__asm __volatile("or 31,31,31");
+ isync(); // HACK!!!
__asm __volatile("or 6,6,6");
/*
@@ -109,10 +117,15 @@
while(smp_started == 0)
;
+ isync(); // HACK!!!
/* Start per-CPU event timers. */
cpu_initclocks_ap();
+ *(unsigned long*)0xc0000000000000f0 = 0x50; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
+
/* Announce ourselves awake, and enter the scheduler */
sched_throw(NULL);
}
@@ -241,11 +254,13 @@
timeout = 2000; /* wait 2sec for the AP */
while (!pc->pc_awake && --timeout > 0)
DELAY(1000);
+ isync(); // HACK!!!
}
} else {
pc->pc_awake = 1;
}
if (pc->pc_awake) {
+ isync(); // HACK!!!
if (bootverbose)
printf("Adding CPU %d, hwref=%jx, awake=%x\n",
pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
@@ -256,6 +271,8 @@
}
ap_awake = 1;
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
/* Provide our current DEC and TB values for APs */
ap_timebase = mftb() + 10;
@@ -268,6 +285,7 @@
while (ap_awake < smp_cpus)
;
+ isync(); // HACK!!!
if (smp_cpus != cpus || cpus != mp_ncpus) {
printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n",
Index: /usr/src/sys/powerpc/powermac/platform_powermac.c
===================================================================
--- /usr/src/sys/powerpc/powermac/platform_powermac.c (revision 344018)
+++ /usr/src/sys/powerpc/powermac/platform_powermac.c (working copy)
@@ -382,10 +382,22 @@
(void)(*rstvec);
powerpc_sync();
+ isync(); // HACK!!!
+ if (bootverbose) // HACK!!!
+ printf("After reset 4&0 for CPU %d, hwref=%jx, awake=%x, *(unsigned long*)0xc0000000000000f0=0x%jx\n",
+ pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+ pc->pc_awake,(uintmax_t)*(unsigned long*)0xc0000000000000f0);
+
timeout = 10000;
while (!pc->pc_awake && timeout--)
DELAY(100);
+ isync(); // HACK!!!
+ if (bootverbose) // HACK!!!
+ printf("After attempted wait for awake CPU %d, hwref=%jx, awake=%x, *(unsigned long*)0xc0000000000000f0=0x%jx\n",
+ pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+ pc->pc_awake,(uintmax_t)*(unsigned long*)0xc0000000000000f0);
+
return ((pc->pc_awake) ? 0 : EBUSY);
#else
/* No SMP support */
===
Mark Millard
marklmi at yahoo.com
( dsl-only.net went
away in early 2018-Mar)
_______________________________________________
[email protected] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-ppc
To unsubscribe, send any mail to "[email protected]"