[PATCH v4 16/23] x86/boot: choose AP stack based on APIC ID

From: Krystian Hebel <[email protected]>

This is made as the first step of making parallel AP bring-up possible.
It should be enough for pre-C code.

Parallel AP bring-up is necessary because TXT by design releases all APs
at once. In addition to that it reduces number of IPIs (and more
importantly, delays between them) required to start all logical
processors. This results in significant reduction of boot time, even
when DRTM is not used, with performance gain growing with the number of
logical CPUs.

Signed-off-by: Krystian Hebel <[email protected]>
Signed-off-by: Sergii Dmytruk <[email protected]>
---

Notes:
    v4: better comments in boot/trampoline.S
    v4: use %ebp instead of %esp to pass data to boot/x86_64.S
    v4: use `nr_cpu_ids(%rip)` instead of `$NR_CPUS` in boot/x86_64.S
    v4: L_stack_set => L_after_stack_setup
    v4: __ASSEMBLY__ => __ASSEMBLER__

 xen/arch/x86/boot/head.S             |  1 +
 xen/arch/x86/boot/trampoline.S       | 23 +++++++++++++++++++++
 xen/arch/x86/boot/x86_64.S           | 31 +++++++++++++++++++++++++++-
 xen/arch/x86/include/asm/apicdef.h   |  4 ++++
 xen/arch/x86/include/asm/msr-index.h |  3 +++
 xen/arch/x86/setup.c                 |  7 +++++++
 6 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/xen/arch/x86/boot/head.S b/xen/arch/x86/boot/head.S
index 2c1a0f6306..ff46579904 100644
--- a/xen/arch/x86/boot/head.S
+++ b/xen/arch/x86/boot/head.S
@@ -8,6 +8,7 @@
 #include <asm/page.h>
 #include <asm/processor.h>
 #include <asm/msr-index.h>
+#include <asm/apicdef.h>
 #include <asm/cpufeature.h>
 #include <asm/trampoline.h>
 
diff --git a/xen/arch/x86/boot/trampoline.S b/xen/arch/x86/boot/trampoline.S
index a92e399fbe..9306c1bb76 100644
--- a/xen/arch/x86/boot/trampoline.S
+++ b/xen/arch/x86/boot/trampoline.S
@@ -71,6 +71,29 @@ trampoline_protmode_entry:
         mov     $X86_CR4_PAE,%ecx
         mov     %ecx,%cr4
 
+        /*
+         * Get APIC ID while we're in non-paged mode to later derive Xen CPU
+         * index and determine CPU-specific stack. Start by checking if x2APIC
+         * is enabled.
+         */
+        mov     $MSR_APIC_BASE, %ecx
+        rdmsr
+        test    $APIC_BASE_EXTD, %eax
+        jnz     .Lx2apic
+
+        /* Not x2APIC, read APIC ID from MMIO. */
+        and     $APIC_BASE_ADDR_MASK, %eax
+        mov     APIC_ID(%eax), %eax
+        shr     $24, %eax
+        jmp     1f
+
+.Lx2apic:
+        mov     $(MSR_X2APIC_FIRST + (APIC_ID >> MSR_X2APIC_SHIFT)), %ecx
+        rdmsr
+1:
+        /* The value of the APIC ID will be consumed in __high_start. */
+        mov     %eax, %ebp
+
         /* Load pagetable base register. */
         mov     $sym_offs(idle_pg_table),%eax
         add     bootsym_rel(trampoline_xen_phys_start,4,%eax)
diff --git a/xen/arch/x86/boot/x86_64.S b/xen/arch/x86/boot/x86_64.S
index 9705d03f84..19f3062a7b 100644
--- a/xen/arch/x86/boot/x86_64.S
+++ b/xen/arch/x86/boot/x86_64.S
@@ -11,7 +11,36 @@ ENTRY(__high_start)
         mov     %ecx,%gs
         mov     %ecx,%ss
 
-        mov     stack_start(%rip),%rsp
+        /* %ebx is set to non-zero in trampoline.S to indicate an AP. */
+        test    %ebx, %ebx
+        cmovz   stack_start(%rip), %rsp
+        jz      .L_after_stack_setup
+
+        /*
+         * APs only: get stack base from APIC ID saved to %ebp in trampoline.S.
+         */
+        mov     $-1, %rax
+        lea     x86_cpu_to_apicid(%rip), %rcx
+1:
+        inc     %rax
+        cmp     nr_cpu_ids(%rip), %eax
+        jb      2f
+        hlt
+2:
+        cmp     %ebp, (%rcx, %rax, 4)
+        jne     1b
+
+        /* %eax is now Xen CPU index. */
+        lea     stack_base(%rip), %rcx
+        mov     (%rcx, %rax, 8), %rsp
+
+        test    %rsp, %rsp
+        jnz     1f
+        hlt
+1:
+        add     $(STACK_SIZE - CPUINFO_sizeof), %rsp
+
+.L_after_stack_setup:
 
         /* Reset EFLAGS (subsumes CLI and CLD). */
         pushq   $0
diff --git a/xen/arch/x86/include/asm/apicdef.h b/xen/arch/x86/include/asm/apicdef.h
index 112c1dc613..7a09d08b91 100644
--- a/xen/arch/x86/include/asm/apicdef.h
+++ b/xen/arch/x86/include/asm/apicdef.h
@@ -120,6 +120,10 @@
 
 #define MAX_IO_APICS 128
 
+#ifndef __ASSEMBLER__
+
 extern bool x2apic_enabled;
 
+#endif /* !__ASSEMBLER__ */
+
 #endif
diff --git a/xen/arch/x86/include/asm/msr-index.h b/xen/arch/x86/include/asm/msr-index.h
index ad1c6c97f8..0900e21811 100644
--- a/xen/arch/x86/include/asm/msr-index.h
+++ b/xen/arch/x86/include/asm/msr-index.h
@@ -186,6 +186,9 @@
 #define MSR_X2APIC_FIRST                    0x00000800
 #define MSR_X2APIC_LAST                     0x000008ff
 
+/* MSR offset can be obtained by shifting MMIO offset this number of bits to the right. */
+#define MSR_X2APIC_SHIFT                    4
+
 #define MSR_X2APIC_TPR                      0x00000808
 #define MSR_X2APIC_PPR                      0x0000080a
 #define MSR_X2APIC_EOI                      0x0000080b
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 5494fa1621..fbc44b0174 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -2141,6 +2141,7 @@ void asmlinkage __init noreturn __start_xen(void)
      */
     if ( !pv_shim )
     {
+        /* Separate loop to make parallel AP bringup possible. */
         for_each_present_cpu ( i )
         {
             /* Set up cpu_to_node[]. */
@@ -2148,6 +2149,12 @@ void asmlinkage __init noreturn __start_xen(void)
             /* Set up node_to_cpumask based on cpu_to_node[]. */
             numa_add_cpu(i);
 
+            if ( stack_base[i] == NULL )
+                stack_base[i] = cpu_alloc_stack(i);
+        }
+
+        for_each_present_cpu ( i )
+        {
             if ( (park_offline_cpus || num_online_cpus() < max_cpus) &&
                  !cpu_online(i) )
             {
-- 
2.55.0