[PATCH v15 07/10] x86/shadow: fold _shadow_prealloc() with shadow_blow_tables()

The two functions are pretty similar, and hence changes commonly need to
occur in both places. Fold the functions by introducing a new shared
helper, which then will be used from the three relevant places. Note
that this way preemptability can then be communicated down from
shadow_set_allocation() right away.

Signed-off-by: Jan Beulich <[email protected]>
---
To have the compiler eliminate the extra conditionals, should we make
sh_blow_tables() always-inline?
---
v14: New.

--- a/xen/arch/x86/include/asm/perfc_defn.h
+++ b/xen/arch/x86/include/asm/perfc_defn.h
@@ -49,7 +49,8 @@ PERFCOUNTER(shadow_alloc_tlbflush, "shad
 /* STATUS counters do not reset when 'P' is hit */
 PERFSTATUS(shadow_alloc_count,         "number of shadow pages in use")
 PERFCOUNTER(shadow_free,           "calls to shadow_free")
-PERFCOUNTER(shadow_prealloc_1,     "shadow recycles old shadows")
+PERFCOUNTER(shadow_prealloc_0,     "shadow recycles old shadows")
+PERFCOUNTER(shadow_prealloc_1,     "shadow recycles pinned shadows")
 PERFCOUNTER(shadow_prealloc_2,     "shadow recycles in-use shadows")
 PERFCOUNTER(shadow_linear_map_failed, "shadow hit read-only linear map")
 PERFCOUNTER(shadow_a_update,       "shadow A bit update")
--- a/xen/arch/x86/mm/shadow/common.c
+++ b/xen/arch/x86/mm/shadow/common.c
@@ -370,79 +370,8 @@ static inline void trace_shadow_prealloc
     }
 }
 
-/* Make sure there are at least count order-sized pages
- * available in the shadow page pool. */
-static bool __must_check _shadow_prealloc(struct domain *d, unsigned int pages)
-{
-    struct vcpu *v;
-    struct page_info *sp, *t;
-    mfn_t smfn;
-    int i;
-
-    if ( d->arch.paging.free_pages >= pages )
-        return true;
-
-    if ( unlikely(d->is_dying) )
-        /* No reclaim when the domain is dying, teardown will take care of it. */
-        return false;
-
-    /* Nothing to reclaim when there are no vcpus yet. */
-    if ( !d->vcpu[0] )
-        return false;
-
-    /* Stage one: walk the list of pinned pages, unpinning them */
-    perfc_incr(shadow_prealloc_1);
-    foreach_pinned_shadow(d, sp, t)
-    {
-        smfn = page_to_mfn(sp);
-
-        /* Unpin this top-level shadow */
-        trace_shadow_prealloc_unpin(d, smfn);
-        sh_unpin(d, smfn);
-
-        /* See if that freed up enough space */
-        if ( d->arch.paging.free_pages >= pages )
-            return true;
-    }
-
-    /* Stage two: all shadow pages are in use in hierarchies that are
-     * loaded in cr3 on some vcpu.  Walk them, unhooking the non-Xen
-     * mappings. */
-    perfc_incr(shadow_prealloc_2);
-
-    for_each_vcpu(d, v)
-        for ( i = 0; i < ARRAY_SIZE(v->arch.paging.shadow.shadow_table); i++ )
-        {
-            if ( !pagetable_is_null(v->arch.paging.shadow.shadow_table[i]) )
-            {
-                TRACE_SHADOW_PATH_FLAG(TRCE_SFLAG_PREALLOC_UNHOOK);
-                shadow_unhook_mappings(
-                    d,
-                    pagetable_get_mfn(v->arch.paging.shadow.shadow_table[i]),
-                    0);
-
-                /* See if that freed up enough space */
-                if ( d->arch.paging.free_pages >= pages )
-                {
-                    guest_flush_tlb_mask(d, d->dirty_cpumask);
-                    return true;
-                }
-            }
-        }
-
-    /* Nothing more we can do: all remaining shadows are of pages that
-     * hold Xen mappings for some vcpu.  This can never happen. */
-    printk(XENLOG_ERR "Can't pre-allocate %u shadow pages!\n"
-           "  shadow pages total = %u, free = %u, p2m=%u\n",
-           pages, d->arch.paging.total_pages,
-           d->arch.paging.free_pages, d->arch.paging.p2m_pages);
-
-    ASSERT_UNREACHABLE();
-
-    guest_flush_tlb_mask(d, d->dirty_cpumask);
-
-    return false;
-}
+static bool sh_blow_tables(struct domain *d, unsigned int goal,
+                           bool *preempted);
 
 /* Make sure there are at least count pages of the order according to
  * type available in the shadow page pool.
@@ -466,7 +395,7 @@ bool shadow_prealloc(struct domain *d, u
          ((SHF_L1_ANY | SHF_FL1_ANY) & (1u << type)) )
         count += paging_logdirty_levels();
 
-    ret = _shadow_prealloc(d, count);
+    ret = sh_blow_tables(d, count, NULL);
     if ( !ret && (!d->is_shutting_down || d->shutdown_code != SHUTDOWN_crash) )
         /*
          * Failing to allocate memory required for shadow usage can only result in
@@ -477,9 +406,15 @@ bool shadow_prealloc(struct domain *d, u
     return ret;
 }
 
-/* Deliberately free all the memory we can: this will tear down all of
- * this domain's shadows */
-void shadow_blow_tables(struct domain *d, bool *preempted)
+/*
+ * When @goal is zero: Deliberately free all the memory we can: This will
+ * tear down all of this domain's shadows.
+ *
+ * When @goal is non-zero: Make sure there are at least as many pages
+ * available in the shadow page pool.
+ */
+static bool sh_blow_tables(struct domain *d, unsigned int goal,
+                           bool *preempted)
 {
     struct page_info *sp, *t;
     struct vcpu *v;
@@ -487,19 +422,25 @@ void shadow_blow_tables(struct domain *d
     int i;
     unsigned int done = 0;
 
+    if ( goal && d->arch.paging.free_pages >= goal )
+        return true;
+
     /*
      * When the domain is dying a call to shadow_blow_tables() will be
      * performed from the teardown path with preemption support, ignore any
      * other calls as we want to do the final teardown with preemption support.
      * Teardown of shadow related data can only be avoided when all domain
      * vCPUs are stopped.
+     *
+     * For the pre-allocation case, no reclaim when the domain is dying,
+     * teardown will take care of it.
      */
-    if ( unlikely(d->is_dying) && !preempted )
-        return;
+    if ( unlikely(d->is_dying) && (goal || !preempted) )
+        return false;
 
     /* Nothing to do when there are no vcpus yet. */
     if ( !d->vcpu[0] )
-        return;
+        return false;
 
     /* First pass: reduce page table depth */
     if ( preempted )
@@ -513,16 +454,23 @@ void shadow_blow_tables(struct domain *d
 #endif
         };
 
+        if ( goal )
+            perfc_incr(shadow_prealloc_0);
+
         HASH_CALLBACKS_CHECK(SHF_page_type_mask & ~(SHF_L1_ANY | SHF_FL1_ANY));
 
         for ( i = 0; i < ARRAY_SIZE(masks); ++i )
         {
             while ( hash_foreach(d, masks[i], callbacks, _mfn(0)) )
             {
+                /* See if that freed up enough space */
+                if ( goal && d->arch.paging.free_pages >= goal )
+                    return true;
+
                 if ( general_preempt_check() )
                 {
                     *preempted = true;
-                    return;
+                    return false;
                 }
             }
         }
@@ -530,18 +478,31 @@ void shadow_blow_tables(struct domain *d
     }
 
     /* Second pass: unpin all pinned pages */
+    if ( goal )
+        perfc_incr(shadow_prealloc_1);
     foreach_pinned_shadow(d, sp, t)
     {
         smfn = page_to_mfn(sp);
+
+        /* Unpin this top-level shadow */
+        if ( goal )
+            trace_shadow_prealloc_unpin(d, smfn);
         sh_unpin(d, smfn);
+
+        /* See if that freed up enough space */
+        if ( goal && d->arch.paging.free_pages >= goal )
+            return true;
+
         if ( preempted && !(++done & 0xff) && general_preempt_check() )
         {
             *preempted = true;
-            return;
+            return false;
         }
     }
 
     /* Third pass: unhook entries of in-use shadows */
+    if ( goal )
+        perfc_incr(shadow_prealloc_2);
     for_each_vcpu(d, v)
         for ( i = 0; i < ARRAY_SIZE(v->arch.paging.shadow.shadow_table); i++ )
             if ( !pagetable_is_null(v->arch.paging.shadow.shadow_table[i]) )
@@ -549,11 +510,20 @@ void shadow_blow_tables(struct domain *d
                 unsigned int num = d->arch.paging.total_pages -
                                    d->arch.paging.free_pages;
 
+                if ( goal )
+                    TRACE_SHADOW_PATH_FLAG(TRCE_SFLAG_PREALLOC_UNHOOK);
                 shadow_unhook_mappings(
                     d,
                     pagetable_get_mfn(v->arch.paging.shadow.shadow_table[i]),
                     0);
 
+                /* See if that freed up enough space */
+                if ( goal && d->arch.paging.free_pages >= goal )
+                {
+                    guest_flush_tlb_mask(d, d->dirty_cpumask);
+                    return true;
+                }
+
                 /*
                  * Make sure we are making progress before yielding: if domain
                  * is dying progress will be seen by total_pages decreasing, if
@@ -573,9 +543,32 @@ void shadow_blow_tables(struct domain *d
                 }
             }
 
+    if ( goal )
+    {
+        /*
+         * Nothing more we can do: All remaining shadows are of pages that
+         * hold Xen mappings for some vcpu.  This can never happen.
+         */
+        printk(XENLOG_ERR "Can't pre-allocate %u shadow pages!\n"
+               "  shadow pages total = %u, free = %u, p2m=%u\n",
+               goal, d->arch.paging.total_pages,
+               d->arch.paging.free_pages, d->arch.paging.p2m_pages);
+
+        ASSERT_UNREACHABLE();
+    }
+
  out:
     /* Make sure everyone sees the unshadowings */
     guest_flush_tlb_mask(d, d->dirty_cpumask);
+
+    return false;
+}
+
+/* Deliberately free all the memory we can: this will tear down all of
+ * this domain's shadows */
+void shadow_blow_tables(struct domain *d, bool *preempted)
+{
+    sh_blow_tables(d, 0, preempted);
 }
 
 void shadow_blow_tables_per_domain(struct domain *d)
@@ -910,8 +903,8 @@ int shadow_set_allocation(struct domain
         else if ( d->arch.paging.total_pages > pages )
         {
             /* Need to return memory to domheap */
-            if ( !_shadow_prealloc(d, 1) )
-                return -ENOMEM;
+            if ( !sh_blow_tables(d, 1, preempted) )
+                return preempted && *preempted ? 0 : -ENOMEM;
 
             sp = page_list_remove_head(&d->arch.paging.freelist);
             ASSERT(sp);
@@ -2320,7 +2313,7 @@ void shadow_teardown(struct domain *d, b
 
     /*
      * Reclaim all shadow memory so that shadow_set_allocation() doesn't find
-     * in-use pages, as _shadow_prealloc() will no longer try to reclaim pages
+     * in-use pages, as sh_blow_tables() will no longer try to reclaim pages
      * because the domain is dying.
      */
     paging_lock(d);