[SSI] openssi/kernel/cluster/ssi/vproc as_xscribe.c,1.19,1.20

Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/vproc
In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv11332/cluster/ssi/vproc

Modified Files:
      Tag: OPENSSI-FC
	as_xscribe.c 
Log Message:
- as_do_pg() use pud_alloc() like install_arg_page() in Linux base; based on John Hughes' findings.
- Fix inappropriate inlining of as_pull_pgdir(), as_count_pgdir().
- No need to hold page_table_lock while traversing struct mm->mmap list of VMA's in as_count_mm_info().
- Use cond_resched_lock() instead of run_over_maxchunk. Prevent needless re-traversing of the page table.
- Fix memory corruption due to do_swap_page() unmapping unmapped PTE in as_pull_pgdir() path. Memory corruption occurs when the PTE passed to do_swap_page() is not the first PTE in the list of PTE's because in as_pull_pgdir() only the first PTE is mapped. Bug only affects kernels compiled with CONFIG_HIGHPTE defined.
- Process migration to support 4-level page table.


Index: as_xscribe.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/vproc/as_xscribe.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- as_xscribe.c	17 Dec 2009 06:43:51 -0000	1.19
+++ as_xscribe.c	18 Apr 2010 20:44:24 -0000	1.20
@@ -67,10 +67,6 @@
 #define	AS_VMA_MAX	(PAGE_SIZE / sizeof(as_vma_info))
 #define	AS_PAGE_MAX	((ICS_MAX_OOL_DATA_SIZE + PAGE_SIZE - 1) / PAGE_SIZE)
 
-#define	RUN_OVER_MAGIC	126094
-
-static int run_over_maxchunk;
-
 static inline int
 as_vma_is_private(struct vm_area_struct *vma)
 {
@@ -175,8 +171,9 @@
 	if (pte_present(*(pte_t *)ptep)) {
 		flags |= 0x0008;
 		if (pfn_valid(pte_pfn(*(pte_t *)ptep))) {
-			page = pte_page(*(pte_t *)ptep);
 			flags |= 0x10;
+			page = pte_page(*(pte_t *)ptep);
+			flush_dcache_page(page);
 			if (PageAnon(page))
 				flags |= 0x0020;
 			if (PageSwapCache(page))
@@ -198,11 +195,13 @@
 static struct page *
 asx_hist_get_page(void *ptep)
 {
-	struct page *page = NULL;
+	struct page *page;
 
-	if (pte_present(*(pte_t *)ptep))
-		page = pte_page(*(pte_t *)ptep);
+	if (!pte_present(*(pte_t *)ptep))
+		return NULL;
 
+	page = pte_page(*(pte_t *)ptep);
+	flush_dcache_page(page);
 	return page;
 }
 
@@ -300,31 +299,28 @@
 }
 
 static void
-asx_print_pgdir(struct mm_struct *mm, struct vm_area_struct *vma,
-		unsigned long *addr, pgd_t *pgdir, unsigned long pgd_end)
+asx_print_pudir(struct mm_struct *mm, struct vm_area_struct *vma,
+		unsigned long *addr, pud_t *pudir, unsigned long pud_end)
 {
-	pud_t *pudir;
 	pmd_t *pmdir;
 	pte_t *pte;
 	pte_t *pte_dir;
 	unsigned long pmd_end;
-	unsigned long pte_end;
 	int flags;
 	struct page *page;
 	void  *kaddr;
 	int csum;
 
-	pudir = pud_offset(pgdir, *addr);
 	pmdir = pmd_offset(pudir, *addr);
-	for(; *addr < pgd_end; *addr = pmd_end, pmdir++) {
+	for(; *addr < pud_end; *addr = pmd_end, pmdir++) {
 		pmd_end = (*addr + PMD_SIZE) & PMD_MASK;
-		if (pmd_end > pgd_end)
-			pmd_end = pgd_end;
+		if (pmd_end > pud_end || pmd_end <= *addr)
+			pmd_end = pud_end;
 		if (pmd_none(*pmdir) || pmd_bad(*pmdir))
 			continue;
+
 		pte = pte_dir = pte_offset_map(pmdir, *addr);
-		for(; *addr < pmd_end; *addr = pte_end, pte++) {
-			pte_end = *addr + PAGE_SIZE;
+		for(; *addr < pmd_end; *addr += PAGE_SIZE, pte++) {
 			flags = asx_hist_get_flags(vma, pte);
 			page = asx_hist_get_page(pte);
 			kaddr = asx_hist_get_kaddr(page);
@@ -341,6 +337,24 @@
 	}
 }
 
+static void
+asx_print_pgdir(struct mm_struct *mm, struct vm_area_struct *vma,
+		unsigned long *addr, pgd_t *pgdir, unsigned long pgd_end)
+{
+	pud_t *pudir;
+	unsigned long pud_end;
+
+	pudir = pud_offset(pgdir, *addr);
+	for(; *addr < pgd_end; *addr = pud_end, pudir++) {
+		pud_end = (*addr + PUD_SIZE) & PUD_MASK;
+		if (pud_end > pgd_end || pud_end <= *addr)
+			pud_end = pgd_end;
+		if (pud_none(*pudir) || pud_bad(*pudir))
+			continue;
+		asx_print_pudir(mm, vma, addr, pudir, pud_end);
+	}
+}
+
 void
 asx_print_mm(struct task_struct *p, struct mm_struct *mm,
 	     unsigned long saddr, unsigned long eaddr, int vma_only)
@@ -445,33 +459,29 @@
 {
 }
 
-#endif
+#endif /* !DEBUG_TOOLS */
 
-static inline int
-as_count_pgdir(as_info *asip, struct vm_area_struct *vma,
+static int
+as_count_pudir(as_info *asip, struct vm_area_struct *vma,
 	       int isprivate, unsigned long *addr,
-	       pgd_t *pgdir, unsigned long pgd_end)
+	       pud_t *pudir, unsigned long pud_end)
 {
 	int error = 0;
-	pud_t *pudir;
 	pmd_t *pmdir;
 	pte_t *pte;
 	pte_t *pte_dir;
 	unsigned long pmd_end;
-	unsigned long pte_end;
 	struct page *page;
 
-	pudir = pud_offset(pgdir, *addr);
 	pmdir = pmd_offset(pudir, *addr);
-	for(; *addr < pgd_end && !error; *addr = pmd_end, pmdir++) {
+	for(; *addr < pud_end && !error; *addr = pmd_end, pmdir++) {
 		pmd_end = (*addr + PMD_SIZE) & PMD_MASK;
-		if (pmd_end > pgd_end)
-			pmd_end = pgd_end;
+		if (pmd_end > pud_end || pmd_end <= *addr)
+			pmd_end = pud_end;
 		if (pmd_none(*pmdir) || pmd_bad(*pmdir))
 			continue;
 		pte = pte_dir = pte_offset_map(pmdir, *addr);
-		for(; *addr < pmd_end; *addr = pte_end, pte++) {
-			pte_end = *addr + PAGE_SIZE;
+		for(; *addr < pmd_end; *addr += PAGE_SIZE, pte++) {
 			asx_hist(0, asip->as_pid, *addr, vma, pte);
 			if (!pte_present(*pte)) {
 				if (pte_none(*pte))
@@ -482,12 +492,13 @@
 					goto out;
 				}
 				page = pte_page(*pte);
+				flush_dcache_page(page);
 				if (PageReserved(page)) {
 					if (page != ZERO_PAGE(*addr)) {
 						error = -EBUSY;
 						goto out;
-					} else
-						continue;
+					}
+					continue;
 				}
 				if (!isprivate || !PageAnon(page))
 					continue;
@@ -502,53 +513,55 @@
 	return 0;
 }
 
+/* We hold mm semaphore and page_table_lock on entry. */
 static int
-as_count_mm_info(as_info *asip, enum asx_op op)
+as_count_pgdir(as_info *asip, struct vm_area_struct *vma,
+	       int isprivate, unsigned long *addr,
+	       pgd_t *pgdir, unsigned long pgd_end)
 {
+	pud_t *pudir;
+	unsigned long pud_end;
 	int error = 0;
+
+	pudir = pud_offset(pgdir, *addr);
+	for(; *addr < pgd_end && !error; *addr = pud_end, pudir++) {
+		pud_end = (*addr + PUD_SIZE) & PUD_MASK;
+		if (pud_end > pgd_end || pud_end <= *addr)
+			pud_end = pgd_end;
+		if (pud_none(*pudir) || pud_bad(*pudir))
+			continue;
+		error = as_count_pudir(asip, vma, isprivate,
+					addr, pudir, pud_end);
+		if (error < 0)
+			break;
+	}
+	return error;
+}
+
+static int
+as_count_mm_info(as_info *asip, enum asx_op op)
+{
 	struct mm_struct *mm;
 	struct vm_area_struct *vma;
-	struct task_struct *p;
-	unsigned long start, addr, pgd_end;
 	pgd_t *pgdir;
-	int let_go;
+	unsigned long addr, pgd_end;
 	int isfile;
 	int isclustermap;
 	int isprivate;
+	int error = 0;
 
 	asip->as_total_pg = 0;
 	asip->as_total_vma = 0;
-	if (!run_over_maxchunk) {
-		/* this routine can take very long for huge processes:
-		 * we must allow other processes to take over at least
-		 * about every millisecond.
-		 */
-		run_over_maxchunk = loops_per_jiffy / RUN_OVER_MAGIC;
-		if (!run_over_maxchunk)	/* slow processors */
-			run_over_maxchunk = 1;
-	}
-	p = current;
-	mm = p->mm;
-	addr = 0;
-	start = 0;
- loop:
-	let_go = run_over_maxchunk;
+	mm = current->mm;
+
 	down_read(&mm->mmap_sem);
-	spin_lock(&mm->page_table_lock);
 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
-		if (addr >= vma->vm_end)
-			continue;
-		start = vma->vm_start;
-		if (start < addr)
-			start = addr;
 		if (op == ASX_RFORK) {
 			if (vma->vm_flags & VM_DONTCOPY)
 				continue;
-		} else {
-			if (vma->vm_flags & VM_LOCKED) {
-				error = -EBUSY;
-				goto out_unlock;
-			}
+		} else if (vma->vm_flags & VM_LOCKED) {
+			error = -EBUSY;
+			break;
 		}
 		isfile = as_vma_is_file(vma);
 		isclustermap = as_vma_is_clustermap(vma);
@@ -556,32 +569,34 @@
 		SSI_ASSERT(isprivate || isfile);
 		if (!isprivate && (!isfile || !isclustermap)) {
 			error = -EBUSY;
-			goto out_unlock;
+			break;
 		}
-		pgdir = pgd_offset(mm, start);
-		for (addr = start; addr < vma->vm_end;
-		     addr = pgd_end, pgdir++) {
-			if (!let_go--) {
-				spin_unlock(&mm->page_table_lock);
-				up_read(&mm->mmap_sem);
-				/* yield(); */
-				goto loop;
-			}
+
+		addr = vma->vm_start;
+		pgdir = pgd_offset(mm, addr);
+
+		spin_lock(&mm->page_table_lock);
+		for (; addr < vma->vm_end; addr = pgd_end, pgdir++) {
+			cond_resched_lock(&mm->page_table_lock);
+
 			asx_hist(2, asip->as_pid, addr, vma, pgdir);
 			pgd_end = (addr + PGDIR_SIZE) & PGDIR_MASK;
-			if (pgd_end > vma->vm_end)
+			if (pgd_end > vma->vm_end || pgd_end <= addr)
 				pgd_end = vma->vm_end;
 			if (pgd_none(*pgdir) || pgd_bad(*pgdir))
 				continue;
+
 			error = as_count_pgdir(asip, vma, isprivate,
 					       &addr, pgdir, pgd_end);
-			if (error < 0)
+			if (error < 0) {
+				spin_unlock(&mm->page_table_lock);
 				goto out_unlock;
+			}
 		}
+		spin_unlock(&mm->page_table_lock);
 		asip->as_total_vma++;
 	}
- out_unlock:
-	spin_unlock(&mm->page_table_lock);
+out_unlock:
 	up_read(&mm->mmap_sem);
 
 	return error;
@@ -645,8 +660,9 @@
 	mm = p->mm;
 #endif /* !TASK_HOLD_VPROC */
 	*error = 0;
-	down_read(&mm->mmap_sem);
 	datasize = 0;
+
+	down_read(&mm->mmap_sem);
 	for (vma = mm->mmap;
 	     vma && avap->ava_data.ava_data_len < AS_VMA_MAX;
 	     vma = vma->vm_next) {
@@ -711,47 +727,67 @@
 	return 0;
 }
 
-static inline int
-as_pull_pgdir(pid_t pid, as_pg_info *apip, int *apipp_len,
+static int
+as_pull_pudir(pid_t pid, as_pg_info *apip, int *apipp_len,
 	      struct mm_struct *mm, struct vm_area_struct *vma,
 	      int isprivate, unsigned long *addr,
-	      pgd_t *pgdir, unsigned long pgd_end)
+	      pud_t *pudir, unsigned long pud_end)
 {
 	int error = 0;
-	pud_t *pudir;
 	pmd_t *pmdir;
 	pte_t *pte;
 	pte_t *pte_dir;
 	unsigned long pmd_end;
-	unsigned long pte_end;
 	struct page *page;
 
-	pudir = pud_offset(pgdir, *addr);
+	/* See Linux get_user_pages() */
+#ifdef SSI_HUGETLB_PAGE
+	page = follow_huge_addr(mm, *addr, 0);
+	if (! IS_ERR(page)) {
+		*addr += PAGE_SIZE;
+		goto pull_page;
+	}
+#endif
+
 	pmdir = pmd_offset(pudir, *addr);
-	for(; *addr < pgd_end; *addr = pmd_end, pmdir++) {
+	for(; *addr < pud_end; *addr = pmd_end, pmdir++) {
 		pmd_end = (*addr + PMD_SIZE) & PMD_MASK;
-		if (pmd_end > pgd_end)
-			pmd_end = pgd_end;
+		if (pmd_end > pud_end || pmd_end <= *addr)
+			pmd_end = pud_end;
 		if (pmd_none(*pmdir) || pmd_bad(*pmdir))
 			continue;
+#ifdef SSI_HUGETLB_PAGE
+		if (pmd_huge(*pmdir)) {
+			struct page *map;
+			map = follow_huge_pmd(mm, *addr, pmdir, 0);
+			page = get_page_map(map);
+			if (!page)
+				return -EFAULT;
+			goto pull_page;
+		}
+#endif
 		pte = pte_dir = pte_offset_map(pmdir, *addr);
-		for(; *addr < pmd_end; *addr = pte_end, pte++) {
-			pte_end = *addr + PAGE_SIZE;
+		for(; *addr < pmd_end; *addr += PAGE_SIZE, pte++) {
 			asx_hist(3, pid, *addr, vma, pte);
 			if (!pte_present(*pte)) {
 				if (pte_none(*pte))
 					continue;
+#if defined(CONFIG_HIGHPTE)
+				if (pte != pte_dir) {
+					pte_unmap(pte_dir);
+					pte = pte_dir =
+						pte_offset_map(pmdir, *addr);
+				}
+#endif
 				while (1) {
 					error = !do_swap_page(mm, vma, *addr,
 							      pte, pmdir, *pte,
 							      0);
 					spin_lock(&mm->page_table_lock);
+					if (error)
+						return -EBUSY;
 					pte = pte_dir =
 						pte_offset_map(pmdir, *addr);
-					if (error) {
-						error = -EBUSY;
-						goto out;
-					}
 					if (pte_present(*pte) ||
 					    pte_none(*pte))
 						break;
@@ -761,29 +797,30 @@
 			}
 			if (!pfn_valid(pte_pfn(*pte))) {
 				error = -EBUSY;
-				goto out;
+				break;
 			}
 			page = pte_page(*pte);
+			flush_dcache_page(page);
 			if (PageReserved(page)) {
 				if (page != ZERO_PAGE(*addr)) {
 					error = -EBUSY;
-					goto out;
-				} else
-					continue;
+					break;
+				}
+				continue;
 			}
 			if (!isprivate || !PageAnon(page))
 				continue;
 			page_cache_get(page);
 			asx_hist(4, pid, *addr, vma, pte);
+
 			apip[(*apipp_len)].api_addr = *addr;
 			apip[(*apipp_len)++].api_page = page;
 			if (*apipp_len >= AS_PAGE_MAX) {
-				*addr = pte_end;
+				*addr += PAGE_SIZE;
 				error = 1;
-				goto out;
+				break;
 			}
 		}
- out:
 		pte_unmap(pte_dir);
 		if (error)
 			break;
@@ -792,23 +829,48 @@
 	return error;
 }
 
+/*
+ * We hold the mm semaphore and the page_table_lock on entry.
+ */
+static int
+as_pull_pgdir(pid_t pid, as_pg_info *apip, int *apipp_len,
+	      struct mm_struct *mm, struct vm_area_struct *vma,
+	      int isprivate, unsigned long *addr,
+	      pgd_t *pgdir, unsigned long pgd_end)
+{
+	pud_t *pudir;
+	unsigned long pud_end;
+	int error = 0;
+
+	pudir = pud_offset(pgdir, *addr);
+	for(; *addr < pgd_end && !error; *addr = pud_end, pudir++) {
+		pud_end = (*addr + PUD_SIZE) & PUD_MASK;
+		if (pud_end > pgd_end || pud_end <= *addr)
+			pud_end = pgd_end;
+		if (pud_none(*pudir) || pud_bad(*pudir))
+			continue;
+		error = as_pull_pudir(pid, apip, apipp_len,
+				       mm, vma, isprivate,
+				       addr, pudir, pud_end);
+		if (error)
+			break;
+	}
+	return error;
+}
+
 int
 ras_pull_pg_info(clusternode_t node, int *error, pid_t pid,
 		 enum asx_op op, u_long *addr, as_pg_info **apipp,
 		 int *apipp_len)
 {
-	struct mm_struct *mm = NULL;
 	struct vm_area_struct *vma;
+	struct mm_struct *mm = NULL;
 	struct task_struct *p;
 	struct vproc *v = NULL;
 	as_pg_info *apip;
-	int isfile;
-	int isclustermap;
-	int isprivate;
-	unsigned long start;
-	unsigned long pgd_end;
 	pgd_t *pgdir;
-	int let_go;
+	unsigned long start, pgd_end;
+	int isfile, isclustermap, isprivate;
 	int i;
 
 	asx_hist2(7, pid, (short)__LINE__,  *addr, 0, 0, 0, 0);
@@ -855,17 +917,11 @@
 	mm = p->mm;
 #endif /* !TASK_HOLD_VPROC */
 	*error = 0;
-	start = 0;
- loop:
-	let_go = run_over_maxchunk;
+
 	down_read(&mm->mmap_sem);
-	//spin_lock(&mm->page_table_lock);
 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 		if (*addr >= vma->vm_end)
 			continue;
-		start = vma->vm_start;
-		if (start < *addr)
-			start = *addr;
 		isfile = as_vma_is_file(vma);
 		isclustermap = as_vma_is_clustermap(vma);
 		isprivate = as_vma_is_private(vma);
@@ -875,22 +931,23 @@
 			goto out_unlock;
 		}
 
-		spin_lock(&mm->page_table_lock);
+		start = vma->vm_start;
+		if (start < *addr)
+			start = *addr;
 		pgdir = pgd_offset(mm, start);
+
+		spin_lock(&mm->page_table_lock);
 		for (*addr = start; *addr < vma->vm_end;
 		     *addr = pgd_end, pgdir++) {
-			if (!let_go--) {
-				spin_unlock(&mm->page_table_lock);
-				up_read(&mm->mmap_sem);
-				/* yield(); */
-				goto loop;
-			}
+			cond_resched_lock(&mm->page_table_lock);
+
 			asx_hist(5, pid, *addr, vma, pgdir);
 			pgd_end = (*addr + PGDIR_SIZE) & PGDIR_MASK;
-			if (pgd_end > vma->vm_end)
+			if (pgd_end > vma->vm_end || pgd_end <= *addr)
 				pgd_end = vma->vm_end;
 			if (pgd_none(*pgdir) || pgd_bad(*pgdir))
 				continue;
+
 			*error = as_pull_pgdir(pid, apip, apipp_len,
 					       mm, vma, isprivate,
 					       addr, pgdir, pgd_end);
@@ -902,7 +959,6 @@
 		spin_unlock(&mm->page_table_lock);
 	}
  out_unlock:
-	//spin_unlock(&mm->page_table_lock);
 	up_read(&mm->mmap_sem);
  out:
 #ifdef TASK_HOLD_VPROC
@@ -974,14 +1030,23 @@
 		error = -EINVAL;
 		goto out_unlock_sem;
 	}
+
+	/*
+	 * Shameless dupe of base Linux-2.6.11
+	 * install_arg_page(vma, apip->api_page, apip->api_addr);
+	 */
 	if (unlikely(anon_vma_prepare(vma))) {
 		error = -EAGAIN;
 		goto out_unlock_sem;
 	}
 	flush_dcache_page(apip->api_page);
 	pgd = pgd_offset(mm, apip->api_addr);
-	pud = pud_offset(pgd, apip->api_addr);
+
 	spin_lock(&mm->page_table_lock);
+	if (!(pud = pud_alloc(mm, pgd, apip->api_addr))) {
+		error = -EAGAIN;
+		goto out_unlock_spin;
+	}
 	if (!(pmd = pmd_alloc(mm, pud, apip->api_addr))) {
 		error = -EAGAIN;
 		goto out_unlock_spin;
@@ -1059,17 +1124,20 @@
 	if (avip->avi_flags & VM_EXEC)
 		prot |= VM_EXEC;
 	len = avip->avi_end - avip->avi_start;
+
 	down_write(&current->mm->mmap_sem);
+
 	error = do_mmap_pgoff(file, avip->avi_start, len, prot, flags, pgoff);
 	if (IS_ERR((void *)error)) {
 		up_write(&current->mm->mmap_sem);
 		goto out;
 	}
+
 	vma = find_vma(current->mm, avip->avi_start);
 	SSI_ASSERT(vma);
 	if (!vma) {
-		error = -EINVAL;
 		up_write(&current->mm->mmap_sem);
+		error = -EINVAL;
 		goto out;
 	}
 	SSI_ASSERT(vma->vm_start == avip->avi_start);
@@ -1077,6 +1145,7 @@
 	vma->vm_flags = avip->avi_flags;
 	SSI_ASSERT(vma->vm_pgoff == pgoff || !avip->avi_file);
 	vma->vm_pgoff = avip->avi_pgoff;
+
 	up_write(&current->mm->mmap_sem);
 	error = 0;
  out:
@@ -1097,6 +1166,7 @@
 	asip->as_node = this_node;
 	asip->as_pid = p->pid;
 	asip->as_op = op;
+
 	error = as_count_mm_info(asip, op);
 	if (error)
 		return error;


------------------------------------------------------------------------------
Download Intel&#174; Parallel Studio Eval
Try the new software tools for yourself. Speed compiling, find bugs
proactively, and fine-tune applications for parallel performance.
See why Intel Parallel Studio got high marks during beta.
http://p.sf.net/sfu/intel-sw-dev