Patch for port of jobserver to Windows

Troy Runkel <[email protected]>
Newsgroups gmane.comp.gnu.make.windows
Message-ID <B96949E43B9A8B44904810DCCE1DA23FBB46DAB985@EXCHANGE-AH.ad.mathworks.com>
I've ported the GNU make jobserver to the Windows platform.  The port uses a Windows named semaphore to implement the jobserver.  I'd like to submit my diffs for consideration and inclusion in the main GNU make branch.  Please let me know if I missed anything in my changes.  Thanks.

--Troy Runkel

_______________________________________________
Make-w32 mailing list
[email protected]
http://lists.gnu.org/mailman/listinfo/make-w32
windows_jobserver.diff (application/octet-stream, 16 KB)
diff -ru make-3.82-orig/config.h.W32 make-3.82/config.h.W32
--- make-3.82-orig/config.h.W32	2010-10-07 14:51:29.697315000 -0400
+++ make-3.82/config.h.W32	2010-10-07 14:53:32.535320000 -0400
@@ -332,7 +332,7 @@
 #define MAKE_HOST "Windows32"
 
 /* Define this to enable job server support in GNU make. */
-/* #undef MAKE_JOBSERVER */
+#define MAKE_JOBSERVER 1
 
 /* Define to 1 if your `struct nlist' has an `n_un' member. Obsolete, depend
    on `HAVE_STRUCT_NLIST_N_UN_N_NAME */
diff -ru make-3.82-orig/job.c make-3.82/job.c
--- make-3.82-orig/job.c	2010-10-07 14:51:30.081244000 -0400
+++ make-3.82/job.c	2010-10-07 15:02:43.566274000 -0400
@@ -108,6 +108,7 @@
 # include "sub_proc.h"
 # include "w32err.h"
 # include "pathstuff.h"
+# define WAIT_NOHANG 1
 #endif /* WINDOWS32 */
 
 #ifdef __EMX__
@@ -509,9 +510,10 @@
 {
 #ifndef WINDOWS32
   WAIT_T status;
+#endif
+
   /* Initially, assume we have some.  */
   int reap_more = 1;
-#endif
 
 #ifdef WAIT_NOHANG
 # define REAP_MORE reap_more
@@ -680,6 +682,7 @@
             HANDLE hPID;
             int werr;
             HANDLE hcTID, hcPID;
+            DWORD dwWaitStatus = 0;
             exit_code = 0;
             exit_sig = 0;
             coredump = 0;
@@ -703,7 +706,7 @@
               }
 
             /* wait for anything to finish */
-            hPID = process_wait_for_any();
+            hPID = process_wait_for_any(block, &dwWaitStatus);
             if (hPID)
               {
 
@@ -725,6 +728,18 @@
 
                 coredump = 0;
               }
+            else if (dwWaitStatus == WAIT_FAILED)
+              {
+                /* The WaitForMultipleObjects() failed miserably.  Punt.  */
+                pfatal_with_name ("WaitForMultipleObjects");
+              }
+            else if (dwWaitStatus == WAIT_TIMEOUT)
+              {
+                /* No child processes are finished.  Give up waiting. */
+                reap_more = 0;
+                break;
+              }
+
             pid = (pid_t) hPID;
           }
 #endif /* WINDOWS32 */
@@ -908,6 +923,15 @@
   /* If we're using the jobserver and this child is not the only outstanding
      job, put a token back into the pipe for it.  */
 
+#ifdef WINDOWS32
+  if (has_jobserver_semaphore() && jobserver_tokens > 1)
+    {
+      if (! release_jobserver_semaphore())
+          pfatal_with_name (_("release jobserver semaphore"));
+
+      DB (DB_JOBS, (_("Released token for child %p (%s).\n"), child, child->file->name));
+    }
+#else
   if (job_fds[1] >= 0 && jobserver_tokens > 1)
     {
       char token = '+';
@@ -922,6 +946,7 @@
       DB (DB_JOBS, (_("Released token for child %p (%s).\n"),
                     child, child->file->name));
     }
+#endif
 
   --jobserver_tokens;
 
@@ -973,7 +998,7 @@
 }
 #endif
 
-#ifdef MAKE_JOBSERVER
+#if defined(MAKE_JOBSERVER) && !defined(WINDOWS32)
 RETSIGTYPE
 job_noop (int sig UNUSED)
 {
@@ -1706,7 +1731,11 @@
      just once).  Also more thought needs to go into the entire algorithm;
      this is where the old parallel job code waits, so...  */
 
+#ifdef WINDOWS32
+  else if (has_jobserver_semaphore())
+#else
   else if (job_fds[0] >= 0)
+#endif
     while (1)
       {
         char token;
@@ -1720,6 +1749,7 @@
         if (!jobserver_tokens)
           break;
 
+#ifndef WINDOWS32
         /* Read a token.  As long as there's no token available we'll block.
            We enable interruptible system calls before the read(2) so that if
            we get a SIGCHLD while we're waiting, we'll return with EINTR and
@@ -1748,7 +1778,7 @@
             DB (DB_JOBS, ("Duplicate the job FD\n"));
             job_rfd = dup (job_fds[0]);
           }
-
+#endif
         /* Reap anything that's currently waiting.  */
         reap_children (0, 0);
 
@@ -1766,12 +1796,20 @@
         if (!children)
           fatal (NILF, "INTERNAL: no children as we go to sleep on read\n");
 
+#ifdef WINDOWS32
+        /* On Windows we simply wait for the jobserver semaphore to become
+         * signalled or one of our child processes to terminate.
+         */
+        got_token = wait_for_semaphore_or_child_process();
+        if (got_token < 0)
+            pfatal_with_name (_("semaphore or child process wait"));
+#else
         /* Set interruptible system calls, and read() for a job token.  */
 	set_child_handler_action_flags (1, waiting_jobs != NULL);
 	got_token = read (job_rfd, &token, 1);
 	saved_errno = errno;
 	set_child_handler_action_flags (0, waiting_jobs != NULL);
-
+#endif
         /* If we got one, we're done here.  */
 	if (got_token == 1)
           {
@@ -1780,6 +1818,7 @@
             break;
           }
 
+#ifndef WINDOWS32
         /* If the error _wasn't_ expected (EINTR or EBADF), punt.  Otherwise,
            go back and reap_children(), and try again.  */
 	errno = saved_errno;
@@ -1787,6 +1826,7 @@
           pfatal_with_name (_("read jobs pipe"));
         if (errno == EBADF)
           DB (DB_JOBS, ("Read returned EBADF.\n"));
+#endif
       }
 #endif
 
@@ -2100,7 +2140,7 @@
     }
 
   /* wait and reap last child */
-  hWaitPID = process_wait_for_any();
+  hWaitPID = process_wait_for_any(1, 0);
   while (hWaitPID)
     {
       /* was an error found on this process? */
diff -ru make-3.82-orig/main.c make-3.82/main.c
--- make-3.82-orig/main.c	2010-10-07 14:51:30.089273000 -0400
+++ make-3.82/main.c	2010-10-07 15:11:11.962385000 -0400
@@ -35,6 +35,7 @@
 #include <windows.h>
 #include <io.h>
 #include "pathstuff.h"
+#include "sub_proc.h"
 #endif
 #ifdef __EMX__
 # include <sys/types.h>
@@ -1722,13 +1723,19 @@
 
       cp = jobserver_fds->list[0];
 
+#ifdef WINDOWS32
+      if (! open_jobserver_semaphore(cp))
+        fatal (NILF,_("internal error: unable to open jobserver semaphore `%s'"), cp);
+
+      DB (DB_JOBS, (_("Jobserver client (semaphore %s)\n"), cp));
+#else
       if (sscanf (cp, "%d,%d", &job_fds[0], &job_fds[1]) != 2)
         fatal (NILF,
                _("internal error: invalid --jobserver-fds string `%s'"), cp);
 
       DB (DB_JOBS,
           (_("Jobserver client (fds %d,%d)\n"), job_fds[0], job_fds[1]));
-
+#endif
       /* The combination of a pipe + !job_slots means we're using the
          jobserver.  If !job_slots and we don't have a pipe, we can start
          infinite jobs.  If we see both a pipe and job_slots >0 that means the
@@ -1739,6 +1746,7 @@
         error (NILF,
                _("warning: -jN forced in submake: disabling jobserver mode."));
 
+#ifndef WINDOWS32
       /* Create a duplicate pipe, that will be closed in the SIGCHLD
          handler.  If this fails with EBADF, the parent has closed the pipe
          on us because it didn't think we were a submake.  If so, print a
@@ -1753,11 +1761,13 @@
                  _("warning: jobserver unavailable: using -j1.  Add `+' to parent make rule."));
           job_slots = 1;
         }
-
+#endif
       if (job_slots > 0)
         {
+#ifndef WINDOWS32
           close (job_fds[0]);
           close (job_fds[1]);
+#endif
           job_fds[0] = job_fds[1] = -1;
           free (jobserver_fds->list);
           free (jobserver_fds);
@@ -1773,8 +1783,13 @@
       char *cp;
       char c = '+';
 
+#ifdef WINDOWS32
+      if (! create_jobserver_semaphore(job_slots - 1))
+        pfatal_with_name (_("creating jobserver semaphore"));
+#else
       if (pipe (job_fds) < 0 || (job_rfd = dup (job_fds[0])) < 0)
 	pfatal_with_name (_("creating jobs pipe"));
+#endif
 
       /* Every make assumes that it always has one job it can run.  For the
          submakes it's the token they were given by their parent.  For the
@@ -1783,6 +1798,10 @@
 
       master_job_slots = job_slots;
 
+#ifdef WINDOWS32
+      /* We're using the jobserver so set job_slots to 0. */
+      job_slots = 0;
+#else
       while (--job_slots)
         {
           int r;
@@ -1791,11 +1810,17 @@
           if (r != 1)
             pfatal_with_name (_("init jobserver pipe"));
         }
+#endif
 
       /* Fill in the jobserver_fds struct for our children.  */
 
+#ifdef WINDOWS32
+      cp = xmalloc (MAX_PATH + 1);
+      strcpy(cp, get_jobserver_semaphore_name());
+#else
       cp = xmalloc ((sizeof ("1024")*2)+1);
       sprintf (cp, "%d,%d", job_fds[0], job_fds[1]);
+#endif
 
       jobserver_fds = (struct stringlist *)
                         xmalloc (sizeof (struct stringlist));
@@ -3106,7 +3131,11 @@
      have written all our tokens so do that now.  If tokens are left
      after any other error code, that's bad.  */
 
+#ifdef WINDOWS32
+  if (has_jobserver_semaphore() && jobserver_tokens)
+#else
   if (job_fds[0] != -1 && jobserver_tokens)
+#endif
     {
       if (status != 2)
         error (NILF,
@@ -3115,11 +3144,16 @@
       else
         while (jobserver_tokens--)
           {
+#ifdef WINDOWS32
+            if (! release_jobserver_semaphore())
+              perror_with_name ("release_jobserver_semaphore", "");
+#else
             int r;
 
             EINTRLOOP (r, write (job_fds[1], &token, 1));
             if (r != 1)
               perror_with_name ("write", "");
+#endif
           }
     }
 
@@ -3131,19 +3165,26 @@
       /* We didn't write one for ourself, so start at 1.  */
       unsigned int tcnt = 1;
 
+#ifdef WINDOWS32
+      while (acquire_jobserver_semaphore())
+          ++tcnt;
+#else
       /* Close the write side, so the read() won't hang.  */
       close (job_fds[1]);
 
       while (read (job_fds[0], &token, 1) == 1)
         ++tcnt;
-
+#endif
       if (tcnt != master_job_slots)
         error (NILF,
                "INTERNAL: Exiting with %u jobserver tokens available; should be %u!",
                tcnt, master_job_slots);
 
+#ifdef WINDOWS32
+      free_jobserver_semaphore();
+#else
       close (job_fds[0]);
-
+#endif
       /* Clean out jobserver_fds so we don't pass this information to any
          sub-makes.  Also reset job_slots since it will be put on the command
          line, not in MAKEFLAGS.  */
diff -ru make-3.82-orig/w32/include/sub_proc.h make-3.82/w32/include/sub_proc.h
--- make-3.82-orig/w32/include/sub_proc.h	2010-10-07 14:51:31.390049000 -0400
+++ make-3.82/w32/include/sub_proc.h	2010-10-07 14:53:33.597315000 -0400
@@ -40,7 +40,7 @@
 	int stdin_data_len));
 EXTERN_DECL(long process_file_io, (HANDLE proc));
 EXTERN_DECL(void process_cleanup, (HANDLE proc));
-EXTERN_DECL(HANDLE process_wait_for_any, (VOID_DECL));
+EXTERN_DECL(HANDLE process_wait_for_any, (int block, DWORD* pdwWaitStatus));
 EXTERN_DECL(void process_register, (HANDLE proc));
 EXTERN_DECL(HANDLE process_easy, (char** argv, char** env));
 EXTERN_DECL(BOOL process_kill, (HANDLE proc, int signal));
@@ -57,4 +57,14 @@
 EXTERN_DECL(int process_errcnt, (HANDLE proc));
 EXTERN_DECL(void process_pipes, (HANDLE proc, int pipes[3]));
 
+/* jobserver routines */
+EXTERN_DECL(int open_jobserver_semaphore, (char* name));
+EXTERN_DECL(int create_jobserver_semaphore, (int tokens));
+EXTERN_DECL(void free_jobserver_semaphore, (VOID_DECL));
+EXTERN_DECL(int acquire_jobserver_semaphore, (VOID_DECL));
+EXTERN_DECL(int release_jobserver_semaphore, (VOID_DECL));
+EXTERN_DECL(int has_jobserver_semaphore, (VOID_DECL));
+EXTERN_DECL(char* get_jobserver_semaphore_name, (VOID_DECL));
+EXTERN_DECL(int wait_for_semaphore_or_child_process, (VOID_DECL));
+
 #endif
diff -ru make-3.82-orig/w32/subproc/sub_proc.c make-3.82/w32/subproc/sub_proc.c
--- make-3.82-orig/w32/subproc/sub_proc.c	2010-10-07 14:51:31.419082000 -0400
+++ make-3.82/w32/subproc/sub_proc.c	2010-10-07 14:53:33.626209000 -0400
@@ -58,6 +58,126 @@
 static int proc_index = 0;
 static int fake_exits_pending = 0;
 
+/* Windows jobserver implementation variables */
+static char jobserver_semaphore_name[MAX_PATH + 1];
+static HANDLE jobserver_semaphore = NULL;
+
+/* Open existing jobserver semaphore */
+int open_jobserver_semaphore(char* name)
+{
+    jobserver_semaphore = OpenSemaphore(
+        SEMAPHORE_ALL_ACCESS,	// Semaphore access setting
+        FALSE,			// Child processes DON'T inherit
+        name);			// Semaphore name
+
+    if (jobserver_semaphore == NULL)
+        return 0;
+
+    return 1;        
+}
+
+/* Create new jobserver semaphore */
+int create_jobserver_semaphore(int tokens)
+{
+    sprintf(jobserver_semaphore_name, "gmake_semaphore_%d", _getpid());
+
+    jobserver_semaphore = CreateSemaphore(
+        NULL, 				// Use default security descriptor
+        tokens,				// Initial count
+        tokens, 			// Maximum count
+        jobserver_semaphore_name);	// Semaphore name
+
+    if (jobserver_semaphore == NULL)
+        return 0;
+
+    return 1;        
+}
+
+/* Close jobserver semaphore */
+void free_jobserver_semaphore()
+{
+    if (jobserver_semaphore != NULL)
+    {
+        CloseHandle(jobserver_semaphore);
+        jobserver_semaphore = NULL;
+    }
+}
+
+/* Decrement semaphore count */
+int acquire_jobserver_semaphore()
+{
+    DWORD dwEvent = WaitForSingleObject(
+        jobserver_semaphore,	// Handle to semaphore
+        0);			// DON'T wait on semaphore
+
+    return (dwEvent == WAIT_OBJECT_0);
+}
+
+/* Increment semaphore count */
+int release_jobserver_semaphore()
+{
+    BOOL bResult = ReleaseSemaphore( 
+        jobserver_semaphore,  	// handle to semaphore
+        1,            		// increase count by one
+        NULL);        		// not interested in previous count
+
+    return (bResult);
+}
+
+int has_jobserver_semaphore()
+{
+    return (jobserver_semaphore != NULL);
+}
+
+char* get_jobserver_semaphore_name()
+{
+    return (jobserver_semaphore_name);
+}
+
+/* Wait for either the jobserver semaphore to become signalled or one of our
+ * child processes to terminate.
+ */
+int wait_for_semaphore_or_child_process()
+{
+    HANDLE handles[MAXIMUM_WAIT_OBJECTS];
+    DWORD dwHandleCount = 1;
+    DWORD dwEvent;
+    int i;
+
+    /* Add jobserver semaphore to first slot. */
+    handles[0] = jobserver_semaphore;
+
+    /* Build array of handles to wait for */
+    for (i = 0; i < proc_index; i++) 
+    {
+        /* Don't wait on child processes that have already finished */
+        if (fake_exits_pending && proc_array[i]->exit_code)
+            continue;
+
+        handles[dwHandleCount++] = (HANDLE) proc_array[i]->pid;
+    }
+
+    dwEvent = WaitForMultipleObjects( 
+        dwHandleCount,	// number of objects in array
+        handles,	// array of objects
+        FALSE,		// wait for any object
+        INFINITE);	// wait until object is signalled
+
+    switch(dwEvent)
+    {
+      case WAIT_FAILED:
+        return -1;
+
+      case WAIT_OBJECT_0:
+        /* Indicate that the semaphore was signalled */
+        return 1;
+
+      default:
+        /* Assume that one or more of the child processes terminated. */
+        return 0;
+    }
+}
+
 /*
  * When a process has been waited for, adjust the wait state
  * array so that we don't wait for it again
@@ -87,7 +207,7 @@
  * Waits for any of the registered child processes to finish.
  */
 static sub_process *
-process_wait_for_any_private(void)
+process_wait_for_any_private(int block, DWORD* pdwWaitStatus)
 {
 	HANDLE handles[MAXIMUM_WAIT_OBJECTS];
 	DWORD retval, which;
@@ -106,7 +226,7 @@
 
 	/* wait for someone to exit */
 	if (!fake_exits_pending) {
-		retval = WaitForMultipleObjects(proc_index, handles, FALSE, INFINITE);
+		retval = WaitForMultipleObjects(proc_index, handles, FALSE, (block ? INFINITE : 0));
 		which = retval - WAIT_OBJECT_0;
 	} else {
 		fake_exits_pending--;
@@ -114,13 +234,19 @@
 		which = i;
 	}
 
+        /* If the pointer is not NULL, set the wait status result variable. */
+        if (pdwWaitStatus)
+            *pdwWaitStatus = retval;
+
 	/* return pointer to process */
-	if (retval != WAIT_FAILED) {
+        if ((retval == WAIT_TIMEOUT) || (retval == WAIT_FAILED)) {
+		return NULL;
+        }
+	else {
 		sub_process* pproc = proc_array[which];
 		process_adjust_wait_state(pproc);
 		return pproc;
-	} else
-		return NULL;
+	} 
 }
 
 /*
@@ -179,9 +305,9 @@
  */
 
 HANDLE
-process_wait_for_any(void)
+process_wait_for_any(int block, DWORD* pdwWaitStatus)
 {
-	sub_process* pproc = process_wait_for_any_private();
+	sub_process* pproc = process_wait_for_any_private(block, pdwWaitStatus);
 
 	if (!pproc)
 		return NULL;
@@ -865,7 +991,7 @@
         DWORD ierr;
 
 	if (proc == NULL)
-		pproc = process_wait_for_any_private();
+		pproc = process_wait_for_any_private(1, 0);
 	else
 		pproc = (sub_process *)proc;
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.