Re: Possible deadlocking between jcifs.smb.Dfs and jcifs.smb.SmbTransport

On 03/05/2012 05:02 PM, Michael B Allen wrote:
> Can you please create a small torture test program that triggers the
> deadlock and post it?

Sure -- here is a minimal test case (the code is a bit messy, but should 
not have any thread-unsafe calls).

Please note that the deadlock may be difficule to reproduce depending on 
the environment. We typically run it during few minutes, and get:

** Deadlock detected! **
The following thread:
Thread #3648 (Transport5) group="main" state="BLOCKED" priority="5" 
daemon="true" alive="true" interrupted="false" 
classLoader="sun.misc.Launcher$AppClassLoader"
         jcifs.util.transport.Transport.loop(Transport.java:99)
         jcifs.util.transport.Transport.run(Transport.java:267)
         java.lang.Thread.run(Thread.java:662)

Thread #12 (Thread-4) group="main" state="BLOCKED" priority="5" 
daemon="false" alive="true" interrupted="false" 
classLoader="sun.misc.Launcher$AppClassLoader"
         jcifs.smb.Dfs.insert(Dfs.java:289)
         jcifs.smb.SmbTransport.checkStatus(SmbTransport.java:556)
         jcifs.smb.SmbTransport.send(SmbTransport.java:640)
         jcifs.smb.SmbSession.send(SmbSession.java:238)
         jcifs.smb.SmbTree.send(SmbTree.java:119)
         jcifs.smb.SmbFile.send(SmbFile.java:775)
         jcifs.smb.SmbFile.queryPath(SmbFile.java:1363)
         jcifs.smb.SmbFile.exists(SmbFile.java:1417)
         jcifs.smb.SmbFile.isDirectory(SmbFile.java:1490)
 
test.deadlock.SmbStressTest$FileScanner.scanFiles(SmbStressTest.java:79)
         test.deadlock.SmbStressTest$FileScanner.run(SmbStressTest.java:39)

has been blocked for 0s by:
The following thread:
Thread #12 (Thread-4) group="main" state="BLOCKED" priority="5" 
daemon="false" alive="true" interrupted="false" 
classLoader="sun.misc.Launcher$AppClassLoader"
         jcifs.smb.Dfs.insert(Dfs.java:289)
         jcifs.smb.SmbTransport.checkStatus(SmbTransport.java:556)
         jcifs.smb.SmbTransport.send(SmbTransport.java:640)
         jcifs.smb.SmbSession.send(SmbSession.java:238)
         jcifs.smb.SmbTree.send(SmbTree.java:119)
         jcifs.smb.SmbFile.send(SmbFile.java:775)
         jcifs.smb.SmbFile.queryPath(SmbFile.java:1363)
         jcifs.smb.SmbFile.exists(SmbFile.java:1417)
         jcifs.smb.SmbFile.isDirectory(SmbFile.java:1490)
 
test.deadlock.SmbStressTest$FileScanner.scanFiles(SmbStressTest.java:79)
         test.deadlock.SmbStressTest$FileScanner.run(SmbStressTest.java:39)


Code:
----

package test.deadlock;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.util.Map;
import java.util.Vector;

import jcifs.Config;
import jcifs.smb.ACE;
import jcifs.smb.NtlmPasswordAuthentication;
import jcifs.smb.SmbFile;

/*
  * Stress test for Dfs/Transport deadlock.
  */
public class SmbStressTest {
     protected static final int filesMax = 1000;
     protected static final int maxThreads = 8;

     protected final Vector<String> files = new Vector<String>();
     protected boolean exit = false;

     protected synchronized static void debug(String message) {
         System.err.println(message);
     }

     /*
      * Runner, scanning all directories.
      */
     protected class FileScanner extends Thread {
         @Override
         public void run() {
             try {
                 scanFiles();
             } catch (Exception e) {
                 e.printStackTrace();
             }
             debug("worker thread ended");
         }

         protected void scanFiles() {
             for (;;) {
                 String itemUrl;

                 // Fetch next directory.
                 synchronized (SmbStressTest.this) {
                     // Wait for queue to be not empty.
                     debug("waiting for a file to be available");
                     while (!exit && files.size() == 0) {
                         try {
                             SmbStressTest.this.wait();
                         } catch (InterruptedException e) {
                         }
                     }
                     if (exit) {
                         return;
                     }

                     // Get a random item.
                     itemUrl = files.get((int) (files.size() * 
Math.random()));
                     // SmbStressTest.this.notifyAll();
                     debug("got file: " + url);
                 }

                 // Let's look at this file
                 try {
                     // Recreate file by scanning each segment to 
reproduce the
                     // issue more easily.
                     final String[] segments = 
itemUrl.substring(url.length()).split("/");
                     SmbFile smbFile = new SmbFile(url, new 
NtlmPasswordAuthentication(domain, username, password),
                             SmbFile.FILE_SHARE_READ);
                     for (int i = 0; i < segments.length; i++) {
                         if (i + 1 < segments.length) {
                             smbFile = new SmbFile(smbFile, segments[i] 
+ "/", SmbFile.FILE_SHARE_READ);
                             if (!smbFile.isDirectory()) {
                                 throw new 
IllegalStateException("unexpected file here");
                             }
                         } else {
                             smbFile = new SmbFile(smbFile, segments[i], 
SmbFile.FILE_SHARE_READ);
                             if (smbFile.isDirectory()) {
                                 throw new 
IllegalStateException("unexpected directory here");
                             }
                         }
                         if (!smbFile.exists()) {
                             throw new IllegalStateException("unexpected 
file not found: " + smbFile.getPath());
                         }
                     }

                     final String path = smbFile.getPath();
                     if (smbFile.isDirectory()) {
                         System.err.println(path + " is a directory");
                     } else if (smbFile.isFile()) {
                         System.err.println(path + " is a file");
                     }

                     // Fetch some ACLs
                     for (ACE ace : smbFile.getSecurity(false)) {
                         ace.getFlags();
                     }
                 } catch (IOException io) {
                     System.err.println(io);
                 }

             }
         }
     }

     protected static String getPassword(String pwd) throws IOException {
         if (pwd.length() == 0) {
             System.out.print("password ? ");
             System.out.flush();
             InputStreamReader inp = new InputStreamReader(System.in);
             BufferedReader br = new BufferedReader(inp);
             return br.readLine();
         } else {
             return pwd;
         }
     }

     /*
      * Constructor.
      */
     public SmbStressTest(String[] args) throws IOException {
         if (args.length == 0) {
             throw new RuntimeException("usage: .. sbm://server/dfs/ 
USERNAME PASSWORD DOMAIN");
         }
         url = args[0];
         username = args[1];
         password = getPassword(args[2]);
         domain = args[3];
     }

     protected final String url;
     protected final String username;
     protected final String password;
     protected final String domain;

     /*
      * Main.
      */
     public void main() throws IOException {
         // Root file (DFS)
         final SmbFile dfsDir = new SmbFile(url, new 
NtlmPasswordAuthentication(domain, username, password),
                 SmbFile.FILE_SHARE_READ);

         // Create deadlock watchdog thread
         new DeadlockWatchdog().start();

         // Create worker threads
         for (int i = 0; i < maxThreads; i++) {
             new FileScanner().start();
         }

         // Collect files
         final Vector<SmbFile> dirs = new Vector<SmbFile>();
         dirs.add(dfsDir);
         while (dirs.size() != 0) {
             final SmbFile dir = dirs.remove(0);
             try {
                 for (SmbFile file : dir.listFiles()) {
                     if (file.isDirectory()) {
                         debug("added directory: " + file);
                         dirs.add(file);
                     } else if (file.isFile()) {
                         debug("added file: " + file);
                         synchronized (this) {
                             while (files.size() > maxThreads * 2) {
                                 files.remove((int) (files.size() * 
Math.random()));
                             }
                             // break;
                             for (int i = 0; i < maxThreads; i++) {
                                 files.add(file.toString());
                             }
                             this.notifyAll();
                         }
                     }
                 }
             } catch (Exception io) {
                 System.err.println(io);
             }
         }

         // Wait for a key and exit
         System.in.read();
         debug("exiting ..");
         synchronized (this) {
             exit = true;
             this.notifyAll();
         }
     }

     // Static main().
     public static void main(String[] args) throws IOException {
         Config.setProperty("jcifs.smb.client.responseTimeout", "3000");
         Config.setProperty("jcifs.smb.client.soTimeout", "100");
         Config.setProperty("jcifs.netbios.cachePolicy", "30");
         Config.setProperty("jcifs.smb.client.tcpNoDelay", "true");

         new SmbStressTest(args).main();
     }

     /*
      * Deadlock watchdog.
      */
     protected class DeadlockWatchdog extends Thread {
         private void printStackTrace(PrintStream servletOutputStream, 
Thread thread, StackTraceElement[] stack)
                 throws IOException {
             final String threadName = thread.getName();
             final ClassLoader threadClassLoader = 
thread.getContextClassLoader();
             final String threadClassLoaderName = threadClassLoader != 
null ? threadClassLoader.getClass().getName()
                     : null;
             final ThreadGroup threadGroup = thread.getThreadGroup();
             final String threadGroupName = threadGroup != null ? 
threadGroup.getName() : null;
             servletOutputStream.println("Thread #" + 
Long.toString(thread.getId())
                     + (threadName != null ? (" (" + threadName + ")") : "")
                     + (threadGroupName != null ? (" group=\"" + 
threadGroupName + "\"") : "") + " state=\""
                     + thread.getState().toString() + "\"" + " 
priority=\"" + Integer.toString(thread.getPriority())
                     + "\"" + " daemon=\"" + (thread.isDaemon() ? "true" 
: "false") + "\"" + " alive=\""
                     + (thread.isAlive() ? "true" : "false") + "\"" + " 
interrupted=\""
                     + (thread.isInterrupted() ? "true" : "false") + "\""
                     + (threadClassLoaderName != null ? (" 
classLoader=\"" + threadClassLoaderName + "\"") : ""));
             if (stack.length > 0) {
                 for (StackTraceElement trace : stack) {
                     servletOutputStream.println("\t" + trace.toString());
                 }
             }
         }

         private void printStackTrace(PrintStream servletOutputStream, 
Map<Thread, StackTraceElement[]> stacks, long id)
                 throws IOException {
             for (Map.Entry<Thread, StackTraceElement[]> entry : 
stacks.entrySet()) {
                 final Thread thread = entry.getKey();
                 if (thread.getId() == id) {
                     final StackTraceElement[] stack = entry.getValue();
                     printStackTrace(servletOutputStream, thread, stack);
                     servletOutputStream.println();
                 }
             }
         }

         @Override
         public void run() {
             watchdog();
             debug("watchdog thread ended");
         }

         protected void watchdog() {
             final ThreadMXBean tmx = ManagementFactory.getThreadMXBean();
             for (;;) {
                 final long[] ids = tmx.findMonitorDeadlockedThreads();
                 // Deadlock detected ?
                 if (ids != null && ids.length != 0) {
                     // Notify other threads
                     synchronized (SmbStressTest.this) {
                         exit = true;
                         SmbStressTest.this.notifyAll();

                         // Alert
                         System.err.println();
                         System.err.println("** Deadlock detected! **");

                         // Print offending stack traces
                         final Map<Thread, StackTraceElement[]> stacks = 
Thread.getAllStackTraces();
                         final ThreadInfo[] infos = tmx.getThreadInfo(ids);
                         for (final ThreadInfo ti : infos) {
                             final long ms = ti.getWaitedTime();
                             System.err.println("The following thread:");
                             try {
                                 printStackTrace(System.err, stacks, 
ti.getThreadId());
                                 printStackTrace(System.err, stacks, 
ti.getLockOwnerId());
                             } catch (IOException e) {
                             }
                             System.err.println("has been blocked for " 
+ (ms / 1000) + "s by:");
                         }
                         System.err.flush();
                         // EXIT.
                         System.exit(-1);
                     }
                 }

                 // Sleep ..
                 try {
                     // debug("sleeping ..");
                     Thread.sleep(1000);
                 } catch (InterruptedException e) {
                 }

                 // Exit ?
                 synchronized (SmbStressTest.this) {
                     if (exit) {
                         break;
                     }
                 }
             }
         }
     }
}