Performance Puzzler with a stack trace

This performance puzzler showed up in my inbox and it's kind of interesting and so I asked the owner of this data if I might share it with you all. Here is some background. The JVM is running Tomcat with BlueDragon, a CFML engine installed. The application was previously calm but since the move to OS X Lion, the Java process would ramp up to 135% CPU utilization and then just sit there. My immediate reaction was to request the thread dump that follows. The question is; what is this thread dump telling you and what would you do to confirm. I would only ask that you email me with your answer rather than spoiling other people's fun by posting it here. I'll post the answer in another blog entry.

2011-09-06 19:37:43
Full thread dump Java HotSpot(TM) 64-Bit Server VM (20.1-b02-383 mixed mode):

"Attach Listener" daemon prio=9 tid=7f846ba62800 nid=0x113e15000 waiting on condition [00000000]
   java.lang.Thread.State: RUNNABLE

"0595F2FF-4284-4CBF-BAC941472F23EE14" daemon prio=5 tid=7f846a810000 nid=0x117bbf000 waiting on condition [117bbe000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"C2F18A1A-8893-4FD8-B0B2703665071215" daemon prio=5 tid=7f846b43f000 nid=0x117abc000 waiting on condition [117abb000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"399842CB-BEA7-498E-A9E3C27C8F945844" daemon prio=5 tid=7f846c109000 nid=0x1179b9000 waiting on condition [1179b8000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"8C28E94D-6C2A-4EED-89C0BB9AA1A8BCCC" daemon prio=5 tid=7f846b43e800 nid=0x1178b6000 waiting on condition [1178b5000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"05ED2DC5-849A-4EA4-806567122BEB6192" daemon prio=5 tid=7f846c159800 nid=0x1177b3000 waiting on condition [1177b2000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"BA98F79A-50E1-4849-98A29B148EB006CE" daemon prio=5 tid=7f846b43d800 nid=0x11756e000 waiting on condition [11756d000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"F5066395-F9D9-4F98-A6727B13979B46FA" daemon prio=5 tid=7f846acf8000 nid=0x11746b000 waiting on condition [11746a000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"DAB951C0-F38D-44C9-8CB87B3145FD8360" daemon prio=5 tid=7f846ab2e800 nid=0x116ace000 waiting on condition [116acd000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"org.apache.commons.vfs.cache.SoftRefFilesCache$SoftRefReleaseThread" daemon prio=5 tid=7f8463239000 nid=0x1176b0000 in Object.wait() [1176af000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbb99d8> (a java.lang.ref.ReferenceQueue$Lock)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
    - locked <7dfbb99d8> (a java.lang.ref.ReferenceQueue$Lock)
    at org.apache.commons.vfs.cache.SoftRefFilesCache$SoftRefReleaseThread.run(SoftRefFilesCache.java:79)

"35B3A1DD-CA13-41DB-95902A6A2204FADE" daemon prio=5 tid=7f8466ad6800 nid=0x116991000 waiting on condition [116990000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

"B8DBA591-39F2-4B41-933934D9DE84D963" daemon prio=5 tid=7f8466ad6000 nid=0x11688e000 waiting on condition [11688d000]
   java.lang.Thread.State: WAITING (parking)
    at sun.misc.Unsafe.park(Native Method)
    - parking to wait for  <7a0edb9f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
    at java.lang.Thread.run(Thread.java:680)

""ajp-bio-8009"-AsyncTimeout" daemon prio=5 tid=7f84688a3800 nid=0x116707000 waiting on condition [116706000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at org.apache.tomcat.util.net.JIoEndpoint$AsyncTimeout.run(JIoEndpoint.java:143)
    at java.lang.Thread.run(Thread.java:680)

""ajp-bio-8009"-Acceptor-0" daemon prio=5 tid=7f84689ce800 nid=0x116604000 runnable [116603000]
   java.lang.Thread.State: RUNNABLE
    at java.net.PlainSocketImpl.socketAccept(Native Method)
    at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408)
    - locked <7dfbba628> (a java.net.SocksSocketImpl)
    at java.net.ServerSocket.implAccept(ServerSocket.java:462)
    at java.net.ServerSocket.accept(ServerSocket.java:430)
    at org.apache.tomcat.util.net.DefaultServerSocketFactory.acceptSocket(DefaultServerSocketFactory.java:59)
    at org.apache.tomcat.util.net.JIoEndpoint$Acceptor.run(JIoEndpoint.java:211)
    at java.lang.Thread.run(Thread.java:680)

""http-bio-8080"-AsyncTimeout" daemon prio=5 tid=7f8468d19000 nid=0x116501000 waiting on condition [116500000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at org.apache.tomcat.util.net.JIoEndpoint$AsyncTimeout.run(JIoEndpoint.java:143)
    at java.lang.Thread.run(Thread.java:680)

""http-bio-8080"-Acceptor-0" daemon prio=5 tid=7f84689cf800 nid=0x1163fe000 runnable [1163fd000]
   java.lang.Thread.State: RUNNABLE
    at java.net.PlainSocketImpl.socketAccept(Native Method)
    at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408)
    - locked <7dfbb9d28> (a java.net.SocksSocketImpl)
    at java.net.ServerSocket.implAccept(ServerSocket.java:462)
    at java.net.ServerSocket.accept(ServerSocket.java:430)
    at org.apache.tomcat.util.net.DefaultServerSocketFactory.acceptSocket(DefaultServerSocketFactory.java:59)
    at org.apache.tomcat.util.net.JIoEndpoint$Acceptor.run(JIoEndpoint.java:211)
    at java.lang.Thread.run(Thread.java:680)

"ContainerBackgroundProcessor[StandardEngine[Catalina]]" daemon prio=5 tid=7f8469fe4000 nid=0x1160a4000 waiting on condition [1160a3000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1369)
    at java.lang.Thread.run(Thread.java:680)

"CFQUERY Backgrounder" daemon prio=1 tid=7f846503a800 nid=0x115f1b000 waiting on condition [115f1a000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.naryx.tagfusion.cfm.sql.platform.java.queryBatchServer.run(Unknown Source)

"BlueDragon MailSender" daemon prio=1 tid=7f846680a000 nid=0x1162fb000 in Object.wait() [1162fa000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7a0edbf80> (a java.lang.Object)
    at com.bluedragon.platform.java.smtp.OutgoingMailServer$MailSender.getMail(Unknown Source)
    - locked <7a0edbf80> (a java.lang.Object)
    at com.bluedragon.platform.java.smtp.OutgoingMailServer$MailSender.run(Unknown Source)

"OpenBDAlarmManager" daemon prio=5 tid=7f84652df800 nid=0x1161f8000 waiting on condition [1161f7000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.nary.util.AlarmManager.run(Unknown Source)
    at java.lang.Thread.run(Thread.java:680)

"BlueDragon MailSender" daemon prio=1 tid=7f84651b6000 nid=0x115d4e000 in Object.wait() [115d4d000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7a0edc008> (a java.lang.Object)
    at java.lang.Object.wait(Object.java:485)
    at com.naryx.tagfusion.cfm.mail.mailServer$MailSender.getMail(Unknown Source)
    - locked <7a0edc008> (a java.lang.Object)
    at com.naryx.tagfusion.cfm.mail.mailServer$MailSender.run(Unknown Source)

"CFQUERY Backgrounder" daemon prio=1 tid=7f846699e800 nid=0x115c4b000 waiting on condition [115c4a000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.naryx.tagfusion.cfm.sql.platform.java.queryBatchServer.run(Unknown Source)

"BlueDragon AlarmManager" daemon prio=5 tid=7f8469acc800 nid=0x1156f5000 waiting on condition [1156f4000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.nary.util.AlarmManager.run(Unknown Source)

"BlueDragon MailSender" daemon prio=1 tid=7f846880a000 nid=0x1159fe000 in Object.wait() [1159fd000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbba4f0> (a java.lang.Object)
    at java.lang.Object.wait(Object.java:485)
    at com.naryx.tagfusion.cfm.mail.mailServer$MailSender.getMail(Unknown Source)
    - locked <7dfbba4f0> (a java.lang.Object)
    at com.naryx.tagfusion.cfm.mail.mailServer$MailSender.run(Unknown Source)

"BlueDragon CFQUERY Backgrounder" daemon prio=1 tid=7f8469964800 nid=0x1158fb000 in Object.wait() [1158fa000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbbada8> (a com.naryx.tagfusion.cfm.sql.queryBatchServer)
    at com.naryx.tagfusion.cfm.sql.queryBatchServer.run(Unknown Source)
    - locked <7dfbbada8> (a com.naryx.tagfusion.cfm.sql.queryBatchServer)

"BlueDragon AlarmManager" daemon prio=5 tid=7f846995f800 nid=0x1157f8000 waiting on condition [1157f7000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.nary.util.AlarmManager.run(Unknown Source)

"BlueDragon MailSender" daemon prio=1 tid=7f84669a5000 nid=0x115311000 in Object.wait() [115310000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbba5b0> (a java.lang.Object)
    at java.lang.Object.wait(Object.java:485)
    at com.naryx.tagfusion.cfm.mail.mailServer$MailSender.getMail(Unknown Source)
    - locked <7dfbba5b0> (a java.lang.Object)
    at com.naryx.tagfusion.cfm.mail.mailServer$MailSender.run(Unknown Source)

"CFQUERY Backgrounder" daemon prio=1 tid=7f84669a4000 nid=0x1149c5000 waiting on condition [1149c4000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.naryx.tagfusion.cfm.sql.platform.java.queryBatchServer.run(Unknown Source)

"BlueDragon AlarmManager" daemon prio=5 tid=7f84669a3800 nid=0x1141a7000 waiting on condition [1141a6000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.nary.util.AlarmManager.run(Unknown Source)

"CFQUERY Backgrounder" daemon prio=1 tid=7f84651fb000 nid=0x115b48000 waiting on condition [115b47000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.naryx.tagfusion.cfm.sql.platform.java.queryBatchServer.run(Unknown Source)

"BlueDragon MailSender" daemon prio=1 tid=7f8463bba800 nid=0x115151000 in Object.wait() [115150000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbb9ac8> (a java.lang.Object)
    at com.bluedragon.platform.java.smtp.OutgoingMailServer$MailSender.getMail(Unknown Source)
    - locked <7dfbb9ac8> (a java.lang.Object)
    at com.bluedragon.platform.java.smtp.OutgoingMailServer$MailSender.run(Unknown Source)

"OpenBDAlarmManager" daemon prio=5 tid=7f8466bca800 nid=0x11504e000 waiting on condition [11504d000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.nary.util.AlarmManager.run(Unknown Source)
    at java.lang.Thread.run(Thread.java:680)

"CFQUERY Backgrounder" daemon prio=1 tid=7f8463281800 nid=0x114734000 waiting on condition [114733000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.naryx.tagfusion.cfm.sql.platform.java.queryBatchServer.run(Unknown Source)

"AWT-AppKit" daemon prio=5 tid=7f846325e800 nid=0x7fff70b3e960 runnable [00000000]
   java.lang.Thread.State: RUNNABLE

"BlueDragon MailSender" daemon prio=1 tid=7f846325d800 nid=0x1143ad000 in Object.wait() [1143ac000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbb9a50> (a java.lang.Object)
    at com.bluedragon.platform.java.smtp.OutgoingMailServer$MailSender.getMail(Unknown Source)
    - locked <7dfbb9a50> (a java.lang.Object)
    at com.bluedragon.platform.java.smtp.OutgoingMailServer$MailSender.run(Unknown Source)

"OpenBDAlarmManager" daemon prio=5 tid=7f846392a800 nid=0x1142aa000 waiting on condition [1142a9000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
    at java.lang.Thread.sleep(Native Method)
    at com.nary.util.AlarmManager.run(Unknown Source)
    at java.lang.Thread.run(Thread.java:680)

"Poller SunPKCS11-Darwin" daemon prio=1 tid=7f84638cf800 nid=0x1140a4000 runnable [1140a3000]
   java.lang.Thread.State: RUNNABLE
    at sun.security.pkcs11.wrapper.PKCS11.C_GetSlotInfo(Native Method)
    at sun.security.pkcs11.SunPKCS11.initToken(SunPKCS11.java:767)
    at sun.security.pkcs11.SunPKCS11.access$100(SunPKCS11.java:42)
    at sun.security.pkcs11.SunPKCS11$TokenPoller.run(SunPKCS11.java:700)
    at java.lang.Thread.run(Thread.java:680)

"GC Daemon" daemon prio=2 tid=7f8463b14800 nid=0x113d12000 in Object.wait() [113d11000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfbb9780> (a sun.misc.GC$LatencyLock)
    at sun.misc.GC$Daemon.run(GC.java:100)
    - locked <7dfbb9780> (a sun.misc.GC$LatencyLock)

"Low Memory Detector" daemon prio=5 tid=7f8463094000 nid=0x113538000 runnable [00000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread1" daemon prio=9 tid=7f8463093800 nid=0x113435000 waiting on condition [00000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" daemon prio=9 tid=7f8463092800 nid=0x113332000 waiting on condition [00000000]
   java.lang.Thread.State: RUNNABLE

"Signal Dispatcher" daemon prio=9 tid=7f8463092000 nid=0x11322f000 runnable [00000000]
   java.lang.Thread.State: RUNNABLE

"Surrogate Locker Thread (Concurrent GC)" daemon prio=5 tid=7f8463091000 nid=0x11312c000 waiting on condition [00000000]
   java.lang.Thread.State: RUNNABLE

"Finalizer" daemon prio=8 tid=7f846380b800 nid=0x112e68000 in Object.wait() [112e67000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfa00cf8> (a java.lang.ref.ReferenceQueue$Lock)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
    - locked <7dfa00cf8> (a java.lang.ref.ReferenceQueue$Lock)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
    at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)

"Reference Handler" daemon prio=10 tid=7f846380b000 nid=0x112d65000 in Object.wait() [112d64000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <7dfa00bc0> (a java.lang.ref.Reference$Lock)
    at java.lang.Object.wait(Object.java:485)
    at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
    - locked <7dfa00bc0> (a java.lang.ref.Reference$Lock)

"main" prio=5 tid=7f8463001000 nid=0x10b110000 runnable [10b10e000]
   java.lang.Thread.State: RUNNABLE
    at java.net.PlainSocketImpl.socketAccept(Native Method)
    at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:408)
    - locked <7a1061e30> (a java.net.SocksSocketImpl)
    at java.net.ServerSocket.implAccept(ServerSocket.java:462)
    at java.net.ServerSocket.accept(ServerSocket.java:430)
    at org.apache.catalina.core.StandardServer.await(StandardServer.java:447)
    at org.apache.catalina.startup.Catalina.await(Catalina.java:707)
    at org.apache.catalina.startup.Catalina.start(Catalina.java:653)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:303)
    at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:431)

"VM Thread" prio=9 tid=7f8463806000 nid=0x112c62000 runnable

"Gang worker#0 (Parallel GC Threads)" prio=9 tid=7f8463002800 nid=0x10e44a000 runnable

"Gang worker#1 (Parallel GC Threads)" prio=9 tid=7f8463003000 nid=0x10e54d000 runnable

"Gang worker#2 (Parallel GC Threads)" prio=9 tid=7f8463003800 nid=0x10e650000 runnable

"Gang worker#3 (Parallel GC Threads)" prio=9 tid=7f8463004000 nid=0x10e753000 runnable

"Concurrent Mark-Sweep GC Thread" prio=9 tid=7f846307f800 nid=0x11249a000 runnable
"VM Periodic Task Thread" prio=10 tid=7f84630a5800 nid=0x11363b000 waiting on condition

"Exception Catcher Thread" prio=10 tid=7f8463001800 nid=0x10b272000 runnable
JNI global references: 1756

tags: stacktrace  java  performance  puzzler