).*");
+ this.setJvmVendor(JVM_VENDOR_LIST[OPENJDK_VM]);
+
+ parseJvmVersion(bis);
+ }
+
+ /**
+ * @returns true, if a class histogram was found and added during parsing.
+ */
+ public boolean isFoundClassHistograms() {
+ return (foundClassHistograms);
+ }
+
+ public MutableTreeNode parseNext() {
+ this.mmap = new MonitorMap();
+ return super.parseNext();
+ }
+
+ /**
+ * add a monitor link for monitor navigation
+ *
+ * @param line
+ * containing monitor
+ */
+ protected String linkifyMonitor(String line) {
+ try {
+ if (line != null && line.indexOf('<') >= 0) {
+ String begin = line.substring(0, line.indexOf('<'));
+ String monitor = line.substring(line.indexOf('<'), line.indexOf('>') + 1);
+ String end = line.substring(line.indexOf('>') + 1);
+ monitor = monitor.replaceAll("<", "<");
+ monitor = monitor.substring(0, monitor.length() - 1) + ">";
+ return (begin + monitor + end);
+ } else if (line != null && line.indexOf('@') >= 0) {
+ String begin = line.substring(0, line.indexOf('@') + 1);
+ String monitor = line.substring(line.indexOf('@'));
+ monitor = monitor.replaceAll("@", "@\">");
+ monitor = monitor.substring(0, monitor.length() - 1) + "";
+ return (begin + monitor);
+ } else {
+ return (line);
+ }
+ } catch(Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * add a monitor link for monitor navigation
+ *
+ * @param line
+ * containing monitor
+ */
+ public String linkifyDeadlockInfo(String line) {
+ if (line != null && line.indexOf("Ox") >= 0) {
+ String begin = line.substring(0, line.indexOf("0x"));
+ int objectBegin = line.lastIndexOf("0x");
+ int monitorBegin = line.indexOf("0x");
+ String monitorHex = line.substring(monitorBegin, monitorBegin + 10);
+
+ String monitor = line.substring(objectBegin, objectBegin + 10);
+ String end = line.substring(line.indexOf("0x") + 10);
+
+ monitor = "\">" + monitorHex + "";
+ return (begin + monitor + end);
+ } else {
+ return (line);
+ }
+ }
+
+ /**
+ * checks for the next class histogram and adds it to the tree node passed
+ *
+ * @param threadDump
+ * which tree node to add the histogram.
+ */
+ public boolean checkForClassHistogram(DefaultMutableTreeNode threadDump) throws IOException {
+ HistogramTableModel classHistogram = parseNextClassHistogram(getBis());
+
+ if (classHistogram.getRowCount() > 0) {
+ addHistogramToDump(threadDump, classHistogram);
+ }
+
+ return (classHistogram.getRowCount() > 0);
+ }
+
+ /**
+ * checks for the Lock Chains and adds it to the tree node passed
+ *
+ * @param threadDump
+ * which tree node to add the lock chain info.
+ */
+ public boolean checkForLockChains(DefaultMutableTreeNode threadDump) throws IOException {
+ return false;
+ }
+
+
+ private void addHistogramToDump(DefaultMutableTreeNode threadDump, HistogramTableModel classHistogram) {
+ DefaultMutableTreeNode catHistogram;
+ HistogramInfo hi = new HistogramInfo("Class Histogram of Dump", classHistogram);
+ catHistogram = new DefaultMutableTreeNode(hi);
+ threadDump.add(catHistogram);
+ }
+
+ /**
+ * parses the next class histogram found in the stream, uses the max check
+ * lines option to check how many lines to parse in advance.
+ *
+ * This could be called from parseLoggcFile, which is outside our normal
+ * calling stream. Thus, we have to pass in the LineNumberReader. However, to
+ * handle a WrappedSunJDKParser, we have to use getNextLine() if possible.
+ *
+ * @param bis
+ * the stream to read.
+ */
+ private HistogramTableModel parseNextClassHistogram(LineNumberReader bis) throws IOException {
+ boolean finished = false;
+ boolean found = false;
+ HistogramTableModel classHistogram = new HistogramTableModel();
+ int maxLinesCounter = 0;
+
+ boolean isNormalBis = bis == getBis();
+
+ while (bis.ready() && !finished) {
+ String line = (isNormalBis) ? getNextLine().trim() : bis.readLine().trim();
+ if (!found && !line.equals("")) {
+ if (line.startsWith("num #instances #bytes class name")) {
+ found = true;
+ } else if (maxLinesCounter >= getMaxCheckLines()) {
+ finished = true;
+ } else {
+ maxLinesCounter++;
+ }
+ } else if (found) {
+ if (line.startsWith("Total ")) {
+ // split string.
+ String newLine = line.replaceAll("(\\s)+", ";");
+ String[] elems = newLine.split(";");
+ classHistogram.setBytes(Long.parseLong(elems[2]));
+ classHistogram.setInstances(Long.parseLong(elems[1]));
+ finished = true;
+ } else if (!line.startsWith("-------")) {
+ // removed blank, breaks splitting using blank...
+ String newLine = line.replaceAll("", "");
+
+ // split string.
+ newLine = newLine.replaceAll("(\\s)+", ";");
+ String[] elems = newLine.split(";");
+
+ if (elems.length == 4) {
+ classHistogram.addEntry(elems[3].trim(), Integer.parseInt(elems[2].trim()),
+ Integer.parseInt(elems[1].trim()));
+ } else {
+ classHistogram.setIncomplete(true);
+ finished = true;
+ }
+
+ }
+ }
+ }
+
+ return (classHistogram);
+ }
+
+ /**
+ * Heap PSYoungGen total 6656K, used 3855K [0xb0850000, 0xb0f50000,
+ * 0xb4130000) eden space 6144K, 54% used [0xb0850000,0xb0b97740,0xb0e50000)
+ * from space 512K, 97% used [0xb0ed0000,0xb0f4c5c0,0xb0f50000) to space 512K,
+ * 0% used [0xb0e50000,0xb0e50000,0xb0ed0000) PSOldGen total 15552K, used
+ * 13536K [0x94130000, 0x95060000, 0xb0850000) object space 15552K, 87% used
+ * [0x94130000,0x94e68168,0x95060000) PSPermGen total 16384K, used 13145K
+ * [0x90130000, 0x91130000, 0x94130000) object space 16384K, 80% used
+ * [0x90130000,0x90e06610,0x91130000)
+ *
+ * @param threadDump
+ * @return
+ * @throws java.io.IOException
+ *
+ public boolean checkThreadDumpStatData(ThreadDumpInfo tdi) throws IOException {
+ boolean finished = false;
+ boolean found = false;
+ StringBuffer hContent = new StringBuffer();
+ int heapLineCounter = 0;
+ int lines = 0;
+
+ while (getBis().ready() && !finished) {
+ String line = getNextLine();
+ if (!found && !line.equals("")) {
+ if (line.trim().startsWith("Heap")) {
+ found = true;
+ } else if (lines >= getMaxCheckLines()) {
+ finished = true;
+ } else {
+ lines++;
+ }
+ } else if (found) {
+ if (heapLineCounter < 7) {
+ hContent.append(line).append("\n");
+ } else {
+ finished = true;
+ }
+ heapLineCounter++;
+ }
+ }
+ if (hContent.length() > 0) {
+ tdi.setHeapInfo(new HeapInfo(hContent.toString()));
+ theLogger.finest("Found heap info:" + hContent.toString());
+ }
+
+ return (found);
+ }
+ * */
+
+ /**
+ * check if any dead lock information is logged in the stream
+ *
+ * @param threadDump
+ * which tree node to add the histogram.
+ */
+ public int checkForDeadlocks(DefaultMutableTreeNode threadDump) throws IOException {
+ boolean finished = false;
+ boolean found = false;
+ int deadlocks = 0;
+ int lineCounter = 0;
+ StringBuffer dContent = new StringBuffer();
+ TreeCategory deadlockCat = new TreeCategory("Deadlocks", IconFactory.DEADLOCKS);
+ DefaultMutableTreeNode catDeadlocks = new DefaultMutableTreeNode(deadlockCat);
+ boolean first = true;
+
+ while (getBis().ready() && !finished) {
+ String line = getNextLine();
+
+ if (!found && !line.equals("")) {
+ if (line.trim().startsWith("Found one Java-level deadlock")) {
+ found = true;
+ dContent.append("");
+ dContent.append("Found one Java-level deadlock");
+ dContent.append("
\n");
+ } else if (lineCounter >= getMaxCheckLines()) {
+ finished = true;
+ } else {
+ lineCounter++;
+ }
+ } else if (found) {
+ if (line.startsWith("Found one Java-level deadlock")) {
+ if (dContent.length() > 0) {
+ deadlocks++;
+ addToCategory(catDeadlocks, "Deadlock No. " + (deadlocks), null, dContent.toString(), 0, false);
+ }
+ dContent = new StringBuffer();
+ dContent.append("");
+ dContent.append("Found one Java-level deadlock");
+ dContent.append("
\n");
+ first = true;
+ } else if ((line.indexOf("Found") >= 0) && (line.endsWith("deadlocks.") || line.endsWith("deadlock."))) {
+ finished = true;
+ } else if (line.startsWith("=======")) {
+ // ignore this line
+ } else if (line.indexOf(" monitor 0x") >= 0) {
+ dContent.append(linkifyDeadlockInfo(line));
+ dContent.append("\n");
+ } else if (line.indexOf("Java stack information for the threads listed above") >= 0) {
+ dContent.append("
");
+ dContent.append("Java stack information for the threads listed above");
+ dContent.append("
");
+ first = true;
+ } else if ((line.indexOf("- waiting on") >= 0) || (line.indexOf("- waiting to") >= 0)
+ || (line.indexOf("- locked") >= 0) || (line.indexOf("- parking to wait") >= 0)) {
+
+ dContent.append(linkifyMonitor(line));
+ dContent.append("\n");
+
+ } else if (line.trim().startsWith("\"")) {
+ dContent.append("");
+ if (first) {
+ first = false;
+ } else {
+ dContent.append("
");
+ }
+ dContent.append("");
+ dContent.append(line);
+ dContent.append("");
+ } else {
+ dContent.append(line);
+ dContent.append("\n");
+ }
+ }
+ }
+ if (dContent.length() > 0) {
+ deadlocks++;
+ addToCategory(catDeadlocks, "Deadlock No. " + (deadlocks), null, dContent.toString(), 0, false);
+ }
+
+ if (deadlocks > 0) {
+ threadDump.add(catDeadlocks);
+ ((ThreadDumpInfo) threadDump.getUserObject()).setDeadlocks((TreeCategory) catDeadlocks.getUserObject());
+ deadlockCat.setName("Deadlocks (" + deadlocks + (deadlocks == 1 ? " deadlock)" : " deadlocks)"));
+ }
+
+ return (deadlocks);
+ }
+
+ /**
+ * parses a loggc file stream and reads any found class histograms and adds
+ * the to the dump store
+ *
+ * @param loggcFileStream
+ * the stream to read
+ * @param root
+ * the root node of the dumps.
+ */
+ public void parseLoggcFile(InputStream loggcFileStream, DefaultMutableTreeNode root) {
+ LineNumberReader bis = new LineNumberReader(new InputStreamReader(loggcFileStream));
+ Vector histograms = new Vector();
+
+ try {
+ while (bis.ready()) {
+ bis.mark(getMarkSize());
+ String nextLine = bis.readLine();
+ if (nextLine.startsWith("num #instances #bytes class name")) {
+ bis.reset();
+ histograms.add(parseNextClassHistogram(bis));
+ }
+ }
+
+ // now add the found histograms to the tree.
+ for (int i = histograms.size() - 1; i >= 0; i--) {
+ DefaultMutableTreeNode dump = getNextDumpForHistogram(root);
+ if (dump != null) {
+ addHistogramToDump(dump, (HistogramTableModel) histograms.get(i));
+ }
+ }
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ /**
+ * generate thread info token for table view.
+ *
+ * @param name
+ * the thread info.
+ * @return thread tokens.
+ */
+ public String[] getThreadTokens(String name) {
+
+ // Try for pattern matching
+ // Example: "pool-107-thread-25" prio=10 tid=0x000000004b7de800 nid=0x13b9 waiting on condition [0x0000000077305000]
+
+ String patternMask = "^.*\"([^\\\"]+)\".*tid=([^ ]+|).*nid=([^ ]+) *([^\\[]*).*";
+ name = name.replace("- Thread t@", "tid=");
+
+ String[] tokens = new String[] {};
+ Pattern p = null;
+
+ try {
+ p = Pattern.compile(patternMask);
+ Matcher m = p.matcher(name);
+
+ m.matches();
+
+ tokens = new String[7];
+ tokens[0] = m.group(1); // name
+ // tokens[1] = m.group(4); // prio
+ tokens[1] = m.group(2); // tid
+ tokens[2] = m.group(3); // nid
+ tokens[3] = m.group(4); // State
+
+ // Always treat the tid as a hexa decimal
+ if (!tokens[1].startsWith("0x"))
+ tokens[1] = "0x" + tokens[1];
+
+ } catch(Exception e) {
+
+ // Failure, try different pattern matching
+ // Example: "pool-107-thread-25" nid=0x13b9 state=WAIITING
+
+ patternMask = "^.*\"([^\\\"]+)\".*nid=([^ ]+) *state=([^\\[]*).*";
+
+ try {
+
+ p = Pattern.compile(patternMask);
+ Matcher m = p.matcher(name);
+
+ m.matches();
+ tokens = new String[7];
+ tokens[0] = m.group(1); // name
+ tokens[2] = m.group(2); // nid
+ tokens[3] = m.group(3); // State
+ } catch(Exception e2) {
+ theLogger.finest("WARNING!! Unable to parse partial Thread Tokens with name:" + name);
+ //e.printStackTrace();
+ return doHardParsing(name);
+ }
+ }
+
+ return (tokens);
+ }
+
+ /**
+ * check and parse manually the thread label
+ *
+ * @param nameEntry
+ * the thread name line
+ */
+ private String[] doHardParsing(String nameEntry) {
+
+ String[] tokens = new String[4];
+ int index = nameEntry.indexOf("\"", 1);
+ if (index > 1) {
+ tokens[0] = nameEntry.substring(1, index);
+ } else {
+ tokens[0] = nameEntry.substring(1);
+ return tokens;
+ }
+
+ String remainingLabel = nameEntry.substring(index + 1).trim();
+ String[] remainingTokens = remainingLabel.replace("daemon ","").trim().split(" ");
+
+ // If there are more fields in the thread name
+ // like: "pool-107-thread-25" prio=10 tid=0x000000004b7de800 nid=0x13b9 waiting on condition [0x0000000077305000]
+
+ if (remainingTokens.length >= 3) {
+ for(int i = 1; i < remainingTokens.length; i++) {
+ if (i == 3)
+ break;
+
+ String label = remainingTokens[i].replaceAll(".*=", "");
+ //if (i == 1) // tid
+ if (remainingTokens[i].startsWith("tid") || remainingTokens[i].startsWith("id")) //Derek Kam : Added id as the thread ID
+ tokens[1] = label;
+ //else if (i == 2) // nid
+ if (remainingTokens[i].startsWith("nid"))
+ tokens[2] = label;
+ }
+
+
+ for(int i = 3; i < remainingTokens.length; i++) {
+ if (remainingTokens[i].startsWith("[0"))
+ break;
+
+ tokens[3] = tokens[3] + " " + remainingTokens[i];
+ }
+ } else {
+ // Hotspot thread dump dumped using JMX does not have priority or other data fields..
+ // Example: "SwingWorker-pool-3-thread-6" nid=52 state=WAITING
+ for(int i = 0; i < remainingTokens.length; i++) {
+
+ String label = remainingTokens[i].replaceAll(".*=", "");
+ //if (i == 1) // tid
+ if (remainingTokens[i].startsWith("tid") || remainingTokens[i].startsWith("id")) //Derek Kam : Added id as the thread ID
+ tokens[1] = label;
+ //else if (i == 2) // nid
+ else if (remainingTokens[i].startsWith("nid"))
+ tokens[2] = label;
+ else
+ tokens[3] = tokens[3] + " " + remainingTokens[i];
+ }
+
+ }
+
+ // Always treat the tid as a hexa decimal - Derek Kam: Commented these 2 lines, treat the tid data as it is, no conversion.
+// if (tokens[1] != null && !tokens[1].startsWith("0x"))
+// tokens[1] = "0x" + tokens[1];
+
+ return tokens;
+ }
+
+ /**
+ * check if the passed logline contains the beginning of a sun jdk thread
+ * dump.
+ *
+ * @param logLine
+ * the line of the logfile to test
+ * @return true, if the start of a sun thread dump is detected.
+ */
+ public static boolean checkForSupportedThreadDump(String logLine) {
+ return (logLine.trim().indexOf("Full thread dump") >= 0);
+ }
+
+ /**
+ * @param bis the LineNumberReader
+ */
+ protected void parseJvmVersion(LineNumberReader bis) {
+ int count = 0;
+ try {
+ bis.reset();
+ while (bis.ready() && count++ < 15) {
+ String line = bis.readLine();
+ if (line != null) {
+ int index = line.indexOf("OpenJDK");
+ if (index > 0) {
+ super.setJvmVersion(line.substring(index).trim().replaceAll(":", ""));
+ theLogger.info("JVM Version:" + line);
+ return;
+ }
+ }
+ }
+ } catch(Exception e) { }
+
+ }
+
+}
diff --git a/src/java/com/oracle/ateam/threadlogic/resources/AdvisoryMap.xml b/src/java/com/oracle/ateam/threadlogic/resources/AdvisoryMap.xml
index 87655de..3da91da 100644
--- a/src/java/com/oracle/ateam/threadlogic/resources/AdvisoryMap.xml
+++ b/src/java/com/oracle/ateam/threadlogic/resources/AdvisoryMap.xml
@@ -14,7 +14,7 @@ Advisory is used to tag threads that match or display certain characteristics
The behavior might be coming from the thread (name/stack/state) or a combination of conditions (like multiple threads blocked for same lock or it got marked as STUCK etc). Each of the advisory has an associated health level that can be used to filter/analyze or build on for further analysis.
Each Advisory entry has a Name, Health, Keyword (pattern to search against), Description and Advice
-The Name is used as short id/reference to an Advice
+The Name is used as short id/reference to an Advice
The Health can be one of the following - IGNORE, NORMAL, WATCH, WARNING, FATAL (increasing level of severity)
The Thread with a certain set of advisories is marked with the health level that matches the most severe of its tagged advisories.
Similarly, the most severe advisory across multiple threads in the thread group are promoted to the group and so on to the Thread dump
@@ -29,17 +29,17 @@ Example:
Using Native IO via Select or Poll
Ignore
-
+
Use . (period) instead of / package paths in the keyword entry.
Use wildcard as needed; Use . (period character) for escaping $, ?, _ etc.
The advisory is a match for a thread when the keyword search against the thread is successful.
-There can be multiple keywords separated by a ", ".
+There can be multiple keywords separated by a ", ".
For example:
DB Execute
- WATCH
+ WATCH
PreparedStatement.execute, Statement.executeQuery
Executing operation or query on DB
Check/Monitor Database SQL Executions if it takes longer and also check for socket connection disruption to database if thread continues to show same pattern
@@ -50,12 +50,12 @@ In above example, there are 2 keywords (PreparedStatement.execute and Statement.
The next entry is the Descrp which is a just a textual description of the advisory
The last entry is the Advice - as to actions or solutions can be applied.
-For multi-line pattern, use PatternA.*PatternB to match against all lines that start off
+For multi-line pattern, use PatternA.*PatternB to match against all lines that start off
with PatternA with some content in the middle and ending with PatternB.
Use with caution as it can do greedy grab of everything within the specified patterns.
Example:
-
+
The Name of the advice can be used within the GroupDefns to downgrade certain advisories for specific thread groups.
Ensure the Advice Name in AdvisoryMap matches with the AdvisoryId inside GroupDefns.
Example:
@@ -72,7 +72,7 @@ Example:
DB Execute
Socket Read
-
+
In the above example, DB Execute and Socket Read Advisories are excluded for the thread group: Oracle AQ Adapter
@@ -84,7 +84,7 @@ If '&' is really required, use &
-->
-
WLS Synchronous JMS Receiver
@@ -96,7 +96,7 @@ xmlns="http://java.net/projects/threadlogic">
JMS Producer Flow control
WARNING
- weblogic.jms.client.JMSProducer.doFlowControl
+ weblogic.jms.client.JMSProducer.doFlowControl
Producer message flow is controlled (slowed down)
Try to maintain equilibrium between producers and consumers by speeding up consumers or slowing down producers; Tune the flow control configuration parameters as needed
@@ -239,7 +239,7 @@ xmlns="http://java.net/projects/threadlogic">
afjca15.AbstractResourceAdapter.endpointDeactivation
IWay Adapter Endpoint Deactivation invoked
IWay Adapter endpoint is getting deactivated, there would be service disruption,; Ensure the call completes by checking in successive thread dumps
-
+
Stellent WCM
NORMAL
@@ -288,7 +288,7 @@ xmlns="http://java.net/projects/threadlogic">
com.bea.wli.sb.pipeline.RouterCallback.onReceiveResponse
OSB handling response
Normal OSB response handling
-
+
OSB Proxy waiting for response
WATCH
@@ -302,7 +302,7 @@ xmlns="http://java.net/projects/threadlogic">
com.bea.wli.sb.service.resultcache.ResultCache.get
OSB attempting to lookup cached service response from Coherence layer
OSB is attempting to check for a cached service response from Coherence layer,; Should return immediately and not get into waiting or blocked state
-
+
OSB Service Response save in cache
NORMAL
@@ -316,21 +316,21 @@ xmlns="http://java.net/projects/threadlogic">
stages.transform.runtime.JavaCalloutRuntimeStep
OSB Proxy service invoking Java Callout
Normal java callout activity
-
+
XPath Contention
WARNING
a java.lang.Class for org.apache.xmlbeans.impl.store.Path, org.apache.xmlbeans.impl.store.Path.getCompiledPath
Apache XMLBeans might be hitting static synchronized method during xpath execution
Apply Bug #9727796 if threads appear blocked in xpath compilation
-
+
WLS JMS Paging
WARNING
mesaging.kernel.internal.PagingImpl
WebLogic JMS paging messages to disk
- WLS might have started paging message to disk either because consumers cannot keep up with producers and messages have started accumulating increasing memory pressure or for temporary storage due to delayed delivery; Paging will slow down performance; Increase, speed up or tune consumers or use flow controls/quotas to slow down producers and in-flow rates; Or increase number of servers to share the load.
+ WLS might have started paging message to disk either because consumers cannot keep up with producers and messages have started accumulating increasing memory pressure or for temporary storage due to delayed delivery; Paging will slow down performance; Increase, speed up or tune consumers or use flow controls/quotas to slow down producers and in-flow rates; Or increase number of servers to share the load.
@@ -412,7 +412,7 @@ xmlns="http://java.net/projects/threadlogic">
DB Execute
- WATCH
+ WATCH
Statement.execute, Statement.fetch
Executing operation or query on DB
Check/Monitor Database SQL Executions if it takes longer and also check for socket connection disruption to database; if the thread continues to show same pattern or remains stuck (use AWR reports to debug)
@@ -501,7 +501,7 @@ xmlns="http://java.net/projects/threadlogic">
WLS JMS Cluster has undergone some change - either change (add/drop) in cluster membership or change in consumers/producers. This can be seen during cluster start when members join the cluster. Can signify cluster instability or messaging.
Ensure cluster members or clients are not getting added/dropped too frequently after cluster has been up. Turn ON Cluster and JMS related debugs if there are repeat drop in clients
-
+
SOAP outbound calls from SOA
NORMAL
@@ -676,7 +676,7 @@ xmlns="http://java.net/projects/threadlogic">
DeploymentServerService.start
Deployment Service has been started
Server bringing up deployment service. Deployments should complete in a finite time. Precompile application bits with the correct server version ahead of deployment if possible to speed up deployment time
-
+
JSP Compilation
NORMAL
@@ -690,7 +690,7 @@ xmlns="http://java.net/projects/threadlogic">
weblogic.jsp.internal.client.JobWaiter.blockUntilFinished
JSP compilation job blocking request
Thread waiting for jsp compilation to finish. Precompile application bits and JPSs against the correct server version ahead of deployment if possible
-
+
WLS App undeployment
WATCH
@@ -775,7 +775,7 @@ xmlns="http://java.net/projects/threadlogic">
Parallel GC Threads
Large number of Parallel GC Threads used by JVM
Reduce number of Parallel GC Threads, based on number of CPU/Cores and JVMs running on the host machine; Use -XXgcthreads:N for JRockit and -XX:ParallelGCThreads=N for HotSpot to limit number of parallel GC threads. Recommended # of GC Threads = (# of CPUs x # of Hardware Threads per core)/(# of JVMs on same machine). Example: Exalogic node 12 core, hyperthreaded means 24 hardware threads, and 2 JVMs imply 12 ParallelGC threads for each JVM
-
+
Waiting for Event while blocking others
WARNING
@@ -824,7 +824,7 @@ xmlns="http://java.net/projects/threadlogic">
com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll
Poll for data/cache entry from remote Coherence members
Normal
-
+
Coherence waiting for EventQueue to drain
WARNING
@@ -866,7 +866,7 @@ xmlns="http://java.net/projects/threadlogic">
oracle.tip.adapter.aq.*AbstractDequeueAgent
AQ Message dequeuing
Normal AQ message dequeue thread
-
+
AQ JMS Consumer
NORMAL
@@ -894,28 +894,28 @@ xmlns="http://java.net/projects/threadlogic">
oracle.tip.mediator.dispatch.db.DBContainerIdManager
Oracle Mediator container id refresh wait
Normal
-
+
Oracle Mediator Locker
NORMAL
oracle.tip.mediator.common.listener.DBLocker
Oracle Mediator locker thread locking the message for worker thread processing
Normal
-
+
Oracle SOA Cluster Node Heartbeat Locker
NORMAL
com.collaxa.cube.cluster.ClusterService
Oracle SOA cluster node heartbeat service maintaining all cluster nodes heartbeat
Normal
-
+
Threaddump generation
IGNORE
racle.dfw.sampling.DumpSampling
Threaddump generation
Ignore
-
+
Oracle SOA JMS Adapter
NORMAL
@@ -929,7 +929,7 @@ xmlns="http://java.net/projects/threadlogic">
oracle.tip.adapter.db.inbound,oracle.tip.adapter.db.InboundWork
Oracle SOA DB Adapter running
Normal
-
+
Oracle Mediator waiting
NORMAL
@@ -943,7 +943,7 @@ xmlns="http://java.net/projects/threadlogic">
oracle.tip.mediator.service,oracle.tip.mediator.serviceEngine.MediatorServiceEngine
Oracle Mediator Service running
Normal
-
+
Oracle Mediator Resequencer
WATCH
@@ -957,35 +957,35 @@ xmlns="http://java.net/projects/threadlogic">
oracle.tip.resequencer.threadpool.WorkerThread
Oracle Mediator resequencer worker thread
Mediator resequencer under stress, please check the number of worker threads configured in EM and the target system log, this might be due to target system is slow
-
+
Oracle Mediator Resequencer
WATCH
oracle.tip.mediator.resequencer.ResequencerDBLocker
Oracle Mediator resequencer locker thread
Mediator resequencer locker thread, there will be only 1 resequencer db locker thread to lock the group, if it is under stress or slow, please check the database performance
-
+
Oracle Mediator Resequencer
WATCH
oracle.tip.resequencer.threadpool.ResequencerDBLocker, oracle.tip.resequencer.threadpool.AbstractLocker
Oracle Mediator resequencer locker thread
Mediator resequencer locker thread, there will be only 1 resequencer db locker thread to lock the group, if it is under stress or slow, please check the database performance
-
+
Oracle Mediator Resequencer
NORMAL
oracle.tip.resequencer
Oracle Mediator resequencer thread
Ignore
-
+
WLS Persistent Store Thread
NORMAL
weblogic.store.internal.PersistentStoreImpl.getOutstandingWork
Thread handling WLS Persistent Store
Normal thread handling wls file based persistence
-
+
WLS StoreIO Writes
NORMAL
@@ -1000,7 +1000,7 @@ xmlns="http://java.net/projects/threadlogic">
WLS starting recovery of persistent stores (JMS filestores or default stores) at startup
The recovery will affect server startup based on number of messasges in the jms stores; compact JMS file stores if possible before restart; Refer to JMS Documentation on weblogic.store.Admin utility
-
+
WLS NM Client
NORMAL
@@ -1035,8 +1035,8 @@ xmlns="http://java.net/projects/threadlogic">
WriteLock.lock
Thread attempting to acquire java concurrent Write Lock
- Treat it as blocked thread; Check if the thread continues in Parked state or acquired the lock;
- Use -XX:+PrintConcurrentLocks to print the details of locks; Enable jvm flag -XX:+UseMembar if threads are blocked for Reentrant locks with OSB
+ Treat it as blocked thread; Check if the thread continues in Parked state or acquired the lock;
+ Use -XX:+PrintConcurrentLocks to print the details of locks; Enable jvm flag -XX:+UseMembar if threads are blocked for Reentrant locks with OSB
@@ -1210,7 +1210,7 @@ xmlns="http://java.net/projects/threadlogic">
WLS Replicated Session Secondary
NORMAL
- ReplicatedSessionData.update
+ ReplicatedSessionData.update
WLS webapp is using replicated session (in clustered instance)
Normal, HTTP Session is replicated on secondary
@@ -1297,7 +1297,7 @@ xmlns="http://java.net/projects/threadlogic">
com.sun.hss.domain.util.spawn.UNIXProcess.waitForProcessExit
Thread waiting for death of spawned off process
Normal
-
+
Sun xVM Appliance Details
NORMAL
@@ -1337,7 +1337,7 @@ xmlns="http://java.net/projects/threadlogic">
NIOMuxer Bug 13962335
WARNING
NIOSocketMuxer.*MuxableSocketT3.connect
Check if WLS NIOMuxer is blocked for long, attempting to read from an unreachable endpoint, which can lead to stuck thread and severe Server Hang situations
@@ -1370,74 +1370,74 @@ xmlns="http://java.net/projects/threadlogic">
com.ibm.mq.jmqi.remote.internal.system.RemoteConnection.sendTSH.*MQSession.close
Threads communicating with MQ Series might go into wait state for long periods on closure of connections possibly due to shared conversations in MQ 7; Disable shared conversations if threads appear to hang or stay in wait state for long.
Set the Sharing Conversation property of the Server Connection Channel to zero on MQ Series. Refer to Oracle Support Note Doc ID 1511617.1 or IBM Documentation on Shared Conversations
-
+
ONS Event Wait
NORMAL
- oracle.ons.NotificationQueue.dequeue
+ oracle.ons.NotificationQueue.dequeue
Thread waiting for ONS Event
Normal ONS event thread with Gridlink
-
+
Cavisson SQL Dump
WARNING
- com.cavisson.ndutils.NDSys.dumpSQL
+ com.cavisson.ndutils.NDSys.dumpSQL
Cavisson capturing SQL dump; can introduce bottleneck among threads and degrade perf
Remove or disable Cavisson SQL dump instrumentation
BTM Instrumentation
WATCH
- com.amberpoint.nanoagent
+ com.amberpoint.nanoagent
Heavy code instrumentation or dynamic discovery can introduce overhead, slowness or degrade perf under heavy loads
Disable BTM Amberpoint code instrumentation on perf degrade or slowness
UCP Pool Conn
WATCH
- UCPRACModuleImpl.getConnection
+ UCPRACModuleImpl.getConnection
Trying to get jdbc connection from UCP Gridlink pool for RAC
If multiple threads appear with this pattern, set to Seconds to Trust Idle connection to 10 or 15 seconds, enable Pinned-to-thread and apply Bug# 17038851
OJDL Logging
WATCH
- oracle.core.ojdl.logging.ODLLogger
+ oracle.core.ojdl.logging.ODLLogger
Verbose logging levels in OJDL can slow down threads
If multiple threads appear in blocked state waiting for lock during OJDL logging, set the logging level to less verbose (ex: ERROR instead of INFO)
BPEL wait for Response
NORMAL
- com.collaxa.cube.engine.delivery.DeliveryService.getResult
+ com.collaxa.cube.engine.delivery.DeliveryService.getResult
Waiting for service response
If its a long invocation, try to use async model
BPEL Execution
NORMAL
- com.collaxa.cube.engine.ejb.impl.bpel.BPELDeliveryBean
+ com.collaxa.cube.engine.ejb.impl.bpel.BPELDeliveryBean
BPELDelivery Bean execution for sync processes
Normal
BPEL Outbound WS Invoke
NORMAL
- com.collaxa.cube.ws.WSInvocationManager.invoke
+ com.collaxa.cube.ws.WSInvocationManager.invoke
BPEL Layer invoking external Web Service
Normal
Oracle DMS
NORMAL
- oracle.dms
+ oracle.dms
Oracle DMS Thread
Normal
Apache TSCCM ConnectionPool blockage
WARNING
- org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking
+ org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking
Apache limits number of concurrent outbound http clients to 2 for a given destination and overall to 20
Modify Apache Tsccm code to allow higher number of connections
@@ -1510,14 +1510,14 @@ xmlns="http://java.net/projects/threadlogic">
oracle.dfw
Oracle Diagnostic Framework
Ignore
-
+
Oracle SOA Cache Store for Audit
IGNORE
com.collaxa.cube.persistence.dao.impl.coherence.cachestore.AuditCachestore
Oracle SOA write behind thread for audit cache store using coherence
Ignore
-
+
Oracle SOA Cache Store for delievery Messages
IGNORE
@@ -1531,21 +1531,21 @@ xmlns="http://java.net/projects/threadlogic">
com.collaxa.cube.persistence.dao.impl.coherence.cachestore.XmlDocumentCacheStore
Oracle SOA write behind thread for xml document using coherence
Ignore
-
+
Oracle SOA Cache Store for invoke messages
IGNORE
com.collaxa.cube.persistence.dao.impl.coherence.cachestore.InvokeMessageCacheStore
Oracle SOA write behind thread for invoke messages using coherence
Ignore
-
+
Oracle SOA Cache Store for delivery subscription
IGNORE
com.collaxa.cube.persistence.dao.impl.coherence.cachestore.DeliverySubscriptionCacheStore
Oracle SOA write behind thread for delivery subscription using coherence
Ignore
-
+
Oracle SOA Cache Store for cube instance
IGNORE
@@ -1559,28 +1559,28 @@ xmlns="http://java.net/projects/threadlogic">
oracle.soa.tracking
Oracle SOA Instance Tracking
Normal
-
+
Oracle AIA Session Pool
IGNORE
oracle.apps.aia.core.sessionpool
Oracle AIA Session Pool Manager Thread
Ignore
-
+
Oracle Mediator
IGNORE
oracle.tip.mediator.common.listener.AbstractWorker
Oracle Mediator worker thread running
Ignore
-
+
Oracle EDN
IGNORE
oracle.integration.platform.blocks.event.saq.SAQBusinessEventBus
Oracle EDN-DB uses Oracle AQ DB as backing store
Ignore
-
+
Oracle EDN
IGNORE
@@ -1594,138 +1594,145 @@ xmlns="http://java.net/projects/threadlogic">
oracle.integration.platform.blocks.event.jms2.EdnJmsConnectionFactory
Oracle EDN uses WLS JMS as backing store
Ignore
-
+
Oracle EDN
IGNORE
oracle.integration.platform.blocks.event.jms2.EdnAqConnectionFactory
Oracle EDN uses AQ JMS as backing store
Ignore
-
+
MDS
IGNORE
oracle.mds
MDS Thread
Ignore
-
+
BPEL Execution
NORMAL
- com.collaxa.cube.engine.ejb.impl.bpel.BPELEngineBean,com.collaxa.cube.engine.ejb.impl.bpel.BPELDispatcherBean
+ com.collaxa.cube.engine.ejb.impl.bpel.BPELEngineBean,com.collaxa.cube.engine.ejb.impl.bpel.BPELDispatcherBean
BPEL Engine execution
Normal
-
+
BPEL Engine Overloaded
WARNING
- BPELEngineBlocked
+ BPELEngineBlocked
BPEL Engine overloaded
Possible BPEL Engine overloaded or slow if you see the same thread blocked in subsequence thread dumps, please review the stack trace and server logs, and check the all timeout settings, number of threads and the target system is able to handle the load
-
+
BPEL Audit Trail
NORMAL
- com.collaxa.cube.engine.ejb.impl.bpel.BPELAuditTrailBean
+ com.collaxa.cube.engine.ejb.impl.bpel.BPELAuditTrailBean
For BPEL Audit Trail
Normal
-
+
Oracle SOA Coherence Adapter
NORMAL
- oracle.tip.adapter.coherence
+ oracle.tip.adapter.coherence
Oracle SOA Cloud Adapter running
Normal
-
+
Oracle SOA FTP Adapter
NORMAL
- oracle.tip.adapter.ftp
+ oracle.tip.adapter.ftp
Oracle SOA FTP Adapter running
Normal
-
+
Oracle SOA LDAP Adapter
NORMAL
- oracle.tip.adapter.ldap
+ oracle.tip.adapter.ldap
Oracle SOA LDAP Adapter running
Normal
-
+
Oracle SOA MQ Adapter
NORMAL
- oracle.tip.adapter.mq
+ oracle.tip.adapter.mq
Oracle SOA MQ Adapter running
Normal
Oracle SOA MSMQ Adapter
NORMAL
- oracle.tip.adapter.msmq
+ oracle.tip.adapter.msmq
Oracle SOA MSMQ Adapter running
- Normal
-
+ Normal
+
Oracle SOA Socket Adapter
NORMAL
- oracle.tip.adapter.socket
+ oracle.tip.adapter.socket
Oracle SOA Socket Adapter running
Normal
Oracle SOA UMS Adapter
NORMAL
- oracle.tip.adapter.ums
+ oracle.tip.adapter.ums
Oracle SOA UMS Adapter running
Normal
-
+
DVM and XRef
NORMAL
- oracle.tip.dvm
+ oracle.tip.dvm
Oracle SOA DVM and XRef
Normal
HTTP Client Contention
FATAL
- HTTP Client Contention
+ HTTP Client Contention
HTTP requests using HTTPClient may cause a Heap retention and in consequence an OutOfMemory
Please reivew: SOA 11g: "STUCK" Threads due to Contention in the Classes HTTPClient/StreamDemultiplexor Result in Server Performance Issues (Doc ID 1564631.1)
-
+
DMS Collector
FATAL
- DMS Collector Stucked
+ DMS Collector Stucked
Related to the DMS metrics engine, and probably proportional to having a lot of active managed servers.
If have a very high transactional throughput or disruption, then the JVM needs the headroom to process this in addition to the inherent DMS metrics load. Apply patch for Bug 16298679 - DMS METRIC SAMPLER HAS PERFORMANCE BOTTLENECK. The patch has significantly improved behaviour,and with the memory increase and some other limited JRockit tuning, The admin server shuold be stable.
BPEL XPATH Function
FATAL
- BPELXPATHFUNCTIONRESOLVER Stucked
+ BPELXPATHFUNCTIONRESOLVER Stucked
The issue has been identified in Bug:11778352 and is caused by the HashMap.get method call not thread safe in BPELXPATHFUNCTIONRESOLVER
Apply Patch:13353142
-
+
Cluster Deployment
FATAL
- Cluster Deployment Stucked
+ Cluster Deployment Stucked
You are unable to deploy a SOA composite in a clustered environment.
The problem is likely to be caused by incorrect Coherence configuration. Please refer to support note: 1437883.1 -
-
+
Oracle MFT Idle Thread
IGNORE
Oracle MFT Idle Thread
MFT Idle Thread
Ignore
-
+
Oracle MFT Thread
NORMAL
oracle.tip.mft
MFT Thread running
Normal
-
-
\ No newline at end of file
+
+
+ Idle Task Thread
+ IGNORE
+ java.util.concurrent.LinkedBlockingQueue.poll
+ Idle Task Thread waiting for an event
+ Ignore
+
+
diff --git a/src/java/com/oracle/ateam/threadlogic/resources/NonWLSGroups.xml b/src/java/com/oracle/ateam/threadlogic/resources/NonWLSGroups.xml
index e7ee62f..a8da85a 100644
--- a/src/java/com/oracle/ateam/threadlogic/resources/NonWLSGroups.xml
+++ b/src/java/com/oracle/ateam/threadlogic/resources/NonWLSGroups.xml
@@ -44,7 +44,7 @@ Using individual pattern entries for easier read:
Code Optimization Thread
VM Thread
-
+
Or using | as delimiter to provide a choice of patterns:
@@ -57,7 +57,7 @@ Or using | as delimiter to provide a choice of patterns:
Socket Read
-
+
# MatchLocation used to locate for a matching pattern against thread stack or thread name : only allowed values are stack or name
@@ -90,10 +90,10 @@ Or using | as delimiter to provide a choice of patterns:
DB Execute
Socket Read
-
+
-# In above sample, the Complex Group "Oracle AQ Adapter" uses 2 underlying simple groups.
+# In above sample, the Complex Group "Oracle AQ Adapter" uses 2 underlying simple groups.
# All threads belong to the simple group referred by "Oracle AQ AdapterTemp" should be included while all threads matching the "Oracle SOA DFW" should be excluded.
# Also both Simple and Complex Groups can exclude certain advisories - as in downgrade those advisories from overall thread health consideration
@@ -102,7 +102,7 @@ Or using | as delimiter to provide a choice of patterns:
-->
-
Non WLS Groups
@@ -141,30 +141,40 @@ xmlns="http://java.net/projects/threadlogic">
- IWay Adapter
+ Tomcat Task Threads
true
true
stack
- com.ibi.adapters.util
+ org.apache.tomcat.util.threads.TaskThread
- SAP Connector
+ Tomcat NIO Threads
true
true
stack
- com.sap.conn.jco
+ org.apache.tomcat.util.net.NioBlockingSelector
+ org.apache.tomcat.util.net.NioEndpoint
- Other Poller
+ IWay Adapter
true
true
stack
- SelectorImpl.select
+ com.ibi.adapters.util
+
+
+
+ SAP Connector
+ true
+ true
+ stack
+
+ com.sap.conn.jco
@@ -177,8 +187,8 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
-
+
+
LDAP
true
@@ -190,7 +200,7 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
+
RMI TCP Threads
@@ -203,7 +213,7 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
+
Java Timer
@@ -232,10 +242,10 @@ xmlns="http://java.net/projects/threadlogic">
oracle.repackaged.ons
oracle.repackaged.ucp
-
+
Socket Read
-
+
Quartz Pool
true
@@ -259,7 +269,7 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
+
Wily
@@ -276,8 +286,8 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
-
+
+
EHCache
true
@@ -286,18 +296,18 @@ xmlns="http://java.net/projects/threadlogic">
net.sf.ehcache.store.DiskStore
-
+
Adventnet SNMP Agent
true
true
stack
-
- com.adventnet
+
+ com.adventnet
Socket Read
-
+
Sun Norm Updaters
@@ -309,8 +319,8 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
-
+
+
Sun HSS Grouping Manager Threads
true
@@ -318,7 +328,7 @@ xmlns="http://java.net/projects/threadlogic">
name
grouping-manager-attr-thread
-
+
Sun JMX Client Notification Forwarders
@@ -330,7 +340,7 @@ xmlns="http://java.net/projects/threadlogic">
Socket Read
-
+
ZK Web Framework Component Library
@@ -339,8 +349,8 @@ xmlns="http://java.net/projects/threadlogic">
stack
org.zkoss.zk
-
-
+
+
Grizzly Threads
true
@@ -348,8 +358,8 @@ xmlns="http://java.net/projects/threadlogic">
stack
com.sun.grizzly
-
-
+
+
HK2 Transactions Notifiers
true
@@ -357,8 +367,8 @@ xmlns="http://java.net/projects/threadlogic">
stack
org.jvnet.hk2.config.Transactions.Notifier
-
-
+
+
Catalina Container Background Processors
true
@@ -366,34 +376,35 @@ xmlns="http://java.net/projects/threadlogic">
stack
org.apache.catalina.core.ContainerBase.ContainerBackgroundProcessor
-
-
+
+
- JMX Client Heartbeat
+ Datastax Threads
true
true
- name
+ stack
- JMX client heartbeat
-
-
+ com.datastax.shaded.netty.util.DefaultThreadFactory
+ com.datastax.shaded.netty.util.concurrent.DefaultThreadFactory
+
+
- JMX Server Connection Timeout
+ JMX Client Heartbeat
true
true
name
- JMX server connection timeout
-
+ JMX client heartbeat
+
- Log4J
+ JMX Server Connection Timeout
true
true
name
- Log4J
-
+ JMX server connection timeout
+
@@ -413,7 +424,7 @@ xmlns="http://java.net/projects/threadlogic">
name
UCP-worker-thread
-
+
Sun Cacao
@@ -422,7 +433,7 @@ xmlns="http://java.net/projects/threadlogic">
stack
com.sun.cacao
-
+
Sun HSS Domain Temp
@@ -432,7 +443,7 @@ xmlns="http://java.net/projects/threadlogic">
com.sun.hss.domain
com.sun.hss.services
-
+
Sun HSS Domain
@@ -445,8 +456,8 @@ xmlns="http://java.net/projects/threadlogic">
Sun Norm Updaters
Grizzly Threads
Sun Cacao
-
-
+
+
Iona CORBA
true
@@ -454,7 +465,7 @@ xmlns="http://java.net/projects/threadlogic">
stack
com.iona
-
+
Acsera
@@ -463,7 +474,7 @@ xmlns="http://java.net/projects/threadlogic">
stack
com.acsera.javaagent
-
+
Cavisson
@@ -472,7 +483,7 @@ xmlns="http://java.net/projects/threadlogic">
stack
com.cavisson
-
+
ODI
@@ -482,7 +493,7 @@ xmlns="http://java.net/projects/threadlogic">
oracle.odi
com.sunopsis
-
+
Stellent
@@ -491,6 +502,33 @@ xmlns="http://java.net/projects/threadlogic">
stack
com.stellent.cis
-
+
+
+
+ Apache Log4j
+ true
+ true
+ stack
+
+ org.apache.log4j
+
+
+
+ Apache Logging
+ true
+ true
+ stack
+
+ org.apache.juli
+
+
+
+ Other Poller
+ true
+ true
+ stack
+
+ SelectorImpl.select
+