scripts update to fix some bugs; ignore heading 30 latency data in py…

…thon script as part of warmup

src/main/java/baoying/orderbook/testtool/LatencyMessageCallback.java

@@ -40,7 +40,7 @@ public void fromApp(Message paramMessage, SessionID paramSessionID) {
             String clientCompmID = paramMessage.getHeader().getString(56);
             Path e2eTimeFile = Paths.get("log/e2e_"+clientCompmID+".csv");
             if (!Files.exists(e2eTimeFile)) {
-                Files.write(e2eTimeFile, ("clientOrdID,sendTime,sendTimeNano,recvFromClient_sysNano,enterInputQ_sysNano,pickFromInputQ_sysNano,matched_sysNano,pickedFromOutputBus_nano,newER,matchER" + "\n").getBytes(), APPEND, CREATE);
+                Files.write(e2eTimeFile, ("sendTime,sendTimeNano,recvFromClient_sysNano,enterInputQ_sysNano,pickFromInputQ_sysNano,matched_sysNano,pickedFromOutputBus_nano,newER,matchER,clientOrdID" + "\n").getBytes(), APPEND, CREATE);
             }
 
             List<Long> times = erAckTimes.get(clientOrdID);
@@ -49,7 +49,7 @@ public void fromApp(Message paramMessage, SessionID paramSessionID) {
                 String serverTimes = paramMessage.getString(latencyTimesField);
                 long newAck =times.get(0) ;
                 long matchAck = times.get(1);
-                Files.write(e2eTimeFile, (clientOrdID+","+serverTimes+","+newAck+","+matchAck+"\n").getBytes(), APPEND, CREATE);
+                Files.write(e2eTimeFile, (serverTimes+","+newAck+","+matchAck+","+clientOrdID+"\n").getBytes(), APPEND, CREATE);
                 erAckTimes.removeAll(clientOrdID);
             }
 

src/main/resources/DefaultDynamicSessionQFJServer.qfj.config.txt

@@ -8,7 +8,10 @@ ReconnectInterval=5
 #http://www.quickfixj.org/quickfixj/usermanual/1.5.0/usage/configuration.html#Logging
 #SLFJ is used, see log4j2.xml.
 
-PersistMessages=Y
+#Persist is disabled for ultra performance. OMS(or post trade) should be used to query possible loss data if there is disconnection.
+#You can set it to Y and assign FileStorePath if you still expect FIX re-send gap messages.
+PersistMessages=N
+#FileStorePath is still required for seq number etc(?)
 FileStorePath=./log/engineQFJFileStore
 
 SocketAcceptAddress=localhost

src/test/bash/post_parsing_all_zipped_output.sh

@@ -13,7 +13,8 @@ function parseZipFile(){
 
 	unzip -p $zipfile $name/log/e2e_LxTxCx_FIX_RT*          > $name.e2e_LxTxCx_FIX_RT.csv.tmp
     head -1 $name.e2e_LxTxCx_FIX_RT.csv.tmp                 > $name.e2e_LxTxCx_FIX_RT.csv
-    grep -v $(head -1 $name.e2e_LxTxCx_FIX_RT.csv.tmp) $name.e2e_LxTxCx_FIX_RT.csv.tmp >> $name.e2e_LxTxCx_FIX_RT.csv
+    #Sort to avoid possible diagram problem. sendTime(YYY-MM-DD...) is the first column.
+    sort $name.e2e_LxTxCx_FIX_RT.csv.tmp | grep -v $(head -1 $name.e2e_LxTxCx_FIX_RT.csv.tmp)  >> $name.e2e_LxTxCx_FIX_RT.csv
     rm $name.e2e_LxTxCx_FIX_RT.csv.tmp
 
 	python_script_file=$MYSCRIPTDIR/parseLatencyData.py

src/test/bash/run_single_scenario.sh

@@ -92,7 +92,8 @@ function startBackgroundOrder(){
 
 function startLatencyOrder(){
 
-    local tmp_latency_rate_per_min_single_side=${1:-60}
+    local tmp_latency_rate_per_min=${1:-60}
+    local tmp_latency_rate_per_min_single_side=$((${tmp_latency_rate_per_min}/2))
     echo "begin sending latency orders - tmp_latency_rate_per_min_single_side:${tmp_latency_rate_per_min_single_side}"
 
     java ${JVMOptions_sending} -cp ${jarfile} baoying.orderbook.testtool.FirstQFJClientBatch -clientNum 1 -ratePerMinute ${tmp_latency_rate_per_min_single_side} -client_prefix 'LxTxCx_FIX_RT_B' -symbol USDJPY -side Bid   -qty 2 -ordType Market -d ${duration_in_second} &

src/test/python/parseLatencyData.py

@@ -25,6 +25,8 @@ def getE2E(inputE2EFile):
 
     df=pd.read_csv(inputE2EFile)
 
+    df.describe()
+
     df["e2e_new_ms"   ] = (df["newER"    ] - df["sendTimeNano" ]            )/1000000
     df["e2e_match_ms" ] = (df["matchER"  ] - df["sendTimeNano" ]            )/1000000
 
@@ -286,15 +288,16 @@ def genPlotSysInfo(plt, shape, row_start_index, sysInfoContentList):
     #https://stackoverflow.com/questions/16522111/python-syntax-for-if-a-or-b-or-c-but-not-all-of-them
     osLineList      = [x for x in sysInfoContentList if x.startswith("os ")]
     gcLineList      = [x for x in sysInfoContentList if x.startswith("gc ")]
+    os_gc_text = "".join(osLineList) + "".join(gcLineList)
     plt.subplot2grid(shape,(row_start_index,0))
-    plt.text(0, 0 ,"".join(osLineList) + "".join(gcLineList))
+    plt.text(0, 0 , os_gc_text[ :-1])#string[:-1] remove the last carriage return character
 
 
     runtime_prefix_list = ["runtime ClassPath",
                            "runtime InputArguments" ,
-                           "runtime Name"           ,
+                           #"runtime Name"           ,
                            "runtime VmName"         ,
-                           "runtime VmVendor",
+                           #"runtime VmVendor",
                            "runtime VmVersion"  ]
     #https://stackoverflow.com/questions/30919275/inserting-period-after-every-3-chars-in-a-string
     runtimeLineList = ['\n          '.join(x[i:i+100] for i in range(0, len(x), 100)) for x in sysInfoContentList if x.startswith("runtime ") and x.split(":")[0] in runtime_prefix_list]
@@ -304,8 +307,9 @@ def genPlotSysInfo(plt, shape, row_start_index, sysInfoContentList):
     java_command = [re.findall('sun.java.command=.+?,', x)[0] for x in sysInfoContentList if x.startswith("runtime SystemProperties")]
     java_command_string = '\n          '.join(java_command[0][i:i+100] for i in range(0, len(java_command[0]), 100))
 
+    text = java_command_string + '\n' + ( "".join(runtimeLineList))
     plt.subplot2grid(shape,(row_start_index,1), colspan=2)
-    plt.text(0, 0 ,  java_command_string + '\n' + ( "".join(runtimeLineList)) )
+    plt.text(0, 0 ,  text[:-1] ) #text[:-1] remove the last carriage return character
 
     return row_start_index+1
 
@@ -325,7 +329,7 @@ def genPlot(plotTitle,df_e2e,df_sysUsage, df_vmstat, sysInfoContentList, output_
     #http://blog.csdn.net/han_xiaoyang/article/details/49797143
     #https://matplotlib.org/users/gridspec.html
     #================================df_latency==============================
-    shape=(9,4)
+    shape=(10,4)
     next_row_index = 0
     next_row_index = genPlotSysInfo(    plt, shape, next_row_index, sysInfoContentList)
     next_row_index = genPlotE2E(        plt, shape, next_row_index, df_e2e, output_file_prefix)
@@ -338,9 +342,9 @@ def genPlot(plotTitle,df_e2e,df_sysUsage, df_vmstat, sysInfoContentList, output_
 
 
 #python /c/baoying.wang/ws/gitnas/OrderBook/src/test/python/parseLatencyData.py \
-#    BlockingQueue_X_bg5000perSec_20171229_022833_duration600Sec.latency.data.csv \
-#    BlockingQueue_X_bg5000perSec_20171229_022833_duration600Sec.sysUsage.csv \
-#    BlockingQueue_X_bg5000perSec_20171229_022833_duration600Sec.sysInfo.txt \
+#    BlockingQueue_X_bg50perSec_lt60perMin_duration600sec_20180107_115910.e2e_LxTxCx_FIX_RT.csv \
+#    BlockingQueue_X_bg50perSec_lt60perMin_duration600sec_20180107_115910.sysUsage.csv \
+#    BlockingQueue_X_bg50perSec_lt60perMin_duration600sec_20180107_115910.sysInfo.txt \
 #    vmstat_since20171229.log.csv \
 #    output_file_prefix
 
@@ -369,6 +373,7 @@ def genPlot(plotTitle,df_e2e,df_sysUsage, df_vmstat, sysInfoContentList, output_
 #sysInfoContentList = [x.strip() for x in sysInfoContentList]
 
 plotTitle=output_file_prefix
-genPlot(plotTitle,df_e2e,df_sysUsage,df_vmstat, sysInfoContentList,output_file_prefix)
+#df_e2e[30:] means remove the deading 0~29, because the session(FIX) needs setup(e.g. some lazy init) initially.
+genPlot(plotTitle,df_e2e[30:],df_sysUsage,df_vmstat, sysInfoContentList,output_file_prefix)