From f9fc191bce8dbb781d652b99321c576efe53aa68 Mon Sep 17 00:00:00 2001 From: "baoying.wang" Date: Sun, 7 Jan 2018 18:23:51 +0800 Subject: [PATCH] scripts update to fix some bugs; ignore heading 30 latency data in python script as part of warmup --- .../testtool/LatencyMessageCallback.java | 4 ++-- ...aultDynamicSessionQFJServer.qfj.config.txt | 5 +++- .../bash/post_parsing_all_zipped_output.sh | 3 ++- src/test/bash/run_single_scenario.sh | 3 ++- src/test/python/parseLatencyData.py | 23 +++++++++++-------- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/main/java/baoying/orderbook/testtool/LatencyMessageCallback.java b/src/main/java/baoying/orderbook/testtool/LatencyMessageCallback.java index b1191a0..8a5945b 100644 --- a/src/main/java/baoying/orderbook/testtool/LatencyMessageCallback.java +++ b/src/main/java/baoying/orderbook/testtool/LatencyMessageCallback.java @@ -40,7 +40,7 @@ public void fromApp(Message paramMessage, SessionID paramSessionID) { String clientCompmID = paramMessage.getHeader().getString(56); Path e2eTimeFile = Paths.get("log/e2e_"+clientCompmID+".csv"); if (!Files.exists(e2eTimeFile)) { - Files.write(e2eTimeFile, ("clientOrdID,sendTime,sendTimeNano,recvFromClient_sysNano,enterInputQ_sysNano,pickFromInputQ_sysNano,matched_sysNano,pickedFromOutputBus_nano,newER,matchER" + "\n").getBytes(), APPEND, CREATE); + Files.write(e2eTimeFile, ("sendTime,sendTimeNano,recvFromClient_sysNano,enterInputQ_sysNano,pickFromInputQ_sysNano,matched_sysNano,pickedFromOutputBus_nano,newER,matchER,clientOrdID" + "\n").getBytes(), APPEND, CREATE); } List times = erAckTimes.get(clientOrdID); @@ -49,7 +49,7 @@ public void fromApp(Message paramMessage, SessionID paramSessionID) { String serverTimes = paramMessage.getString(latencyTimesField); long newAck =times.get(0) ; long matchAck = times.get(1); - Files.write(e2eTimeFile, (clientOrdID+","+serverTimes+","+newAck+","+matchAck+"\n").getBytes(), APPEND, CREATE); + Files.write(e2eTimeFile, (serverTimes+","+newAck+","+matchAck+","+clientOrdID+"\n").getBytes(), APPEND, CREATE); erAckTimes.removeAll(clientOrdID); } diff --git a/src/main/resources/DefaultDynamicSessionQFJServer.qfj.config.txt b/src/main/resources/DefaultDynamicSessionQFJServer.qfj.config.txt index 00f2c45..ed9b5fb 100644 --- a/src/main/resources/DefaultDynamicSessionQFJServer.qfj.config.txt +++ b/src/main/resources/DefaultDynamicSessionQFJServer.qfj.config.txt @@ -8,7 +8,10 @@ ReconnectInterval=5 #http://www.quickfixj.org/quickfixj/usermanual/1.5.0/usage/configuration.html#Logging #SLFJ is used, see log4j2.xml. -PersistMessages=Y +#Persist is disabled for ultra performance. OMS(or post trade) should be used to query possible loss data if there is disconnection. +#You can set it to Y and assign FileStorePath if you still expect FIX re-send gap messages. +PersistMessages=N +#FileStorePath is still required for seq number etc(?) FileStorePath=./log/engineQFJFileStore SocketAcceptAddress=localhost diff --git a/src/test/bash/post_parsing_all_zipped_output.sh b/src/test/bash/post_parsing_all_zipped_output.sh index 0156f62..3f55e40 100644 --- a/src/test/bash/post_parsing_all_zipped_output.sh +++ b/src/test/bash/post_parsing_all_zipped_output.sh @@ -13,7 +13,8 @@ function parseZipFile(){ unzip -p $zipfile $name/log/e2e_LxTxCx_FIX_RT* > $name.e2e_LxTxCx_FIX_RT.csv.tmp head -1 $name.e2e_LxTxCx_FIX_RT.csv.tmp > $name.e2e_LxTxCx_FIX_RT.csv - grep -v $(head -1 $name.e2e_LxTxCx_FIX_RT.csv.tmp) $name.e2e_LxTxCx_FIX_RT.csv.tmp >> $name.e2e_LxTxCx_FIX_RT.csv + #Sort to avoid possible diagram problem. sendTime(YYY-MM-DD...) is the first column. + sort $name.e2e_LxTxCx_FIX_RT.csv.tmp | grep -v $(head -1 $name.e2e_LxTxCx_FIX_RT.csv.tmp) >> $name.e2e_LxTxCx_FIX_RT.csv rm $name.e2e_LxTxCx_FIX_RT.csv.tmp python_script_file=$MYSCRIPTDIR/parseLatencyData.py diff --git a/src/test/bash/run_single_scenario.sh b/src/test/bash/run_single_scenario.sh index 2f9ff34..eb2c49c 100644 --- a/src/test/bash/run_single_scenario.sh +++ b/src/test/bash/run_single_scenario.sh @@ -92,7 +92,8 @@ function startBackgroundOrder(){ function startLatencyOrder(){ - local tmp_latency_rate_per_min_single_side=${1:-60} + local tmp_latency_rate_per_min=${1:-60} + local tmp_latency_rate_per_min_single_side=$((${tmp_latency_rate_per_min}/2)) echo "begin sending latency orders - tmp_latency_rate_per_min_single_side:${tmp_latency_rate_per_min_single_side}" java ${JVMOptions_sending} -cp ${jarfile} baoying.orderbook.testtool.FirstQFJClientBatch -clientNum 1 -ratePerMinute ${tmp_latency_rate_per_min_single_side} -client_prefix 'LxTxCx_FIX_RT_B' -symbol USDJPY -side Bid -qty 2 -ordType Market -d ${duration_in_second} & diff --git a/src/test/python/parseLatencyData.py b/src/test/python/parseLatencyData.py index f994e80..ee10dac 100644 --- a/src/test/python/parseLatencyData.py +++ b/src/test/python/parseLatencyData.py @@ -25,6 +25,8 @@ def getE2E(inputE2EFile): df=pd.read_csv(inputE2EFile) + df.describe() + df["e2e_new_ms" ] = (df["newER" ] - df["sendTimeNano" ] )/1000000 df["e2e_match_ms" ] = (df["matchER" ] - df["sendTimeNano" ] )/1000000 @@ -286,15 +288,16 @@ def genPlotSysInfo(plt, shape, row_start_index, sysInfoContentList): #https://stackoverflow.com/questions/16522111/python-syntax-for-if-a-or-b-or-c-but-not-all-of-them osLineList = [x for x in sysInfoContentList if x.startswith("os ")] gcLineList = [x for x in sysInfoContentList if x.startswith("gc ")] + os_gc_text = "".join(osLineList) + "".join(gcLineList) plt.subplot2grid(shape,(row_start_index,0)) - plt.text(0, 0 ,"".join(osLineList) + "".join(gcLineList)) + plt.text(0, 0 , os_gc_text[ :-1])#string[:-1] remove the last carriage return character runtime_prefix_list = ["runtime ClassPath", "runtime InputArguments" , - "runtime Name" , + #"runtime Name" , "runtime VmName" , - "runtime VmVendor", + #"runtime VmVendor", "runtime VmVersion" ] #https://stackoverflow.com/questions/30919275/inserting-period-after-every-3-chars-in-a-string runtimeLineList = ['\n '.join(x[i:i+100] for i in range(0, len(x), 100)) for x in sysInfoContentList if x.startswith("runtime ") and x.split(":")[0] in runtime_prefix_list] @@ -304,8 +307,9 @@ def genPlotSysInfo(plt, shape, row_start_index, sysInfoContentList): java_command = [re.findall('sun.java.command=.+?,', x)[0] for x in sysInfoContentList if x.startswith("runtime SystemProperties")] java_command_string = '\n '.join(java_command[0][i:i+100] for i in range(0, len(java_command[0]), 100)) + text = java_command_string + '\n' + ( "".join(runtimeLineList)) plt.subplot2grid(shape,(row_start_index,1), colspan=2) - plt.text(0, 0 , java_command_string + '\n' + ( "".join(runtimeLineList)) ) + plt.text(0, 0 , text[:-1] ) #text[:-1] remove the last carriage return character return row_start_index+1 @@ -325,7 +329,7 @@ def genPlot(plotTitle,df_e2e,df_sysUsage, df_vmstat, sysInfoContentList, output_ #http://blog.csdn.net/han_xiaoyang/article/details/49797143 #https://matplotlib.org/users/gridspec.html #================================df_latency============================== - shape=(9,4) + shape=(10,4) next_row_index = 0 next_row_index = genPlotSysInfo( plt, shape, next_row_index, sysInfoContentList) next_row_index = genPlotE2E( plt, shape, next_row_index, df_e2e, output_file_prefix) @@ -338,9 +342,9 @@ def genPlot(plotTitle,df_e2e,df_sysUsage, df_vmstat, sysInfoContentList, output_ #python /c/baoying.wang/ws/gitnas/OrderBook/src/test/python/parseLatencyData.py \ -# BlockingQueue_X_bg5000perSec_20171229_022833_duration600Sec.latency.data.csv \ -# BlockingQueue_X_bg5000perSec_20171229_022833_duration600Sec.sysUsage.csv \ -# BlockingQueue_X_bg5000perSec_20171229_022833_duration600Sec.sysInfo.txt \ +# BlockingQueue_X_bg50perSec_lt60perMin_duration600sec_20180107_115910.e2e_LxTxCx_FIX_RT.csv \ +# BlockingQueue_X_bg50perSec_lt60perMin_duration600sec_20180107_115910.sysUsage.csv \ +# BlockingQueue_X_bg50perSec_lt60perMin_duration600sec_20180107_115910.sysInfo.txt \ # vmstat_since20171229.log.csv \ # output_file_prefix @@ -369,6 +373,7 @@ def genPlot(plotTitle,df_e2e,df_sysUsage, df_vmstat, sysInfoContentList, output_ #sysInfoContentList = [x.strip() for x in sysInfoContentList] plotTitle=output_file_prefix -genPlot(plotTitle,df_e2e,df_sysUsage,df_vmstat, sysInfoContentList,output_file_prefix) +#df_e2e[30:] means remove the deading 0~29, because the session(FIX) needs setup(e.g. some lazy init) initially. +genPlot(plotTitle,df_e2e[30:],df_sysUsage,df_vmstat, sysInfoContentList,output_file_prefix)