This repository has been archived by the owner on Jan 8, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
FILES ARE BEING WRITTEN WHILE SOURCE IS PROCESSING
Luis Lázaro edited this page Jun 19, 2018
·
1 revision
FILES ARE BEING WRITTEN WHILE SOURCE IS PROCESSING.
| ASSAY NUMBER | ITERATION | FILES COUNT | LINES per file COUNT | File SIZE (bytes) | Line SIZE (bytes) | Source VFS Event COUNT | post.process.file | timeout.start.post.process (default 60 seconds) | delay.between.runs (seconds) | OBSERVATIONS | JSON raw data metrics |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 1 | - | 100 | 1001 | 60923 | 59 | 100100 | delete | 60 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_09:11:55.277","FilesCount":"100","EventCount":"100100","CountSizeProcMb":"5.658775329589844","EventThroughput":"553","LastFileSent":"1529392315277","CountSizeProcKb":"5794","StartTime_Human":"2018-06-19_09:08:53.999","LastEventSent":"1529392315277","StarTime":"1529392133999","CountSizeProcBytes":"5933656","LastEventSent_Human":"2018-06-19_09:11:55.277","RunningTime":"0 0:3:8"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"100100","ChannelSize":"0","EventTakeAttemptCount":"100126","StartTime":"1529392134155","EventPutAttemptCount":"100100","EventPutSuccessCount":"100100","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"1","ConnectionClosedCount":"0","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"100100","StartTime":"1529392134157","EventDrainSuccessCount":"100100","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 2 | - | 1000 | 1001 | 60923 | 59 | 1001000 | delete | 60 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_09:20:18.870","FilesCount":"1000","EventCount":"1001000","CountSizeProcMb":"57.0628023147583","EventThroughput":"11000","LastFileSent":"1529392818870","CountSizeProcKb":"58432","StartTime_Human":"2018-06-19_09:18:46.950","LastEventSent":"1529392818870","StarTime":"1529392726950","CountSizeProcBytes":"59834685","LastEventSent_Human":"2018-06-19_09:20:18.870","RunningTime":"0 0:1:37"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"1001000","ChannelSize":"0","EventTakeAttemptCount":"1001015","StartTime":"1529392727039","EventPutAttemptCount":"1001000","EventPutSuccessCount":"1001000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"1","ConnectionClosedCount":"0","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"1001000","StartTime":"1529392727041","EventDrainSuccessCount":"1001000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 3 | - | 10000 | 1001 | 60923 | 59 | 10010000 | delete | 60 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_09:27:25.241","FilesCount":"10000","EventCount":"10010000","CountSizeProcMb":"571.0998106002808","EventThroughput":"56553","LastFileSent":"1529393245241","CountSizeProcKb":"584806","StartTime_Human":"2018-06-19_09:24:27.719","LastEventSent":"1529393245240","StarTime":"1529393067719","CountSizeProcBytes":"598841555","LastEventSent_Human":"2018-06-19_09:27:25.240","RunningTime":"0 0:3:3"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"10010000","ChannelSize":"0","EventTakeAttemptCount":"10010025","StartTime":"1529393067802","EventPutAttemptCount":"10010000","EventPutSuccessCount":"10010000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"1","ConnectionClosedCount":"0","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"10010000","StartTime":"1529393067804","EventDrainSuccessCount":"10010000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 4 | - | 20000 | 1001 | 60923 | 59 | 20020000 | delete | 60 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_09:38:13.640","FilesCount":"20000","EventCount":"20020000","CountSizeProcMb":"1142.4865064620972","EventThroughput":"47780","LastFileSent":"1529393893640","CountSizeProcKb":"1169906","StartTime_Human":"2018-06-19_09:31:14.116","LastEventSent":"1529393893640","StarTime":"1529393474116","CountSizeProcBytes":"1197983931","LastEventSent_Human":"2018-06-19_09:38:13.640","RunningTime":"0 0:7:18"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"20020000","ChannelSize":"0","EventTakeAttemptCount":"20020053","StartTime":"1529393474194","EventPutAttemptCount":"20020000","EventPutSuccessCount":"20020000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"2","ConnectionClosedCount":"1","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"20020000","StartTime":"1529393474196","EventDrainSuccessCount":"20020000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 5 | - | 40000 | 1001 | 60923 | 59 | 40039538 | delete | 60 | 10 | Fail, Event lost | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_09:53:33.940","FilesCount":"40000","EventCount":"40039538","CountSizeProcMb":"2283.627167701721","EventThroughput":"87998","LastFileSent":"1529394813940","CountSizeProcKb":"2338434","StartTime_Human":"2018-06-19_09:45:58.491","LastEventSent":"1529394813902","StarTime":"1529394358491","CountSizeProcBytes":"2394556641","LastEventSent_Human":"2018-06-19_09:53:33.902","RunningTime":"0 0:35:20"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","ChannelSize":"0","EventTakeSuccessCount":"40039538","EventTakeAttemptCount":"40039793","StartTime":"1529394358572","EventPutAttemptCount":"40039538","EventPutSuccessCount":"40039538","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"6","ConnectionClosedCount":"5","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"40039538","StartTime":"1529394358575","EventDrainSuccessCount":"40039538","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 5 | 1 | 40000 | 1001 | 60923 | 59 | 40040000 | do nothing | - | 10 | If disable post process, ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_10:40:09.165","FilesCount":"40000","EventCount":"40040000","CountSizeProcMb":"2285.010000228882","EventThroughput":"74011","LastFileSent":"1529397609165","CountSizeProcKb":"2339850","StartTime_Human":"2018-06-19_10:31:07.668","LastEventSent":"1529397609165","StarTime":"1529397067668","CountSizeProcBytes":"2396006646","LastEventSent_Human":"2018-06-19_10:40:09.165","RunningTime":"0 0:9:41"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"40040000","ChannelSize":"0","EventTakeAttemptCount":"40040063","StartTime":"1529397067823","EventPutAttemptCount":"40040000","EventPutSuccessCount":"40040000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"2","ConnectionClosedCount":"1","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"40040000","StartTime":"1529397067825","EventDrainSuccessCount":"40040000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 5 | 2 | 40000 | 1001 | 60923 | 59 | 40036997 | delete | 60 | 15 | Increasing delay.between.runs doesnt help here | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_10:58:52.268","FilesCount":"40000","EventCount":"40036997","CountSizeProcMb":"2285.00440120697","EventThroughput":"58964","LastFileSent":"1529398732268","CountSizeProcKb":"2339844","StartTime_Human":"2018-06-19_10:47:32.533","LastEventSent":"1529398732268","StarTime":"1529398052533","CountSizeProcBytes":"2396000775","LastEventSent_Human":"2018-06-19_10:58:52.268","RunningTime":"0 0:14:41"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","ChannelSize":"0","EventTakeSuccessCount":"40036997","EventTakeAttemptCount":"40037091","StartTime":"1529398052634","EventPutAttemptCount":"40036997","EventPutSuccessCount":"40036997","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"3","ConnectionClosedCount":"2","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"40036997","StartTime":"1529398052636","EventDrainSuccessCount":"40036997","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 5 | 3 | 40000 | 1001 | 60923 | 59 | 40040000 | delete | 90 | 10 | If increase timeout for deleting files, ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_11:13:00.443","FilesCount":"40000","EventCount":"40040000","CountSizeProcMb":"2285.260108947754","EventThroughput":"79602","LastFileSent":"1529399580443","CountSizeProcKb":"2340106","StartTime_Human":"2018-06-19_11:04:37.144","LastEventSent":"1529399580443","StarTime":"1529399077144","CountSizeProcBytes":"2396268904","LastEventSent_Human":"2018-06-19_11:13:00.443","RunningTime":"0 0:14:51"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"40040000","ChannelSize":"0","EventTakeAttemptCount":"40040100","StartTime":"1529399077244","EventPutAttemptCount":"40040000","EventPutSuccessCount":"40040000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"3","ConnectionClosedCount":"2","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"40040000","StartTime":"1529399077246","EventDrainSuccessCount":"40040000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 6 | - | 80000 | 1001 | 60923 | 59 | 80080000 | delete | 180 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_12:09:04.412","FilesCount":"80000","EventCount":"80080000","CountSizeProcMb":"4570.473867416382","EventThroughput":"35215","LastFileSent":"1529402944412","CountSizeProcKb":"4680165","StartTime_Human":"2018-06-19_11:31:10.288","LastEventSent":"1529402944412","StarTime":"1529400670288","CountSizeProcBytes":"4792489206","LastEventSent_Human":"2018-06-19_12:09:04.412","RunningTime":"0 0:46:58"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","ChannelSize":"0","EventTakeSuccessCount":"80080000","EventTakeAttemptCount":"80080309","StartTime":"1529400670449","EventPutAttemptCount":"80080000","EventPutSuccessCount":"80080000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"8","ConnectionClosedCount":"7","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"80080000","StartTime":"1529400670451","EventDrainSuccessCount":"80080000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 7 | - | 100 | 10001 | 618987 | 59 | 1000100 | delete | 60 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_12:37:27.120","FilesCount":"100","EventCount":"1000100","CountSizeProcMb":"57.980730056762695","EventThroughput":"24392","LastFileSent":"1529404647120","CountSizeProcKb":"59372","StartTime_Human":"2018-06-19_12:36:45.397","LastEventSent":"1529404647119","StarTime":"1529404605397","CountSizeProcBytes":"60797202","LastEventSent_Human":"2018-06-19_12:37:27.119","RunningTime":"0 0:0:50"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","ChannelSize":"0","EventTakeSuccessCount":"1000100","EventTakeAttemptCount":"1000109","StartTime":"1529404605487","EventPutAttemptCount":"1000100","EventPutSuccessCount":"1000100","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"1","ConnectionClosedCount":"0","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"1000100","StartTime":"1529404605490","EventDrainSuccessCount":"1000100","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 8 | - | 1000 | 10001 | 618987 | 59 | 10001000 | delete | 60 | 10 | ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_12:41:27.951","FilesCount":"1000","EventCount":"10001000","CountSizeProcMb":"580.6235332489014","EventThroughput":"82652","LastFileSent":"1529404887951","CountSizeProcKb":"594558","StartTime_Human":"2018-06-19_12:39:26.107","LastEventSent":"1529404887951","StarTime":"1529404766107","CountSizeProcBytes":"608827902","LastEventSent_Human":"2018-06-19_12:41:27.951","RunningTime":"0 0:2:7"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"10001000","ChannelSize":"0","EventTakeAttemptCount":"10001020","StartTime":"1529404766190","EventPutAttemptCount":"10001000","EventPutSuccessCount":"10001000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"1","ConnectionClosedCount":"0","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"10001000","StartTime":"1529404766192","EventDrainSuccessCount":"10001000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 9 | 1 | 10000 | 10001 | 618987 | 59 | 99958806 | delete | 60 | 10 | Event lost | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_13:01:55.002","FilesCount":"10000","EventCount":"99958806","CountSizeProcMb":"5800.051624298096","EventThroughput":"95562","LastFileSent":"1529406115002","CountSizeProcKb":"5939252","StartTime_Human":"2018-06-19_12:44:28.436","LastEventSent":"1529406115002","StarTime":"1529405068436","CountSizeProcBytes":"6081794932","LastEventSent_Human":"2018-06-19_13:01:55.002","RunningTime":"0 0:17:52"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"99958806","ChannelSize":"0","EventTakeAttemptCount":"99958955","StartTime":"1529405068523","EventPutAttemptCount":"99958806","EventPutSuccessCount":"99958806","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"3","ConnectionClosedCount":"2","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"99958806","StartTime":"1529405068526","EventDrainSuccessCount":"99958806","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 9 | 2 | 10000 | 10001 | 618987 | 59 | 99917510 | do nothing | - | 15 | Timeout for deleting is not the problem. | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_14:36:48.810","FilesCount":"9993","EventCount":"99917510","CountSizeProcMb":"5797.503905296326","EventThroughput":"89773","LastFileSent":"1529411808810","CountSizeProcKb":"5936643","StartTime_Human":"2018-06-19_14:18:15.214","LastEventSent":"1529411808810","StarTime":"1529410695214","CountSizeProcBytes":"6079123455","LastEventSent_Human":"2018-06-19_14:36:48.810","RunningTime":"0 0:19:38"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"99917510","ChannelSize":"0","EventTakeAttemptCount":"99917630","StartTime":"1529410695374","EventPutAttemptCount":"99917510","EventPutSuccessCount":"99917510","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"4","ConnectionClosedCount":"3","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"99917510","StartTime":"1529410695376","EventDrainSuccessCount":"99917510","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 9 | 3 | 10000 | 10001 | 618987 | 59 | 99981899 | delete | 120 | 10 | Increase delay.between.runs. | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_14:03:26.045","FilesCount":"10000","EventCount":"99981899","CountSizeProcMb":"5803.222529411316","EventThroughput":"92404","LastFileSent":"1529409806045","CountSizeProcKb":"5942499","StartTime_Human":"2018-06-19_13:45:23.181","LastEventSent":"1529409806045","StarTime":"1529408723181","CountSizeProcBytes":"6085119867","LastEventSent_Human":"2018-06-19_14:03:26.045","RunningTime":"0 0:30:45"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","EventTakeSuccessCount":"99981899","ChannelSize":"0","EventTakeAttemptCount":"99982135","StartTime":"1529408723391","EventPutAttemptCount":"99981899","EventPutSuccessCount":"99981899","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"6","ConnectionClosedCount":"5","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"99981899","StartTime":"1529408723395","EventDrainSuccessCount":"99981899","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 9 | 4 | 10000 | 10001 | 618987 | 59 | 100010000 | do nothing | - | 20 | If increase delay.between.runs ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_14:58:21.591","FilesCount":"10000","EventCount":"100010000","CountSizeProcMb":"5805.536358833313","EventThroughput":"87805","LastFileSent":"1529413101591","CountSizeProcKb":"5944869","StartTime_Human":"2018-06-19_14:39:22.338","LastEventSent":"1529413101591","StarTime":"1529411962338","CountSizeProcBytes":"6087546093","LastEventSent_Human":"2018-06-19_14:58:21.591","RunningTime":"0 0:20:50"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","ChannelSize":"0","EventTakeSuccessCount":"100010000","EventTakeAttemptCount":"100010125","StartTime":"1529411962492","EventPutAttemptCount":"100010000","EventPutSuccessCount":"100010000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"4","ConnectionClosedCount":"3","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"100010000","StartTime":"1529411962493","EventDrainSuccessCount":"100010000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 9 | 5 | 10000 | 10001 | 618987 | 59 | 100010000 | delete | 120 | 20 | If increase timeout for deleting files, ok | {"SOURCE.stest1":{"LastFileSent_Human":"2018-06-19_15:21:36.335","FilesCount":"10000","EventCount":"100010000","CountSizeProcMb":"5801.23068523407","EventThroughput":"87882","LastFileSent":"1529414496335","CountSizeProcKb":"5940460","StartTime_Human":"2018-06-19_15:02:38.131","LastEventSent":"1529414496334","StarTime":"1529413358131","CountSizeProcBytes":"6083031267","LastEventSent_Human":"2018-06-19_15:21:36.334","RunningTime":"0 0:19:39"},"CHANNEL.ch3":{"ChannelCapacity":"10000000","ChannelFillPercentage":"0.0","Type":"CHANNEL","ChannelSize":"0","EventTakeSuccessCount":"100010000","EventTakeAttemptCount":"100010128","StartTime":"1529413358283","EventPutAttemptCount":"100010000","EventPutSuccessCount":"100010000","StopTime":"0"},"SINK.ktest1":{"ConnectionCreatedCount":"4","ConnectionClosedCount":"3","Type":"SINK","BatchCompleteCount":"0","BatchEmptyCount":"0","EventDrainAttemptCount":"100010000","StartTime":"1529413358286","EventDrainSuccessCount":"100010000","BatchUnderflowCount":"0","StopTime":"0","ConnectionFailedCount":"0"}} |
| 10 | - | 20000 | 10001 | 618987 | 59 | delete | 120 | 20 | |||
| 11 | 10 | 5000001 | delete | 60 | 10 |