MILESTONE: evaluation finished

Co-authored-by: Andrea Settimi <[email protected]>

eval/results_summary/2023-10-02_18-20-59/summary.csv

@@ -0,0 +1,2 @@
+pos_mean__stripe_low,pos_mean__stripe_high,pos_mean__ring_low,pos_mean__ring_high,pos_median__stripe_low,pos_median__stripe_high,pos_median__ring_low,pos_median__ring_high,pos_std__stripe_low,pos_std__stripe_high,pos_std__ring_low,pos_std__ring_high,pos_min__stripe_low,pos_min__stripe_high,pos_min__ring_low,pos_min__ring_high,pos_max__stripe_low,pos_max__stripe_high,pos_max__ring_low,pos_max__ring_high,rot_mean__stripe_low,rot_mean__stripe_high,rot_mean__ring_low,rot_mean__ring_high,rot_median__stripe_low,rot_median__stripe_high,rot_median__ring_low,rot_median__ring_high,rot_std__stripe_low,rot_std__stripe_high,rot_std__ring_low,rot_std__ring_high,rot_min__stripe_low,rot_min__stripe_high,rot_min__ring_low,rot_min__ring_high,rot_max__stripe_low,rot_max__stripe_high,rot_max__ring_low,rot_max__ring_high,tags_mean__stripe_low,tags_mean__stripe_high,tags_mean__ring_low,tags_mean__ring_high,tags_median__stripe_low,tags_median__stripe_high,tags_median__ring_low,tags_median__ring_high,tags_std__stripe_low,tags_std__stripe_high,tags_std__ring_low,tags_std__ring_high,tags_min__stripe_low,tags_min__stripe_high,tags_min__ring_low,tags_min__ring_high,tags_max__stripe_low,tags_max__stripe_high,tags_max__ring_low,tags_max__ring_high,time_tag_lowD_stripe,time_tag_lowD_ring,time_tag_highD_stripe,time_tag_highD_ring,time_mapping_lowD_stripe,time_mapping_lowD_ring,time_mapping_highD_stripe,time_mapping_highD_ring,time_traditional,time_layout,total_nbr_operations,
+0.00011872335898147254,-3.7475499177111866e-05,3.4510924137892736e-05,0.00033896062531618644,0.00010533922647239432,5.1977461406118934e-05,3.264479768889721e-05,2.324875106581787e-05,0.0007333755640075518,0.0006422418375242743,0.0005286182456948346,0.0009789179496128859,-0.0029605443056493507,-0.002624529539300176,-0.002827712675758486,-0.0008470611217176313,0.0023855409043621197,0.0011740890257785968,0.0012694811606316818,0.0054439148180361625,-0.00029724318379136406,-0.0013499527118857294,0.00013803191568475422,-0.0030169827280945773,-6.87149554160296e-05,3.644249478029851e-05,-5.323422483218789e-05,-3.902915495142937e-05,0.0026103434105462266,0.006659696344326049,0.003873719724396535,0.012417585678734418,-0.009649040902260984,-0.037743530100165505,-0.013042246610685211,-0.06771807797152234,0.005840753454015865,0.003430199696319231,0.021246102562876956,0.005595398845535191,7.174546820514824,7.207039974763153,13.82577171035175,7.9403154023247176,6.997963340122199,7.568198048083789,14.844939809335909,8.34147465437788,1.638125779569769,2.0612076195754843,5.261639228151985,2.205102485306456,3.3393627613674077,3.1415211970074814,3.9040129058277877,3.604317819854893,9.916401635620172,11.744347826086956,25.818805093046034,12.283993115318417,8.940000000000001,18.699999999999996,21.449999999999996,26.98,3.3,5.34,3.75,4.5200000000000005,35,22,1344.0,

eval/script/compute_overview.py

@@ -70,6 +70,18 @@ def main(out_subdir : str,
     data_highD_stripe = io_stream.cvt_csv_summary_to_data(csv_paths=csv_sequ_paths_highD_stripe)
     data_highD_ring = io_stream.cvt_csv_summary_to_data(csv_paths=csv_sequ_paths_highD_ring)
 
+    # total number of operations analysed
+    total_nbr_operations : int = 0
+    data1 = data_lowD_stripe[0]
+    data2 = data_highD_stripe[0]
+    data3 = data_lowD_ring[0]
+    data4 = data_highD_ring[0]
+    for i in range(data1.__len__()):
+        total_nbr_operations += data1[i]
+        total_nbr_operations += data2[i]
+        total_nbr_operations += data3[i]
+        total_nbr_operations += data4[i]
+
     # time parsing
     # # get the preparation times from videos and retain only the not pre-fabricated piece
     # # in-fact the comparison will not equal with the manual timing due to the fact that
@@ -145,7 +157,6 @@ def main(out_subdir : str,
                                             ytitle="Position drift (m)",
                                             xthick=0.001)
 
-
     # # rotation drift
     data_lowD_stripe_rot_np = np.array(data_lowD_stripe[7], dtype=float)
     data_highD_stripe_rot_np = np.array(data_highD_stripe[7], dtype=float)
@@ -180,21 +191,34 @@ def main(out_subdir : str,
     io_stream.save_graph(graph=graph_time, path=f"{out_subdir}/summary_time.png")
 
     # #===========================================
-    # #TODO:
-    # # print csv + latex table (at least to have the main values like mean, median, std, min, max)
-    # # - get also the total number of operations
-    # # metrics.compute_summary_table
+    # drop csv + latex table of the summary metrics
+
+    dict_res = metrics.compute_summary_table(pair_pos_stripe=pair_pos_stripe,
+                                    pair_pos_ring=pair_pos_ring,
+                                    pair_rot_stripe=pair_rot_stripe,
+                                    pair_rot_ring=pair_rot_ring,
+                                    pair_tags_stripe=pair_tags_stripe,
+                                    pair_tags_ring=pair_tags_ring,
+                                    mean_time_tag_lowD_stripe=mean_time_tag_lowD_stripe,
+                                    mean_time_tag_lowD_ring=mean_time_tag_lowD_ring,
+                                    mean_time_tag_highD_stripe=mean_time_tag_highD_stripe,
+                                    mean_time_tag_highD_ring=mean_time_tag_highD_ring,
+                                    mean_time_mapping_lowD_stripe=mean_time_mapping_lowD_stripe,
+                                    mean_time_mapping_lowD_ring=mean_time_mapping_lowD_ring,
+                                    mean_time_mapping_highD_stripe=mean_time_mapping_highD_stripe,
+                                    mean_time_mapping_highD_ring=mean_time_mapping_highD_ring,
+                                    total_nbr_operations=total_nbr_operations)
+
+    csv_path = os.path.join(out_subdir, "summary.csv")
+    with open(csv_path, 'w') as f:
+        for key, item in dict_res.items():
+            f.write(f"{key};")
+        f.write("\n")
+        for key, item in dict_res.items():
+            f.write(f"{item};")
+        f.write("\n")
 
-    total_nbr_operations : int = 0
-    # do a mass addition of the total number of operations
-    for data in [data_lowD_stripe, data_highD_stripe, data_lowD_ring, data_highD_ring]:
-        total_nbr_operations += int(data[0])
-    print(f"total_nbr_operations: {total_nbr_operations}")
-
-    # create a new csv file in the out_subdir
-    csv_summary_path : str = f"{out_subdir}/summary.csv"
-    with open(csv_summary_path, 'w') as f:
-        f.write("total_nbr_operations
+    # #===========================================
 
     return None
 

eval/script/metrics.py

@@ -482,97 +482,126 @@ def compute_average_summary_values(data : np.array(float)) -> np.array(float):
         avr_data.append(np.mean(param))
     return np.array(avr_data)
 
-
 def compute_summary_table(pair_pos_stripe : list[list[float]],
                           pair_pos_ring : list[list[float]],
                           pair_rot_stripe : list[list[float]],
                           pair_rot_ring : list[list[float]],
                           pair_tags_stripe : list[list[float]],
                           pair_tags_ring : list[list[float]],
-                          pair_prep_time : list[list[float]]=None ## TODO: add!
-                          ) -> list[float]:
+                          mean_time_tag_lowD_stripe : float,
+                          mean_time_tag_lowD_ring : float,
+                          mean_time_tag_highD_stripe : float,
+                          mean_time_tag_highD_ring : float,
+                          mean_time_mapping_lowD_stripe : float,
+                          mean_time_mapping_lowD_ring : float,
+                          mean_time_mapping_highD_stripe : float,
+                          mean_time_mapping_highD_ring : float,
+                          total_nbr_operations : int,
+                          pair_time_traditional : int=35,
+                          pair_time_layout : int=22,
+                          ) -> dict:
     """
         This function computes the values of the final evaluation table based
         on the averaged values (all tools values merged and averaged):
         - (A) position drift (mean, median, std, min, max)
         - (B) rotation drift (mean, median, std, min, max)
         - (C) nbr tags detected (mean, median, std, min, max)
-        - (D) preparation time
+        - (D) total time (means, traditional, layout)
+        - (E) total number operations
     """
-    #TODO: missing time calcuation
-    return [
-        np.mean(pair_pos_stripe[0]),                # pos_mean__stripe_low
-        np.mean(pair_pos_stripe[1]),                # pos_mean__stripe_high
-        np.mean(pair_pos_ring[0]),                  # pos_mean__ring_low
-        np.mean(pair_pos_ring[1]),                  # pos_mean__ring_high
-
-        np.median(pair_pos_stripe[0]),              # pos_median__stripe_low
-        np.median(pair_pos_stripe[1]),              # pos_median__stripe_high
-        np.median(pair_pos_ring[0]),                # pos_median__ring_low
-        np.median(pair_pos_ring[1]),                # pos_median__ring_high
-
-        np.std(pair_pos_stripe[0]),                 # pos_std__stripe_low
-        np.std(pair_pos_stripe[1]),                 # pos_std__stripe_high
-        np.std(pair_pos_ring[0]),                   # pos_std__ring_low
-        np.std(pair_pos_ring[1]),                   # pos_std__ring_high
-
-        np.min(pair_pos_stripe[0]),                 # pos_min__stripe_low
-        np.min(pair_pos_stripe[1]),                 # pos_min__stripe_high
-        np.min(pair_pos_ring[0]),                   # pos_min__ring_low
-        np.min(pair_pos_ring[1]),                   # pos_min__ring_high
-
-        np.max(pair_pos_stripe[0]),                 # pos_max__stripe_low
-        np.max(pair_pos_stripe[1]),                 # pos_max__stripe_high
-        np.max(pair_pos_ring[0]),                   # pos_max__ring_low
-        np.max(pair_pos_ring[1]),                   # pos_max__ring_high
-
-        np.mean(pair_rot_stripe[0]),                # rot_mean__stripe_low
-        np.mean(pair_rot_stripe[1]),                # rot_mean__stripe_high
-        np.mean(pair_rot_ring[0]),                  # rot_mean__ring_low
-        np.mean(pair_rot_ring[1]),                  # rot_mean__ring_high
-
-        np.median(pair_rot_stripe[0]),              # rot_median__stripe_low
-        np.median(pair_rot_stripe[1]),              # rot_median__stripe_high
-        np.median(pair_rot_ring[0]),                # rot_median__ring_low
-        np.median(pair_rot_ring[1]),                # rot_median__ring_high
-
-        np.std(pair_rot_stripe[0]),                 # rot_std__stripe_low
-        np.std(pair_rot_stripe[1]),                 # rot_std__stripe_high
-        np.std(pair_rot_ring[0]),                   # rot_std__ring_low
-        np.std(pair_rot_ring[1]),                   # rot_std__ring_high
-
-        np.min(pair_rot_stripe[0]),                 # rot_min__stripe_low
-        np.min(pair_rot_stripe[1]),                 # rot_min__stripe_high
-        np.min(pair_rot_ring[0]),                   # rot_min__ring_low
-        np.min(pair_rot_ring[1]),                   # rot_min__ring_high
-
-        np.max(pair_rot_stripe[0]),                 # rot_max__stripe_low
-        np.max(pair_rot_stripe[1]),                 # rot_max__stripe_high
-        np.max(pair_rot_ring[0]),                   # rot_max__ring_low
-        np.max(pair_rot_ring[1]),                   # rot_max__ring_high
-
-        np.mean(pair_tags_stripe[0]),               # tags_mean__stripe_low
-        np.mean(pair_tags_stripe[1]),               # tags_mean__stripe_high
-        np.mean(pair_tags_ring[0]),                 # tags_mean__ring_low
-        np.mean(pair_tags_ring[1]),                 # tags_mean__ring_high
-
-        np.median(pair_tags_stripe[0]),             # tags_median__stripe_low
-        np.median(pair_tags_stripe[1]),             # tags_median__stripe_high
-        np.median(pair_tags_ring[0]),               # tags_median__ring_low
-        np.median(pair_tags_ring[1]),               # tags_median__ring_high
-
-        np.std(pair_tags_stripe[0]),                # tags_std__stripe_low
-        np.std(pair_tags_stripe[1]),                # tags_std__stripe_high
-        np.std(pair_tags_ring[0]),                  # tags_std__ring_low
-        np.std(pair_tags_ring[1]),                  # tags_std__ring_high
-
-        np.min(pair_tags_stripe[0]),                # tags_min__stripe_low
-        np.min(pair_tags_stripe[1]),                # tags_min__stripe_high
-        np.min(pair_tags_ring[0]),                  # tags_min__ring_low
-        np.min(pair_tags_ring[1]),                  # tags_min__ring_high
-
-        np.max(pair_tags_stripe[0]),                # tags_max__stripe_low
-        np.max(pair_tags_stripe[1]),                # tags_max__stripe_high
-        np.max(pair_tags_ring[0]),                  # tags_max__ring_low
-        np.max(pair_tags_ring[1])                   # tags_max__ring_high
-    ]
+    dict_results = {
+        # >>>>>>>>>>>>>> A >>>>>>>>>>>>>>>
+        "pos_mean__stripe_low" : np.mean(pair_pos_stripe[0]),
+        "pos_mean__stripe_high" : np.mean(pair_pos_stripe[1]),
+        "pos_mean__ring_low" : np.mean(pair_pos_ring[0]),
+        "pos_mean__ring_high" : np.mean(pair_pos_ring[1]),
+
+        "pos_median__stripe_low" : np.median(pair_pos_stripe[0]),
+        "pos_median__stripe_high" : np.median(pair_pos_stripe[1]),
+        "pos_median__ring_low" : np.median(pair_pos_ring[0]),
+        "pos_median__ring_high" : np.median(pair_pos_ring[1]),
+
+        "pos_std__stripe_low" : np.std(pair_pos_stripe[0]),
+        "pos_std__stripe_high" : np.std(pair_pos_stripe[1]),
+        "pos_std__ring_low" : np.std(pair_pos_ring[0]),
+        "pos_std__ring_high" : np.std(pair_pos_ring[1]),
+
+        "pos_min__stripe_low" : np.min(pair_pos_stripe[0]),
+        "pos_min__stripe_high" : np.min(pair_pos_stripe[1]),
+        "pos_min__ring_low" : np.min(pair_pos_ring[0]),
+        "pos_min__ring_high" : np.min(pair_pos_ring[1]),
+
+        "pos_max__stripe_low" : np.max(pair_pos_stripe[0]),
+        "pos_max__stripe_high" : np.max(pair_pos_stripe[1]),
+        "pos_max__ring_low" : np.max(pair_pos_ring[0]),
+        "pos_max__ring_high" : np.max(pair_pos_ring[1]),
+
+        # >>>>>>>>>>>>>> B >>>>>>>>>>>>>>>
+        "rot_mean__stripe_low" : np.mean(pair_rot_stripe[0]),
+        "rot_mean__stripe_high" : np.mean(pair_rot_stripe[1]),
+        "rot_mean__ring_low" : np.mean(pair_rot_ring[0]),
+        "rot_mean__ring_high" : np.mean(pair_rot_ring[1]),
+
+        "rot_median__stripe_low" : np.median(pair_rot_stripe[0]),
+        "rot_median__stripe_high" : np.median(pair_rot_stripe[1]),
+        "rot_median__ring_low" : np.median(pair_rot_ring[0]),
+        "rot_median__ring_high" : np.median(pair_rot_ring[1]),
+
+        "rot_std__stripe_low" : np.std(pair_rot_stripe[0]),
+        "rot_std__stripe_high" : np.std(pair_rot_stripe[1]),
+        "rot_std__ring_low" : np.std(pair_rot_ring[0]),
+        "rot_std__ring_high" : np.std(pair_rot_ring[1]),
+
+        "rot_min__stripe_low" : np.min(pair_rot_stripe[0]),
+        "rot_min__stripe_high" : np.min(pair_rot_stripe[1]),
+        "rot_min__ring_low" : np.min(pair_rot_ring[0]),
+        "rot_min__ring_high" : np.min(pair_rot_ring[1]),
+
+        "rot_max__stripe_low" : np.max(pair_rot_stripe[0]),
+        "rot_max__stripe_high" : np.max(pair_rot_stripe[1]),
+        "rot_max__ring_low" : np.max(pair_rot_ring[0]),
+        "rot_max__ring_high" : np.max(pair_rot_ring[1]),
+
+        # >>>>>>>>>>>>>> C >>>>>>>>>>>>>>>
+        "tags_mean__stripe_low" : np.mean(pair_tags_stripe[0]),
+        "tags_mean__stripe_high" : np.mean(pair_tags_stripe[1]),
+        "tags_mean__ring_low" : np.mean(pair_tags_ring[0]),
+        "tags_mean__ring_high" : np.mean(pair_tags_ring[1]),
+
+        "tags_median__stripe_low" : np.median(pair_tags_stripe[0]),
+        "tags_median__stripe_high" : np.median(pair_tags_stripe[1]),
+        "tags_median__ring_low" : np.median(pair_tags_ring[0]),
+        "tags_median__ring_high" : np.median(pair_tags_ring[1]),
+
+        "tags_std__stripe_low" : np.std(pair_tags_stripe[0]),
+        "tags_std__stripe_high" : np.std(pair_tags_stripe[1]),
+        "tags_std__ring_low" : np.std(pair_tags_ring[0]),
+        "tags_std__ring_high" : np.std(pair_tags_ring[1]),
+
+        "tags_min__stripe_low" : np.min(pair_tags_stripe[0]),
+        "tags_min__stripe_high" : np.min(pair_tags_stripe[1]),
+        "tags_min__ring_low" : np.min(pair_tags_ring[0]),
+        "tags_min__ring_high" : np.min(pair_tags_ring[1]),
+
+        "tags_max__stripe_low" : np.max(pair_tags_stripe[0]),
+        "tags_max__stripe_high" : np.max(pair_tags_stripe[1]),
+        "tags_max__ring_low" : np.max(pair_tags_ring[0]),
+        "tags_max__ring_high" : np.max(pair_tags_ring[1]),
+
+        # >>>>>>>>>>>>>> D >>>>>>>>>>>>>>>
+        "time_tag_lowD_stripe" : mean_time_tag_lowD_stripe,
+        "time_tag_lowD_ring" : mean_time_tag_lowD_ring,
+        "time_tag_highD_stripe" : mean_time_tag_highD_stripe,
+        "time_tag_highD_ring" : mean_time_tag_highD_ring,
+        "time_mapping_lowD_stripe" : mean_time_mapping_lowD_stripe,
+        "time_mapping_lowD_ring" : mean_time_mapping_lowD_ring,
+        "time_mapping_highD_stripe" : mean_time_mapping_highD_stripe,
+        "time_mapping_highD_ring" : mean_time_mapping_highD_ring,
+        "time_traditional" : pair_time_traditional,
+        "time_layout" : pair_time_layout,
+
+        # >>>>>>>>>>>>>> E >>>>>>>>>>>>>>>
+        "total_nbr_operations" : total_nbr_operations
+    }
+
+    return dict_results