refactor utils.py to use script A_preprocessing_data_from_multiple-fi…

…les.py (#15)
mxochicale · Sep 30, 2023 · 7c39c2b · 7c39c2b
1 parent d85422f
commit 7c39c2b
Show file tree

Hide file tree

Showing 4 changed files with 1,859 additions and 1,060 deletions.
diff --git a/rtt4ssa/data_analysis/A_analysis_of_data_from_multiple-files.ipynb b/rtt4ssa/data_analysis/A_analysis_of_data_from_multiple-files.ipynb
diff --git a/rtt4ssa/data_analysis/A_preprocessing_data_from_multiple-files.ipynb b/rtt4ssa/data_analysis/A_preprocessing_data_from_multiple-files.ipynb
diff --git a/rtt4ssa/data_analysis/A_preprocessing_data_from_multiple-files.py b/rtt4ssa/data_analysis/A_preprocessing_data_from_multiple-files.py
@@ -0,0 +1,86 @@
+from rtt4ssa.utils.utils import video_to_tensor, compute_texture_array_and_plot
+from rtt4ssa.utils.utils import data_frame_of_texture_analysis
+from rtt4ssa.utils.utils import get_and_plot_imu_data_analysis
+
+import os
+import pandas as pd
+import numpy as np
+import torch
+import matplotlib.pyplot as plt
+import cv2
+import skimage
+from typing import Tuple, List
+
+HOME_PATH = os.path.expanduser(f'~')
+USERNAME = os.path.split(HOME_PATH)[1]
+REPOSITORY_PATH='repositories/rtt4ssa'
+
+###########################
+###SETTING DATA_PATHS
+#DATA_PATH='repositories/datasets/in2research2023/Thu-27-Jul-2023' 
+
+RAW_DATA_PATH='repositories/datasets/in2research2023/Thu-24-Aug-2023'
+PREPROCESSED_DATA_PATH='repositories/datasets/in2research2023/Thu-24-Aug-2023-preprocessed'
+
+FULL_REPO_DATA_PATH = HOME_PATH +'/' + RAW_DATA_PATH
+FULL_REPO_PREPROCESSED_DATA_PATH = HOME_PATH +'/' + PREPROCESSED_DATA_PATH +'/'
+os.makedirs(FULL_REPO_PREPROCESSED_DATA_PATH, exist_ok=True) 
+
+## Printing Versions and paths
+#print(FULL_REPO_DATA_PATH)
+#print(f'PyTorch Version: {torch.__version__}')
+#print(f'pandas Version: {pd.__version__}')
+#print(f'numpy Version: {np.__version__}')
+#print(f'cv2 Version: {cv2.__version__}')
+#print(f'skimage Version: {skimage.__version__}')
+
+
+PARTICIPANTNN = 'participant01'
+PARTICIPANTNN_TESTNN = 'participant01-test01-rep01-1g-5mins' #51,328
+# PARTICIPANTNN_TESTNN = 'participant01-test01-rep02-1g-5mins' #51,178
+# PARTICIPANTNN_TESTNN = 'participant01-test02-rep01-1g-5mins' #49,183
+# PARTICIPANTNN_TESTNN = 'participant01-test02-rep02-1g-5mins' #47,577
+# PARTICIPANTNN_TESTNN = 'participant01-test03-rep01-1g-5mins' #48,688
+# PARTICIPANTNN_TESTNN = 'participant01-test03-rep02-1g-5mins'#48,789
+
+# PARTICIPANTNN = participant02
+# PARTICIPANTNN_TESTNN = 'participant02-test01-rep01-1g-5mins'#49,490
+# PARTICIPANTNN_TESTNN = 'participant02-test01-rep02-1g-5mins'#49,219
+# PARTICIPANTNN_TESTNN = 'participant02-test02-rep01-1g-5mins'#48,043
+# PARTICIPANTNN_TESTNN = 'participant02-test02-rep02-1g-5mins'#49,606
+# PARTICIPANTNN_TESTNN = 'participant02-test03-rep01-1g-5mins'#48,875
+# PARTICIPANTNN_TESTNN = 'participant02-test03-rep02-1g-5mins'#48,050
+
+start_frame_number = 0
+end_frame_number = 39000 #(resulted samples are end_frame_number-2)
+display_factor_for_texture_analysis_array = 100000
+
+
+CSV_FILENAME_FOR_TEXTURE_ANALYSIS=PARTICIPANTNN_TESTNN+'.csv'
+FULL_PATH_AND_AVI_FILE = os.path.join(FULL_REPO_DATA_PATH, PARTICIPANTNN, PARTICIPANTNN_TESTNN+'.avi')
+FULL_PATH_AND_CSV_FILE = os.path.join(FULL_REPO_DATA_PATH, PARTICIPANTNN, PARTICIPANTNN_TESTNN+'.avi.csv')
+
+total_number_of_frames = end_frame_number - start_frame_number
+
+
+video, frames_timestam = video_to_tensor(FULL_PATH_AND_AVI_FILE, start_frame_number, end_frame_number)
+
+num_frames, height, width = video.shape
+#print(f'num_frames: {num_frames}')
+#print(f'height: {height}')
+#print(f'width: {width}')
+
+display_figures=False
+#display_figures=True
+
+texture_analysis_array = compute_texture_array_and_plot(video, frames_timestam, display_figures, display_factor_for_texture_analysis_array)
+
+df_texture_analysis = data_frame_of_texture_analysis(texture_analysis_array, start_frame_number, end_frame_number)
+df, ndf, nqdf = get_and_plot_imu_data_analysis(FULL_PATH_AND_CSV_FILE, start_frame_number, end_frame_number, display_figures)
+
+df_a = df_texture_analysis[['frame_i', 'Contrast_normalised', 'Correlation_normalised', 'Dissimilarity_normalised', 'Energy_normalised', 'Homogeneity_normalised', 'ASM_normalised']]
+df_b = df[['q0', 'q1', 'q2', 'q3']]
+dff = pd.concat([df_a, df_b], axis=1)
+
+dff.to_csv(FULL_REPO_PREPROCESSED_DATA_PATH+PARTICIPANTNN_TESTNN+'_normalised_quaternions'+'.csv', index=False) 
+
diff --git a/rtt4ssa/utils/utils.py b/rtt4ssa/utils/utils.py
@@ -196,7 +196,7 @@ def display_videoframe_and_hist(im):
     plt.show()
 
 
-def compute_texture_array_and_plot(video, frames_timestam, display_interval):
+def compute_texture_array_and_plot(video, frames_timestam, displayfig, display_interval):
     """
     O'Byrne, Michael, Bidisha Ghosh, Vikram Pakrashi, and Franck Schoefs.
     "Texture analysis based detection and classification of surface features on ageing infrastructure elements."
@@ -231,11 +231,11 @@ def compute_texture_array_and_plot(video, frames_timestam, display_interval):
         texture_analysis_array.append([con, cor, dis, en, homo, asm])
         # print(con,cor,dis,en,homo, asm)
 
-        if frame_i % display_interval == 0:
-            print(f'frame_i: {frame_i}, timestamp {frames_timestam[frame_i]}')
-            display_videoframe_and_hist(image_frame)
-
-        # plt.savefig('filename'+str(frame_i)+'.png', dpi=300)
+        if displayfig==True:
+            if frame_i % display_interval == 0:
+                print(f'frame_i: {frame_i}, timestamp {frames_timestam[frame_i]}')
+                display_videoframe_and_hist(image_frame)
+                # plt.savefig('filename'+str(frame_i)+'.png', dpi=300)
 
     return texture_analysis_array
 
@@ -274,9 +274,9 @@ def data_frame_of_texture_analysis(texture_analysis_array, start_frame_number, e
     return df_texture_analysis
 
 
-def get_and_plot_imu_data_analysis(FULL_PATH_AND_CSV_FILE, start_frame_number, end_frame_number):
+def get_and_plot_imu_data_analysis(FULL_PATH_AND_CSV_FILE, start_frame_number, end_frame_number, displayfig):
     df = pd.read_csv(FULL_PATH_AND_CSV_FILE)
-    df = df[start_frame_number:end_frame_number]
+    df = df[start_frame_number:end_frame_number-1] # -1 to match length of data_frame_of_texture_analysis()
 
     # # df = df.rename(columns={'Euler_computed [Roll, Pitch, Yaw]': 'Euler_computed'})
     df = df.rename(columns={'Sample number': 'Sample_number'})
@@ -316,8 +316,7 @@ def get_and_plot_imu_data_analysis(FULL_PATH_AND_CSV_FILE, start_frame_number, e
     ndf = pd.concat([ndf_a, ndf_b, ndf_c], ignore_index=True)
     # print(ndf)
 
-    sns.lineplot(data=ndf, x='Sample_number', y='Euler_val', hue='Euler_angle', lw=2)
-    plt.show()
+
 
     ## QUATERNIONS
     nqdf_q0 = pd.DataFrame(data=df['Sample_number'])
@@ -339,7 +338,10 @@ def get_and_plot_imu_data_analysis(FULL_PATH_AND_CSV_FILE, start_frame_number, e
     nqdf = pd.concat([nqdf_q0, nqdf_q1, nqdf_q2, nqdf_q3], ignore_index=True)
     # print(nqdf)
 
-    sns.lineplot(data=nqdf, x='Sample_number', y='Q_val', hue='Quaternion', lw=2)
-    plt.show()
+    if displayfig==True:
+        sns.lineplot(data=ndf, x='Sample_number', y='Euler_val', hue='Euler_angle', lw=2)
+        plt.show()
+        sns.lineplot(data=nqdf, x='Sample_number', y='Q_val', hue='Quaternion', lw=2)
+        plt.show()
 
     return df, ndf, nqdf