Merge pull request #86 from boostcampaitech3/feat/85

[Feat] Change code for making timelines

model/final_shorts/final_timeline.py

@@ -2,7 +2,7 @@ def make_final_timeline(laughter_timeline,person_timeline,max_length=None):
     p_timeline = person_timeline.copy()
     final = []
     total_length = 0
-    for ind,(start,end,interest) in enumerate(laughter_timeline):
+    for ind,(start,end,laugh_len,laugh_db) in enumerate(laughter_timeline):
         shot_length = end-start
         length=0
         while p_timeline:
@@ -18,33 +18,33 @@ def make_final_timeline(laughter_timeline,person_timeline,max_length=None):
         if shot_length < 35:
             if length/shot_length > 0.35:
                 person_interest = (length/shot_length - 0.3)/0.4
-                total_interest = (interest + person_interest*0.66)/1.5
-                final.append((start,end,round(total_interest,2),round(length/shot_length,3)))
+                total_interest = laugh_len*3 + laugh_db*2 + person_interest*1
+                final.append((round(start-5,2),end,round(total_interest,2),round(length/shot_length,3)))
                 total_length += shot_length
         else:
             if length/shot_length > 0.30:
                 person_interest = (length/shot_length - 0.3)/0.4
-                total_interest = (interest + person_interest*0.66)/1.5
-                final.append((start,end,round(total_interest,2),round(length/shot_length,3)))
+                total_interest = laugh_len*3 + laugh_db*2 + person_interest*1
+                final.append((round(start-5,2),end,round(total_interest,2),round(length/shot_length,3)))
                 total_length += shot_length  
 
-    # max_length 넘어가는 경우 등장비율 높은 순서로 max_length 이내로 선택
+    # max_length 넘어가는 경우 흥미도 높은 순서로 max_length 이내로 선택
     if max_length and total_length > max_length:
-        sorted_timeline = sorted(final,key=lambda x:-x[3])
+        sorted_timeline = sorted(final,key=lambda x:-x[2])
         choose_index = [False for _ in range(len(sorted_timeline))]
         total_length = 0
-        for start,end,ratio,interest in sorted_timeline:
+        for start,end,interest,ratio in sorted_timeline:
             length = end - start
             if total_length + length > max_length:
                 break
             else:
                 total_length += length
-                choose_index[final.index((start,end,ratio,interest))] = True
+                choose_index[final.index((start,end,interest,ratio))] = True
 
         new_final = []
-        for ind, (s,e,r,i) in enumerate(final):
+        for ind, (s,e,i,r) in enumerate(final):
             if choose_index[ind]:
-                new_final.append((s,e,r,i))
+                new_final.append((s,e,i,r))
         final = new_final
 
     # interest 순서로 정렬

model/laughter/laughter_detector/laughter_detector.py

@@ -114,6 +114,6 @@ def calculate_interest(self,input_timeline):
                 if f2 > feat_2_max:
                     feat_2_max = f2
             for start, end, f1, f2 in input_timeline:
-                infos = (start, end, round(0.66*(f1-feat_1_min)/(feat_1_max-feat_1_min+1e-6) + 0.33*(f2-feat_2_min)/(feat_2_max-feat_2_min+1e-6),2))
+                infos = (start, end, round(0.66*(f1-feat_1_min)/(feat_1_max-feat_1_min+1e-6),2),round(0.33*(f2-feat_2_min)/(feat_2_max-feat_2_min+1e-6),2))
                 output_timeline.append(infos)
             return output_timeline

serving/backend/app/ml/final_shorts/final_timeline.py

@@ -2,11 +2,11 @@ def make_final_timeline(laughter_timeline,person_timeline,max_length=None):
     p_timeline = person_timeline.copy()
     final = []
     total_length = 0
-    for ind,[start,end,interest] in enumerate(laughter_timeline):
+    for ind,(start,end,laugh_len,laugh_db) in enumerate(laughter_timeline):
         shot_length = end-start
         length=0
         while p_timeline:
-            s, e = p_timeline.pop(0)
+            s,e = p_timeline.pop(0)
             if e<start:
                 continue
             if s>end:
@@ -17,30 +17,34 @@ def make_final_timeline(laughter_timeline,person_timeline,max_length=None):
             length += (e-s)
         if shot_length < 35:
             if length/shot_length > 0.35:
-                final.append((start,end,interest,shot_length))
+                person_interest = (length/shot_length - 0.3)/0.4
+                total_interest = laugh_len*3 + laugh_db*2 + person_interest*1
+                final.append((round(start-5,2),end,round(total_interest,2),round(length/shot_length,3)))
                 total_length += shot_length
         else:
             if length/shot_length > 0.30:
-                final.append((start,end,interest,shot_length))
+                person_interest = (length/shot_length - 0.3)/0.4
+                total_interest = laugh_len*3 + laugh_db*2 + person_interest*1
+                final.append((round(start-5,2),end,round(total_interest,2),round(length/shot_length,3)))
                 total_length += shot_length  
 
-    # max_length 넘어가는 경우 등장비율 높은 순서로 max_length 이내로 선택
+    # max_length 넘어가는 경우 흥미도 높은 순서로 max_length 이내로 선택
     if max_length and total_length > max_length:
-        sorted_timeline = sorted(final,key=lambda x:-x[3])
+        sorted_timeline = sorted(final,key=lambda x:-x[2])
         choose_index = [False for _ in range(len(sorted_timeline))]
         total_length = 0
-        for start,end,ratio,interest in sorted_timeline:
+        for start,end,interest,ratio in sorted_timeline:
             length = end - start
             if total_length + length > max_length:
                 break
             else:
                 total_length += length
-                choose_index[final.index((start,end,ratio,interest))] = True
+                choose_index[final.index((start,end,interest,ratio))] = True
 
         new_final = []
-        for ind, (s,e,r,i) in enumerate(final):
+        for ind, (s,e,i,r) in enumerate(final):
             if choose_index[ind]:
-                new_final.append((s,e,r,i))
+                new_final.append((s,e,i,r))
         final = new_final
 
     # interest 순서로 정렬

serving/backend/app_laughter/ml/laughter_detector/laughter_detector.py

@@ -73,21 +73,21 @@ def make_laugh_timeline(self,input_timeline,whole_length):
             if end<s-12:
                 if end-start>20:
                     mean_amp /= mean_amp_count
-                    laugh_length /= (end-start+8.5)
-                    output_timeline.append((round(start-10,2),round(end+1.5,2),round(laugh_length,2),round(mean_amp,4)))
+                    laugh_length /= (end-start+10.5)
+                    output_timeline.append((round(start-10,2),round(end+0.5,2),round(laugh_length,2),round(mean_amp,4)))
                     laugh_length = 0
                     mean_amp = 0
                     mean_amp_count = 0
                 start,end = s,e
-                laugh_length += (e-s)
-                mean_amp += audio_utils.get_mean_amplitude(audio,sr,s,e)
-                mean_amp_count += 1
+                laugh_length = (e-s)
+                mean_amp = audio_utils.get_mean_amplitude(audio,sr,s,e)
+                mean_amp_count = 1
             else:
                 laugh_length += (e-s)
                 mean_amp += audio_utils.get_mean_amplitude(audio,sr,s,e)
                 mean_amp_count += 1
                 end = e
-        output_timeline.append((round(start-10,2),round(end+1.5,2),round(laugh_length/(end-start+8.5),2),round(mean_amp/mean_amp_count,4)))
+        output_timeline.append((round(start-10,2),round(end+0.5,2),round(laugh_length/(end-start+10.5),2),round(mean_amp/mean_amp_count,4)))
         if output_timeline[0][0]<0:
             output_timeline[0]=(0,output_timeline[0][1],output_timeline[0][2],output_timeline[0][3]) # 시작값 0보다 작은경우
         if output_timeline[-1][1]>whole_length:
@@ -114,6 +114,6 @@ def calculate_interest(self,input_timeline):
                 if f2 > feat_2_max:
                     feat_2_max = f2
             for start, end, f1, f2 in input_timeline:
-                infos = (start, end, round(0.66*(f1-feat_1_min)/(feat_1_max-feat_1_min+1e-6) + 0.33*(f2-feat_2_min)/(feat_2_max-feat_2_min+1e-6),2))
+                infos = (start, end, round(0.66*(f1-feat_1_min)/(feat_1_max-feat_1_min+1e-6),2),round(0.33*(f2-feat_2_min)/(feat_2_max-feat_2_min+1e-6),2))
                 output_timeline.append(infos)
             return output_timeline