diff --git a/dourflow.py b/dourflow.py index 234e335..d32a320 100644 --- a/dourflow.py +++ b/dourflow.py @@ -4,7 +4,7 @@ from yolov2 import YoloV2, YoloInferenceModel import os -from net.neteval import gen_anchors +#from net.neteval import gen_anchors # Add CPU option diff --git a/net/netarch.py b/net/netarch.py index ef950b5..f260456 100644 --- a/net/netarch.py +++ b/net/netarch.py @@ -6,7 +6,7 @@ import tensorflow as tf import numpy as np -import pickle, argparse, json, os +import pickle, argparse, json, os, cv2 from keras.utils.vis_utils import plot_model @@ -14,6 +14,48 @@ from net.netdecode import YoloOutProcess + +class YoloInferenceModel(object): + + def __init__(self, model): + self._yolo_out = YoloOutProcess() + self._inf_model = self._extend_processing(model) + + def _extend_processing(self, model): + output = Lambda(self._yolo_out, name='lambda_2')(model.output) + return Model(model.input, output) + + + def _prepro_single_image(self, image): + image = cv2.resize(image, + (YoloParams.INPUT_SIZE, YoloParams.INPUT_SIZE)) + # yolo normalize + image = image / 255. + image = image[:,:,::-1] + # cv2 has the channel as bgr, revert to to rgb for Yolo Pass + image = np.expand_dims(image, 0) + + return image + + def predict(self, image): + + image = self._prepro_single_image(image) + output = self._inf_model.predict(image)[0] + + if output.size == 0: + return [np.array([])]*4 + + boxes = output[:,:4] + scores = output[:,4] + label_idxs = output[:,5].astype(int) + + labels = [YoloParams.CLASS_LABELS[l] for l in label_idxs] + + return boxes, scores, label_idxs, labels + + + + class YoloArchitecture(object): def __init__(self): @@ -21,9 +63,9 @@ def __init__(self): self.in_model_name = YoloParams.IN_MODEL self.plot_name = YoloParams.ARCH_FNAME - def get_model(self, loss_func): + def get_model(self): - yolo_model = self._load_yolo_model(loss_func) + yolo_model = self._load_yolo_model() if YoloParams.YOLO_MODE == 'train': new_yolo_model = self._setup_transfer_learning(yolo_model) @@ -43,11 +85,11 @@ def get_model(self, loss_func): return new_yolo_model - def _load_yolo_model(self, loss_func): - # Error if not compiled with yolo_loss? + def _load_yolo_model(self): if os.path.isfile(self.in_model_name): - model = load_model(self.in_model_name, - custom_objects={'yolo_loss': loss_func}) + + model = load_model(self.in_model_name, compile=False) + return model else: raise ValueError('Need to load full model in order to do ' diff --git a/net/netdecode.py b/net/netdecode.py index 43ecbbe..bac0b64 100644 --- a/net/netdecode.py +++ b/net/netdecode.py @@ -21,6 +21,8 @@ def process_outs(b, s, c): return K.expand_dims(output_stack, axis=0) + + class YoloOutProcess(object): diff --git a/net/neteval.py b/net/neteval.py index ad0915e..5c7b484 100644 --- a/net/neteval.py +++ b/net/neteval.py @@ -1,18 +1,16 @@ - - from net.netparams import YoloParams -from net.netdecode import YoloOutProcess +import tensorflow as tf from tqdm import tqdm import matplotlib.pyplot as plt import numpy as np import cv2, os import keras from net.utils import draw_boxes, compute_iou, mkdir_p, \ -yolo_normalize, mkdir_p, handle_empty_indexing, parse_annotation - +mkdir_p, handle_empty_indexing, parse_annotation +from keras import backend as K ''' @@ -174,7 +172,7 @@ def _data_to_yolo_output(self, batch_images): y_batch[j, grid_y, grid_x, anchor_winner, 4+1+obj_indx] = 1 # number of labels per instance !> than true_box_buffer, add check in processing (?) - x_batch[j] = yolo_normalize(img) + x_batch[j] = img / 255. ############################################################ # x_batch -> list of input images @@ -313,10 +311,7 @@ def compute_ap(self, detections, num_gts): return self._interp_ap(np.array(precision), np.array(recall)) - - - - def __call__(self): + def comp_map(self): detection_results = [] detection_labels = np.array([0]*YoloParams.NUM_CLASSES) @@ -352,4 +347,120 @@ def __call__(self): +class Callback_MAP(keras.callbacks.Callback): + + def __init__(self, generator, model, tensorboard): + + self.yolo_eval = YoloEvaluate(generator=generator, model=model) + self.tensorboard = tensorboard + + def on_epoch_end(self, epoch, logs={}): + + mAP_dict = self.yolo_eval.comp_map() + + summary = tf.Summary() + summary_value = summary.value.add() + summary_value.simple_value = np.mean(list(mAP_dict.values())) + summary_value.tag = "mAP" + #self.tensorboard.writer.add_summary(summary, epoch) + + self.tensorboard.val_writer.add_summary(summary, epoch) + + + self.tensorboard.val_writer.flush() + + + + +def yolo_recall(y_true, y_pred): + + truth = y_true[...,4] + + pred_scores = K.expand_dims(K.sigmoid(y_pred[..., 4]), axis=-1) * K.softmax(y_pred[...,5:]) + preds = K.cast(K.max(pred_scores, axis=-1) > YoloParams.DETECTION_THRESHOLD, np.float32) + + tp = K.sum(truth * preds) + tpfn = K.sum(truth) + + return tp / (tpfn + 1e-8) + + + +# https://stackoverflow.com/questions/47877475/keras-tensorboard-plot-train-and-validation-scalars-in-a-same-figure?rq=1 + +def in_loss_decmop(k): + return any([term in k for term in ['coord','obj','class']]) + +class YoloTensorBoard(keras.callbacks.TensorBoard): + def __init__(self, log_dir='./logs', **kwargs): + # Make the original `TensorBoard` log to a subdirectory 'training' + training_log_dir = os.path.join(log_dir, 'training') + super(YoloTensorBoard, self).__init__(training_log_dir, **kwargs) + + # Log the validation metrics to a separate subdirectory + self.val_log_dir = os.path.join(log_dir, 'validation') + + + self.loss_dir = { + 'training':{ + 'coord':os.path.join(training_log_dir, 'coordinate'), + 'obj':os.path.join(training_log_dir, 'confidence'), + 'class':os.path.join(training_log_dir, 'class') + }, + 'validation':{ + 'coord':os.path.join(self.val_log_dir, 'coordinate'), + 'obj':os.path.join(self.val_log_dir, 'confidence'), + 'class':os.path.join(self.val_log_dir, 'class') + } + } + + def set_model(self, model): + # Setup writer for validation metrics + self.val_writer = tf.summary.FileWriter(self.val_log_dir) + + self.loss_writer = {} + for k,v in self.loss_dir.items(): + self.loss_writer[k] = {} + for l,m in v.items(): + self.loss_writer[k][l] = tf.summary.FileWriter(m) + + super(YoloTensorBoard, self).set_model(model) + + def on_epoch_end(self, epoch, logs=None): + logs = logs or {} + + loss_logs = {k:v for k, v in logs.items() if in_loss_decmop(k)} + logs = {k:v for k, v in logs.items() if not in_loss_decmop(k)} + + for name, value in loss_logs.items(): + summary = tf.Summary() + summary_value = summary.value.add() + summary_value.simple_value = value.item() + + decomp_part = name.replace('val_', '').replace('l_', '') + + key = ('val', 'validation') if name.startswith('val_') else ('train', 'training') + + summary_value.tag = key[0] + '_loss_decomp' + self.loss_writer[key[1]][decomp_part].add_summary(summary, epoch) + self.loss_writer[key[1]][decomp_part].flush() + + + val_logs = {k.replace('val_', ''): v for k, v in logs.items() if k.startswith('val_')} + for name, value in val_logs.items(): + summary = tf.Summary() + summary_value = summary.value.add() + summary_value.simple_value = value.item() + summary_value.tag = name + self.val_writer.add_summary(summary, epoch) + self.val_writer.flush() + + logs = {k: v for k, v in logs.items() if not k.startswith('val_')} + super(YoloTensorBoard, self).on_epoch_end(epoch, logs) + + def on_train_end(self, logs=None): + super(YoloTensorBoard, self).on_train_end(logs) + self.val_writer.close() + + diff --git a/net/netloss.py b/net/netloss.py index 448a90a..e7646e3 100644 --- a/net/netloss.py +++ b/net/netloss.py @@ -112,8 +112,7 @@ def class_loss(self, y_true, y_pred): #b_class_pred = y_pred[..., 5:] #loss_class_arg = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=b_class, logits=b_class_pred) - indicator_class = y_true[..., 4] * K.gather( - YoloParams.CLASS_WEIGHTS, b_class) * self.lambda_class + indicator_class = y_true[..., 4] * self.lambda_class norm_class = 1 if self.norm: @@ -133,6 +132,16 @@ def _transform_netout(self, y_pred_raw): return K.concatenate([y_pred_xy, y_pred_wh, y_pred_conf, y_pred_class], axis=-1) + def l_coord(self, y_true, y_pred_raw): + return self.coord_loss(y_true, self._transform_netout(y_pred_raw)) + + def l_obj(self, y_true, y_pred_raw): + return self.obj_loss(y_true, self._transform_netout(y_pred_raw)) + + def l_class(self, y_true, y_pred_raw): + return self.class_loss(y_true, self._transform_netout(y_pred_raw)) + + def __call__(self, y_true, y_pred_raw): @@ -144,11 +153,6 @@ def __call__(self, y_true, y_pred_raw): loss = total_coord_loss + total_obj_loss + total_class_loss - #loss = tf.Print(loss, [total_coord_loss], message='\nCoord Loss \t', summarize=1000) - #loss = tf.Print(loss, [total_obj_loss], message='Conf Loss \t', summarize=1000) - #oss = tf.Print(loss, [total_class_loss], message='Class Loss \t', summarize=1000) - #oss = loss = tf.Print(loss, [loss], message='Total Loss \t', summarize=1000) - return loss diff --git a/net/netparams.py b/net/netparams.py index 919c774..5fed3e2 100644 --- a/net/netparams.py +++ b/net/netparams.py @@ -118,7 +118,6 @@ class YoloParams(object): CLASS_LABELS = [x.rstrip() for x in open(config['config_path']['labels'])] NUM_CLASSES = len(CLASS_LABELS) CLASS_TO_INDEX = dict(zip(CLASS_LABELS, np.arange(NUM_CLASSES))) - CLASS_WEIGHTS = np.ones(NUM_CLASSES, dtype='float32') # Infrastructure params INPUT_SIZE = config['model']['input_size'] diff --git a/net/utils.py b/net/utils.py index 0b1fd16..5562d09 100755 --- a/net/utils.py +++ b/net/utils.py @@ -7,8 +7,6 @@ import tensorflow as tf import copy import cv2 -from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard - @@ -209,46 +207,6 @@ def parse_annotation(ann_dir, img_dir, labels=[]): -def yolo_normalize(image): - return image / 255. - - - -# https://stackoverflow.com/questions/47877475/keras-tensorboard-plot-train-and-validation-scalars-in-a-same-figure?rq=1 - -class TrainValTensorBoard(TensorBoard): - def __init__(self, log_dir='./logs', **kwargs): - # Make the original `TensorBoard` log to a subdirectory 'training' - training_log_dir = os.path.join(log_dir, 'training') - super(TrainValTensorBoard, self).__init__(training_log_dir, **kwargs) - - # Log the validation metrics to a separate subdirectory - self.val_log_dir = os.path.join(log_dir, 'validation') - - def set_model(self, model): - # Setup writer for validation metrics - self.val_writer = tf.summary.FileWriter(self.val_log_dir) - super(TrainValTensorBoard, self).set_model(model) - - def on_epoch_end(self, epoch, logs=None): - logs = logs or {} - val_logs = {k.replace('val_', ''): v for k, v in logs.items() if k.startswith('val_')} - for name, value in val_logs.items(): - summary = tf.Summary() - summary_value = summary.value.add() - summary_value.simple_value = value.item() - summary_value.tag = name - self.val_writer.add_summary(summary, epoch) - self.val_writer.flush() - - logs = {k: v for k, v in logs.items() if not k.startswith('val_')} - super(TrainValTensorBoard, self).on_epoch_end(epoch, logs) - - def on_train_end(self, logs=None): - super(TrainValTensorBoard, self).on_train_end(logs) - self.val_writer.close() - - def setup_logging(logging_path='logs'): diff --git a/result_plots/tbexam.png b/result_plots/tbexam.png index 3f75125..119dc6c 100644 Binary files a/result_plots/tbexam.png and b/result_plots/tbexam.png differ diff --git a/yolov2.py b/yolov2.py index 5afa2ca..7fb2dd9 100644 --- a/yolov2.py +++ b/yolov2.py @@ -11,53 +11,17 @@ from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard from keras.optimizers import SGD, Adam, RMSprop -from net.utils import parse_annotation, yolo_normalize, mkdir_p, \ -setup_logging, TrainValTensorBoard, draw_boxes +from net.utils import parse_annotation, mkdir_p, \ +setup_logging, draw_boxes from net.netparams import YoloParams from net.netloss import YoloLoss -from net.neteval import YoloDataGenerator, YoloEvaluate -from net.netdecode import YoloOutProcess -from net.netarch import YoloArchitecture +from net.neteval import YoloDataGenerator, YoloEvaluate, \ +YoloTensorBoard, Callback_MAP, yolo_recall +from net.netarch import YoloArchitecture, YoloInferenceModel -class YoloInferenceModel(object): - def __init__(self, model): - self._yolo_out = YoloOutProcess() - self._inf_model = self._extend_processing(model) - - def _extend_processing(self, model): - output = Lambda(self._yolo_out, name='lambda_2')(model.output) - return Model(model.input, output) - - - def _prepro_single_image(self, image): - image = cv2.resize(image, - (YoloParams.INPUT_SIZE, YoloParams.INPUT_SIZE)) - # yolo normalize - image = yolo_normalize(image) - image = image[:,:,::-1] - # cv2 has the channel as bgr, revert to to rgb for Yolo Pass - image = np.expand_dims(image, 0) - - return image - - def predict(self, image): - - image = self._prepro_single_image(image) - output = self._inf_model.predict(image)[0] - - if output.size == 0: - return [np.array([])]*4 - - boxes = output[:,:4] - scores = output[:,4] - label_idxs = output[:,5].astype(int) - - labels = [YoloParams.CLASS_LABELS[l] for l in label_idxs] - - return boxes, scores, label_idxs, labels @@ -75,25 +39,24 @@ def __init__(self): def run(self, **kwargs): - self.model = self.yolo_arch.get_model(self.yolo_loss) + self.model = self.yolo_arch.get_model() + self.inf_model = YoloInferenceModel(self.model) + if YoloParams.YOLO_MODE == 'train': self.training() - else: - self.inf_model = YoloInferenceModel(self.model) - - if YoloParams.YOLO_MODE == 'inference': - self.inference(YoloParams.PREDICT_IMAGE) + elif YoloParams.YOLO_MODE == 'inference': + self.inference(YoloParams.PREDICT_IMAGE) - elif YoloParams.YOLO_MODE == 'validate': - self.validation() + elif YoloParams.YOLO_MODE == 'validate': + self.validation() - elif YoloParams.YOLO_MODE == 'video': - self.video_inference(YoloParams.PREDICT_IMAGE) + elif YoloParams.YOLO_MODE == 'video': + self.video_inference(YoloParams.PREDICT_IMAGE) - elif YoloParams.YOLO_MODE == 'cam': - self.cam_inference(YoloParams.WEBCAM_OUT) + elif YoloParams.YOLO_MODE == 'cam': + self.cam_inference(YoloParams.WEBCAM_OUT) # Sometimes bug: https://github.com/tensorflow/tensorflow/issues/3388 @@ -200,7 +163,7 @@ def validation(self): generator = YoloDataGenerator(valid_data, shuffle=True) yolo_eval = YoloEvaluate(generator=generator, model=self.inf_model) - AP = yolo_eval() + AP = yolo_eval.comp_map() mAP_values = [] for class_label, ap in AP.items(): @@ -248,7 +211,7 @@ def training(self): period=1) #tb_path = os.path.join(log_path, ) - tensorboard = TrainValTensorBoard( + tensorboard = YoloTensorBoard( log_dir=log_path, histogram_freq=0, write_graph=True, @@ -262,8 +225,26 @@ def training(self): decay=0.0) + + map_cbck = Callback_MAP(generator=valid_gen, + model=self.inf_model, + tensorboard=tensorboard) + + # add metrics.. - self.model.compile(loss=self.yolo_loss, optimizer=optimizer) #, metrics=['accuracy']) + yolo_recall.__name__ = 'recall' + + metrics = [ + self.yolo_loss.l_coord, + self.yolo_loss.l_obj, + self.yolo_loss.l_class, + yolo_recall + ] + + + self.model.compile(loss=self.yolo_loss, + optimizer=optimizer, + metrics=metrics) self.model.fit_generator( generator=train_gen, @@ -271,7 +252,7 @@ def training(self): verbose=YoloParams.TRAIN_VERBOSE, validation_data=valid_gen, validation_steps=len(valid_gen), - callbacks=[early_stop, checkpoint, tensorboard], + callbacks=[early_stop, checkpoint, tensorboard, map_cbck], epochs=YoloParams.NUM_EPOCHS, max_queue_size=20)