From 604cbbf86284bfa3a536604f9000056cb05f8f7e Mon Sep 17 00:00:00 2001 From: quangdungluong Date: Wed, 11 Sep 2024 23:03:59 +0700 Subject: [PATCH 1/6] feat: Add model yolo-nas --- .../yolo-nas/requirements.txt | 4 + .../yolo-nas/results_yolo_nas_l.json | 37 ++ .../yolo-nas/results_yolo_nas_m.json | 37 ++ .../yolo-nas/results_yolo_nas_s.json | 37 ++ models/object_detection/yolo-nas/run.py | 97 +++ static/aggregate_results.js | 591 +++++++++++------- 6 files changed, 563 insertions(+), 240 deletions(-) create mode 100644 models/object_detection/yolo-nas/requirements.txt create mode 100644 models/object_detection/yolo-nas/results_yolo_nas_l.json create mode 100644 models/object_detection/yolo-nas/results_yolo_nas_m.json create mode 100644 models/object_detection/yolo-nas/results_yolo_nas_s.json create mode 100644 models/object_detection/yolo-nas/run.py diff --git a/models/object_detection/yolo-nas/requirements.txt b/models/object_detection/yolo-nas/requirements.txt new file mode 100644 index 0000000..51e8e3a --- /dev/null +++ b/models/object_detection/yolo-nas/requirements.txt @@ -0,0 +1,4 @@ +super_gradients==3.7.1 +supervision==0.23.0 +torch==2.2.2 +tqdm==4.66.4 diff --git a/models/object_detection/yolo-nas/results_yolo_nas_l.json b/models/object_detection/yolo-nas/results_yolo_nas_l.json new file mode 100644 index 0000000..725376d --- /dev/null +++ b/models/object_detection/yolo-nas/results_yolo_nas_l.json @@ -0,0 +1,37 @@ +{ + "metadata": { + "model": "yolo_nas_l", + "param_count": 66976392, + "run_date": "2024-09-11T15:34:44.524051+00:00" + }, + "map50_95": 0.4823908978773399, + "map50": 0.6436635942062905, + "map75": 0.5227631511898091, + "small_objects": { + "map50_95": 0.20627144126452443, + "map50": 0.3234195270645268, + "map75": 0.22433010779366858 + }, + "medium_objects": { + "map50_95": 0.4504440067740434, + "map50": 0.6198672027100715, + "map75": 0.5028411865713277 + }, + "large_objects": { + "map50_95": 0.6232514892633709, + "map50": 0.7530164274037385, + "map75": 0.6684965691185336 + }, + "iou_thresholds": [ + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.8999999999999999, + 0.95 + ] +} diff --git a/models/object_detection/yolo-nas/results_yolo_nas_m.json b/models/object_detection/yolo-nas/results_yolo_nas_m.json new file mode 100644 index 0000000..793af05 --- /dev/null +++ b/models/object_detection/yolo-nas/results_yolo_nas_m.json @@ -0,0 +1,37 @@ +{ + "metadata": { + "model": "yolo_nas_m", + "param_count": 51182658, + "run_date": "2024-09-11T15:13:41.408248+00:00" + }, + "map50_95": 0.47600776753268037, + "map50": 0.6371598084661176, + "map75": 0.5179628054543638, + "small_objects": { + "map50_95": 0.19989784197863897, + "map50": 0.315776115787245, + "map75": 0.21839721685059119 + }, + "medium_objects": { + "map50_95": 0.4439337891094138, + "map50": 0.6124309996092572, + "map75": 0.4995806950166813 + }, + "large_objects": { + "map50_95": 0.6217121959458887, + "map50": 0.7532565846774951, + "map75": 0.6717871346846185 + }, + "iou_thresholds": [ + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.8999999999999999, + 0.95 + ] +} diff --git a/models/object_detection/yolo-nas/results_yolo_nas_s.json b/models/object_detection/yolo-nas/results_yolo_nas_s.json new file mode 100644 index 0000000..fc9b902 --- /dev/null +++ b/models/object_detection/yolo-nas/results_yolo_nas_s.json @@ -0,0 +1,37 @@ +{ + "metadata": { + "model": "yolo_nas_s", + "param_count": 19053888, + "run_date": "2024-09-11T14:55:31.676004+00:00" + }, + "map50_95": 0.43622945288618536, + "map50": 0.5995001368949364, + "map75": 0.4723807816255251, + "small_objects": { + "map50_95": 0.17054831954083777, + "map50": 0.2698303872724884, + "map75": 0.18250067396376432 + }, + "medium_objects": { + "map50_95": 0.3963692663207268, + "map50": 0.5649695419910257, + "map75": 0.4383779405933604 + }, + "large_objects": { + "map50_95": 0.580714409729971, + "map50": 0.723936892681672, + "map75": 0.6297402510946248 + }, + "iou_thresholds": [ + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.8999999999999999, + 0.95 + ] +} diff --git a/models/object_detection/yolo-nas/run.py b/models/object_detection/yolo-nas/run.py new file mode 100644 index 0000000..16f785a --- /dev/null +++ b/models/object_detection/yolo-nas/run.py @@ -0,0 +1,97 @@ +import argparse +import sys +from pathlib import Path +from typing import List, Optional + +import super_gradients +import super_gradients.training +import super_gradients.training.models +import supervision as sv +import torch +from tqdm import tqdm + +sys.path.append(str(Path(__file__).resolve().parent.parent)) +from utils import ( + load_detections_dataset, + result_json_already_exists, + write_result_json, +) + +MODEL_IDS = ["yolo_nas_s", "yolo_nas_m", "yolo_nas_l"] +DATASET_DIR = "../../../data/coco-val-2017" +CONFIDENCE_THRESHOLD = 0.001 + + +def run_on_image(model, image) -> sv.Detections: + model_params = dict( + iou=0.6, + conf=0.001, + ) + result = model.predict(image, **model_params) + detections = sv.Detections.from_yolo_nas(result) + detections = detections[detections.confidence > CONFIDENCE_THRESHOLD] + return detections + + +def run( + model_ids: List[str], + skip_if_result_exists=False, + dataset: Optional[sv.DetectionDataset] = None, +) -> None: + """ + Run the evaluation for the given models and dataset. + + Arguments: + model_ids: List of model ids to evaluate. Evaluate all models if None. + skip_if_result_exists: If True, skip the evaluation if the result json already exists. + dataset: If provided, use this dataset for evaluation. Otherwise, load the dataset from the default directory. + """ # noqa: E501 // docs + if not model_ids: + model_ids = MODEL_IDS + + for model_id in model_ids: + print(f"\nEvaluating model: {model_id}") + + if skip_if_result_exists and result_json_already_exists(model_id): + print(f"Skipping {model_id}. Result already exists!") + continue + + if dataset is None: + dataset = load_detections_dataset(DATASET_DIR) + + model = super_gradients.training.models.get(model_id, pretrained_weights="coco") + if torch.cuda.is_available(): + model = model.cuda() + + predictions = [] + targets = [] + print("Evaluating...") + for _, image, target_detections in tqdm(dataset, total=len(dataset)): + # Run model + detections = run_on_image(model, image) + predictions.append(detections) + targets.append(target_detections) + + mAP_metric = sv.metrics.MeanAveragePrecision() + mAP_result = mAP_metric.update(predictions, targets).compute() + + write_result_json( + model_id=model_id, model_name=model_id, model=model, mAP_result=mAP_result + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "model_ids", + nargs="*", + help="Model ids to evaluate. If not provided, evaluate all models.", + ) + parser.add_argument( + "--skip_if_result_exists", + help="If specified, skip the evaluation if the result json already exists.", + action="store_true", + ) + args = parser.parse_args() + + run(args.model_ids, args.skip_if_result_exists) diff --git a/static/aggregate_results.js b/static/aggregate_results.js index 1590a42..c62222d 100644 --- a/static/aggregate_results.js +++ b/static/aggregate_results.js @@ -1,27 +1,27 @@ const results = [ { "metadata": { - "model": "yolov8l", - "param_count": 43668288, - "run_date": "2024-09-10T23:36:29.969043+00:00" + "model": "yolov10b", + "param_count": 20534512, + "run_date": "2024-09-11T06:26:25.254425+00:00" }, - "map50_95": 0.5113536598644233, - "map50": 0.6818119743607345, - "map75": 0.5532226083613143, + "map50_95": 0.5143792118069956, + "map50": 0.6824004061676368, + "map75": 0.5627853497461194, "small_objects": { - "map50_95": 0.2541488631661326, - "map50": 0.3941095872790704, - "map75": 0.2741537493729201 + "map50_95": 0.2564719395368623, + "map50": 0.3970434043826135, + "map75": 0.2797247747359929 }, "medium_objects": { - "map50_95": 0.47955247178653887, - "map50": 0.6528704380513769, - "map75": 0.5408872552762262 + "map50_95": 0.48297143947641724, + "map50": 0.65475608470514, + "map75": 0.5439932731257328 }, "large_objects": { - "map50_95": 0.6474371533634005, - "map50": 0.7813953355680889, - "map75": 0.69372895366936 + "map50_95": 0.6501877214588737, + "map50": 0.7839027323721222, + "map75": 0.7070045634235113 }, "iou_thresholds": [ 0.5, @@ -38,27 +38,27 @@ const results = [ }, { "metadata": { - "model": "yolov8x", - "param_count": 68200608, - "run_date": "2024-09-10T23:52:10.141435+00:00" + "model": "yolov10x", + "param_count": 31738160, + "run_date": "2024-09-11T06:58:14.018727+00:00" }, - "map50_95": 0.5224729352634083, - "map50": 0.6928428439393775, - "map75": 0.5658418835396923, + "map50_95": 0.5322172579705878, + "map50": 0.7001374705335366, + "map75": 0.5797446524456603, "small_objects": { - "map50_95": 0.25598979356112733, - "map50": 0.39654255057136006, - "map75": 0.274085101372328 + "map50_95": 0.27274637042720357, + "map50": 0.4222710720281591, + "map75": 0.2957131128311451 }, "medium_objects": { - "map50_95": 0.48758668056752924, - "map50": 0.6618212906647764, - "map75": 0.5442969728329826 + "map50_95": 0.5004643874851188, + "map50": 0.6724499844328559, + "map75": 0.5629014808086344 }, "large_objects": { - "map50_95": 0.6571236038359161, - "map50": 0.7926417959505571, - "map75": 0.706921017993406 + "map50_95": 0.6687752341231782, + "map50": 0.7987958798254203, + "map75": 0.7207387845010169 }, "iou_thresholds": [ 0.5, @@ -75,27 +75,27 @@ const results = [ }, { "metadata": { - "model": "yolov8m", - "param_count": 25886080, - "run_date": "2024-09-10T23:24:56.134038+00:00" + "model": "yolov10n", + "param_count": 2762608, + "run_date": "2024-09-11T06:00:26.460240+00:00" }, - "map50_95": 0.4850764482852273, - "map50": 0.6557558710184571, - "map75": 0.5236025818200577, + "map50_95": 0.3765088416879911, + "map50": 0.5260609848471127, + "map75": 0.4090723182347159, "small_objects": { - "map50_95": 0.21862758887116926, - "map50": 0.3504338545153029, - "map75": 0.2328800190227224 + "map50_95": 0.12016938940212646, + "map50": 0.20362997388940046, + "map75": 0.1268368748435595 }, "medium_objects": { - "map50_95": 0.4505731666771029, - "map50": 0.6262046682033715, - "map75": 0.504870169981875 + "map50_95": 0.32276234193312875, + "map50": 0.4725625917155686, + "map75": 0.3594433755980026 }, "large_objects": { - "map50_95": 0.6209877039148616, - "map50": 0.7622134508520726, - "map75": 0.6666446122269053 + "map50_95": 0.5221844566075309, + "map50": 0.6637191598604325, + "map75": 0.5712079164028042 }, "iou_thresholds": [ 0.5, @@ -112,27 +112,27 @@ const results = [ }, { "metadata": { - "model": "yolov8s", - "param_count": 11156544, - "run_date": "2024-09-10T23:18:11.913012+00:00" + "model": "yolov10l", + "param_count": 25839728, + "run_date": "2024-09-11T06:40:36.736680+00:00" }, - "map50_95": 0.4349514933109458, - "map50": 0.6034741727406848, - "map75": 0.4663653734518539, + "map50_95": 0.5193880242072942, + "map50": 0.6877678233859065, + "map75": 0.5671532124246925, "small_objects": { - "map50_95": 0.17419739522901567, - "map50": 0.2832984180771835, - "map75": 0.1873310846587216 + "map50_95": 0.2584432484516646, + "map50": 0.40473751495371346, + "map75": 0.28128426987698835 }, "medium_objects": { - "map50_95": 0.39280326869155036, - "map50": 0.5620035052841122, - "map75": 0.43216501299799076 + "map50_95": 0.4859813785361321, + "map50": 0.6582199031259204, + "map75": 0.5477517431630547 }, "large_objects": { - "map50_95": 0.574521564036771, - "map50": 0.7237101815423765, - "map75": 0.619344639907595 + "map50_95": 0.6589157327805413, + "map50": 0.7926125584781836, + "map75": 0.714490451301635 }, "iou_thresholds": [ 0.5, @@ -149,27 +149,27 @@ const results = [ }, { "metadata": { - "model": "yolov8n", - "param_count": 3151904, - "run_date": "2024-09-10T23:14:14.800283+00:00" + "model": "yolov10m", + "param_count": 16543024, + "run_date": "2024-09-11T06:15:13.254833+00:00" }, - "map50_95": 0.36107992040614717, - "map50": 0.5177048447680661, - "map75": 0.38832373033586254, + "map50_95": 0.4985133021605519, + "map50": 0.6674225563674506, + "map75": 0.5440483868841379, "small_objects": { - "map50_95": 0.1116102187695158, - "map50": 0.19653257553097706, - "map75": 0.11004165166136577 + "map50_95": 0.24321138515145269, + "map50": 0.3794609907449295, + "map75": 0.25837288385953333 }, "medium_objects": { - "map50_95": 0.31055786390355683, - "map50": 0.46337363140413484, - "map75": 0.3366583462842102 + "map50_95": 0.4666641326436146, + "map50": 0.6402552349229303, + "map75": 0.5266073132389671 }, "large_objects": { - "map50_95": 0.4956530290107429, - "map50": 0.6444217871612856, - "map75": 0.5454911394754738 + "map50_95": 0.6337068975510818, + "map50": 0.769311494847955, + "map75": 0.6872034980360436 }, "iou_thresholds": [ 0.5, @@ -186,27 +186,27 @@ const results = [ }, { "metadata": { - "model": "RT-DETRv1 (r50vd)", - "param_count": 49972332, - "run_date": "2024-09-10T16:18:12.499365+00:00" + "model": "yolov10s", + "param_count": 8096880, + "run_date": "2024-09-11T06:06:20.613349+00:00" }, - "map50_95": 0.5148052539985135, - "map50": 0.6922127474679615, - "map75": 0.558997224099962, + "map50_95": 0.45374339278894915, + "map50": 0.6202786326213161, + "map75": 0.49485980546052266, "small_objects": { - "map50_95": 0.24458802092882242, - "map50": 0.38689885245134953, - "map75": 0.2630160546386321 + "map50_95": 0.18538386854372754, + "map50": 0.29935561442986514, + "map75": 0.19679687800521717 }, "medium_objects": { - "map50_95": 0.46672014734193157, - "map50": 0.6472867660110199, - "map75": 0.5228654414090166 + "map50_95": 0.40763577645735893, + "map50": 0.5762364117411901, + "map75": 0.45695369625119897 }, "large_objects": { - "map50_95": 0.6591424467339809, - "map50": 0.7966885597912683, - "map75": 0.7127943119656794 + "map50_95": 0.6015530294400476, + "map50": 0.7464275332834238, + "map75": 0.6560326855935931 }, "iou_thresholds": [ 0.5, @@ -258,6 +258,80 @@ const results = [ 0.95 ] }, + { + "metadata": { + "model": "RT-DETRv1 (r18vd)", + "param_count": 21955472, + "run_date": "2024-09-10T16:11:52.358781+00:00" + }, + "map50_95": 0.4475030676297469, + "map50": 0.6156237779572045, + "map75": 0.48407456419504113, + "small_objects": { + "map50_95": 0.19035576042172211, + "map50": 0.3089236487784832, + "map75": 0.2030844478437464 + }, + "medium_objects": { + "map50_95": 0.3952415547608772, + "map50": 0.5612930297373171, + "map75": 0.442782433290807 + }, + "large_objects": { + "map50_95": 0.5894997261827915, + "map50": 0.7265982743927857, + "map75": 0.6411468239795821 + }, + "iou_thresholds": [ + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.8999999999999999, + 0.95 + ] + }, + { + "metadata": { + "model": "RT-DETRv1 (r50vd)", + "param_count": 49972332, + "run_date": "2024-09-10T16:18:12.499365+00:00" + }, + "map50_95": 0.5148052539985135, + "map50": 0.6922127474679615, + "map75": 0.558997224099962, + "small_objects": { + "map50_95": 0.24458802092882242, + "map50": 0.38689885245134953, + "map75": 0.2630160546386321 + }, + "medium_objects": { + "map50_95": 0.46672014734193157, + "map50": 0.6472867660110199, + "map75": 0.5228654414090166 + }, + "large_objects": { + "map50_95": 0.6591424467339809, + "map50": 0.7966885597912683, + "map75": 0.7127943119656794 + }, + "iou_thresholds": [ + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.8999999999999999, + 0.95 + ] + }, { "metadata": { "model": "RT-DETRv1 (r101vd)", @@ -297,27 +371,27 @@ const results = [ }, { "metadata": { - "model": "RT-DETRv1 (r18vd)", - "param_count": 21955472, - "run_date": "2024-09-10T16:11:52.358781+00:00" + "model": "yolov8m", + "param_count": 25886080, + "run_date": "2024-09-10T23:24:56.134038+00:00" }, - "map50_95": 0.4475030676297469, - "map50": 0.6156237779572045, - "map75": 0.48407456419504113, + "map50_95": 0.4850764482852273, + "map50": 0.6557558710184571, + "map75": 0.5236025818200577, "small_objects": { - "map50_95": 0.19035576042172211, - "map50": 0.3089236487784832, - "map75": 0.2030844478437464 + "map50_95": 0.21862758887116926, + "map50": 0.3504338545153029, + "map75": 0.2328800190227224 }, "medium_objects": { - "map50_95": 0.3952415547608772, - "map50": 0.5612930297373171, - "map75": 0.442782433290807 + "map50_95": 0.4505731666771029, + "map50": 0.6262046682033715, + "map75": 0.504870169981875 }, "large_objects": { - "map50_95": 0.5894997261827915, - "map50": 0.7265982743927857, - "map75": 0.6411468239795821 + "map50_95": 0.6209877039148616, + "map50": 0.7622134508520726, + "map75": 0.6666446122269053 }, "iou_thresholds": [ 0.5, @@ -334,27 +408,27 @@ const results = [ }, { "metadata": { - "model": "yolov10s", - "param_count": 8096880, - "run_date": "2024-09-11T06:06:20.613349+00:00" + "model": "yolov8s", + "param_count": 11156544, + "run_date": "2024-09-10T23:18:11.913012+00:00" }, - "map50_95": 0.45374339278894915, - "map50": 0.6202786326213161, - "map75": 0.49485980546052266, + "map50_95": 0.4349514933109458, + "map50": 0.6034741727406848, + "map75": 0.4663653734518539, "small_objects": { - "map50_95": 0.18538386854372754, - "map50": 0.29935561442986514, - "map75": 0.19679687800521717 + "map50_95": 0.17419739522901567, + "map50": 0.2832984180771835, + "map75": 0.1873310846587216 }, "medium_objects": { - "map50_95": 0.40763577645735893, - "map50": 0.5762364117411901, - "map75": 0.45695369625119897 + "map50_95": 0.39280326869155036, + "map50": 0.5620035052841122, + "map75": 0.43216501299799076 }, "large_objects": { - "map50_95": 0.6015530294400476, - "map50": 0.7464275332834238, - "map75": 0.6560326855935931 + "map50_95": 0.574521564036771, + "map50": 0.7237101815423765, + "map75": 0.619344639907595 }, "iou_thresholds": [ 0.5, @@ -371,27 +445,27 @@ const results = [ }, { "metadata": { - "model": "yolov10l", - "param_count": 25839728, - "run_date": "2024-09-11T06:40:36.736680+00:00" + "model": "yolov8n", + "param_count": 3151904, + "run_date": "2024-09-10T23:14:14.800283+00:00" }, - "map50_95": 0.5193880242072942, - "map50": 0.6877678233859065, - "map75": 0.5671532124246925, + "map50_95": 0.36107992040614717, + "map50": 0.5177048447680661, + "map75": 0.38832373033586254, "small_objects": { - "map50_95": 0.2584432484516646, - "map50": 0.40473751495371346, - "map75": 0.28128426987698835 + "map50_95": 0.1116102187695158, + "map50": 0.19653257553097706, + "map75": 0.11004165166136577 }, "medium_objects": { - "map50_95": 0.4859813785361321, - "map50": 0.6582199031259204, - "map75": 0.5477517431630547 + "map50_95": 0.31055786390355683, + "map50": 0.46337363140413484, + "map75": 0.3366583462842102 }, "large_objects": { - "map50_95": 0.6589157327805413, - "map50": 0.7926125584781836, - "map75": 0.714490451301635 + "map50_95": 0.4956530290107429, + "map50": 0.6444217871612856, + "map75": 0.5454911394754738 }, "iou_thresholds": [ 0.5, @@ -408,27 +482,27 @@ const results = [ }, { "metadata": { - "model": "yolov10m", - "param_count": 16543024, - "run_date": "2024-09-11T06:15:13.254833+00:00" + "model": "yolov8x", + "param_count": 68200608, + "run_date": "2024-09-10T23:52:10.141435+00:00" }, - "map50_95": 0.4985133021605519, - "map50": 0.6674225563674506, - "map75": 0.5440483868841379, + "map50_95": 0.5224729352634083, + "map50": 0.6928428439393775, + "map75": 0.5658418835396923, "small_objects": { - "map50_95": 0.24321138515145269, - "map50": 0.3794609907449295, - "map75": 0.25837288385953333 + "map50_95": 0.25598979356112733, + "map50": 0.39654255057136006, + "map75": 0.274085101372328 }, "medium_objects": { - "map50_95": 0.4666641326436146, - "map50": 0.6402552349229303, - "map75": 0.5266073132389671 + "map50_95": 0.48758668056752924, + "map50": 0.6618212906647764, + "map75": 0.5442969728329826 }, "large_objects": { - "map50_95": 0.6337068975510818, - "map50": 0.769311494847955, - "map75": 0.6872034980360436 + "map50_95": 0.6571236038359161, + "map50": 0.7926417959505571, + "map75": 0.706921017993406 }, "iou_thresholds": [ 0.5, @@ -445,27 +519,27 @@ const results = [ }, { "metadata": { - "model": "yolov10x", - "param_count": 31738160, - "run_date": "2024-09-11T06:58:14.018727+00:00" + "model": "yolov8l", + "param_count": 43668288, + "run_date": "2024-09-10T23:36:29.969043+00:00" }, - "map50_95": 0.5322172579705878, - "map50": 0.7001374705335366, - "map75": 0.5797446524456603, + "map50_95": 0.5113536598644233, + "map50": 0.6818119743607345, + "map75": 0.5532226083613143, "small_objects": { - "map50_95": 0.27274637042720357, - "map50": 0.4222710720281591, - "map75": 0.2957131128311451 + "map50_95": 0.2541488631661326, + "map50": 0.3941095872790704, + "map75": 0.2741537493729201 }, "medium_objects": { - "map50_95": 0.5004643874851188, - "map50": 0.6724499844328559, - "map75": 0.5629014808086344 + "map50_95": 0.47955247178653887, + "map50": 0.6528704380513769, + "map75": 0.5408872552762262 }, "large_objects": { - "map50_95": 0.6687752341231782, - "map50": 0.7987958798254203, - "map75": 0.7207387845010169 + "map50_95": 0.6474371533634005, + "map50": 0.7813953355680889, + "map75": 0.69372895366936 }, "iou_thresholds": [ 0.5, @@ -482,27 +556,27 @@ const results = [ }, { "metadata": { - "model": "yolov10n", - "param_count": 2762608, - "run_date": "2024-09-11T06:00:26.460240+00:00" + "model": "RT-DETRv2 (r34vd)", + "param_count": 33212676, + "run_date": "2024-09-10T16:30:46.673344+00:00" }, - "map50_95": 0.3765088416879911, - "map50": 0.5260609848471127, - "map75": 0.4090723182347159, + "map50_95": 0.4867825440454518, + "map50": 0.6591044524702075, + "map75": 0.5255439550758243, "small_objects": { - "map50_95": 0.12016938940212646, - "map50": 0.20362997388940046, - "map75": 0.1268368748435595 + "map50_95": 0.22517464073284388, + "map50": 0.36161616082579096, + "map75": 0.23804608448553247 }, "medium_objects": { - "map50_95": 0.32276234193312875, - "map50": 0.4725625917155686, - "map75": 0.3594433755980026 + "map50_95": 0.4361924560564629, + "map50": 0.6090810274677496, + "map75": 0.48462339743332006 }, "large_objects": { - "map50_95": 0.5221844566075309, - "map50": 0.6637191598604325, - "map75": 0.5712079164028042 + "map50_95": 0.6273765615214366, + "map50": 0.7658607828593057, + "map75": 0.6792628879214015 }, "iou_thresholds": [ 0.5, @@ -519,27 +593,27 @@ const results = [ }, { "metadata": { - "model": "yolov10b", - "param_count": 20534512, - "run_date": "2024-09-11T06:26:25.254425+00:00" + "model": "RT-DETRv2 (r18vd)", + "param_count": 21955472, + "run_date": "2024-09-10T16:26:50.070114+00:00" }, - "map50_95": 0.5143792118069956, - "map50": 0.6824004061676368, - "map75": 0.5627853497461194, + "map50_95": 0.4687995594739722, + "map50": 0.6370584442700049, + "map75": 0.506877566545174, "small_objects": { - "map50_95": 0.2564719395368623, - "map50": 0.3970434043826135, - "map75": 0.2797247747359929 + "map50_95": 0.21535478708745523, + "map50": 0.3395428793596645, + "map75": 0.22778600330760196 }, "medium_objects": { - "map50_95": 0.48297143947641724, - "map50": 0.65475608470514, - "map75": 0.5439932731257328 + "map50_95": 0.41591835004016664, + "map50": 0.582710782161214, + "map75": 0.46111270923584724 }, "large_objects": { - "map50_95": 0.6501877214588737, - "map50": 0.7839027323721222, - "map75": 0.7070045634235113 + "map50_95": 0.6075716348351401, + "map50": 0.7449516529285527, + "map75": 0.6571583698150867 }, "iou_thresholds": [ 0.5, @@ -593,27 +667,27 @@ const results = [ }, { "metadata": { - "model": "RT-DETRv2 (r18vd)", - "param_count": 21955472, - "run_date": "2024-09-10T16:26:50.070114+00:00" + "model": "RT-DETRv2 (r101vd)", + "param_count": 92486124, + "run_date": "2024-09-10T16:44:47.186839+00:00" }, - "map50_95": 0.4687995594739722, - "map50": 0.6370584442700049, - "map75": 0.506877566545174, + "map50_95": 0.524330135379127, + "map50": 0.7039527758683548, + "map75": 0.56780497182413, "small_objects": { - "map50_95": 0.21535478708745523, - "map50": 0.3395428793596645, - "map75": 0.22778600330760196 + "map50_95": 0.24785492909149326, + "map50": 0.40066746534705366, + "map75": 0.2710220492958572 }, "medium_objects": { - "map50_95": 0.41591835004016664, - "map50": 0.582710782161214, - "map75": 0.46111270923584724 + "map50_95": 0.47685835692588135, + "map50": 0.66254604001764, + "map75": 0.5350030177768078 }, "large_objects": { - "map50_95": 0.6075716348351401, - "map50": 0.7449516529285527, - "map75": 0.6571583698150867 + "map50_95": 0.6758202302184949, + "map50": 0.8126341241234784, + "map75": 0.7285502979309773 }, "iou_thresholds": [ 0.5, @@ -667,27 +741,27 @@ const results = [ }, { "metadata": { - "model": "RT-DETRv2 (r101vd)", - "param_count": 92486124, - "run_date": "2024-09-10T16:44:47.186839+00:00" + "model": "yolo_nas_m", + "param_count": 51182658, + "run_date": "2024-09-11T15:13:41.408248+00:00" }, - "map50_95": 0.524330135379127, - "map50": 0.7039527758683548, - "map75": 0.56780497182413, + "map50_95": 0.47600776753268037, + "map50": 0.6371598084661176, + "map75": 0.5179628054543638, "small_objects": { - "map50_95": 0.24785492909149326, - "map50": 0.40066746534705366, - "map75": 0.2710220492958572 + "map50_95": 0.19989784197863897, + "map50": 0.315776115787245, + "map75": 0.21839721685059119 }, "medium_objects": { - "map50_95": 0.47685835692588135, - "map50": 0.66254604001764, - "map75": 0.5350030177768078 + "map50_95": 0.4439337891094138, + "map50": 0.6124309996092572, + "map75": 0.4995806950166813 }, "large_objects": { - "map50_95": 0.6758202302184949, - "map50": 0.8126341241234784, - "map75": 0.7285502979309773 + "map50_95": 0.6217121959458887, + "map50": 0.7532565846774951, + "map75": 0.6717871346846185 }, "iou_thresholds": [ 0.5, @@ -704,27 +778,64 @@ const results = [ }, { "metadata": { - "model": "RT-DETRv2 (r34vd)", - "param_count": 33212676, - "run_date": "2024-09-10T16:30:46.673344+00:00" + "model": "yolo_nas_l", + "param_count": 66976392, + "run_date": "2024-09-11T15:34:44.524051+00:00" }, - "map50_95": 0.4867825440454518, - "map50": 0.6591044524702075, - "map75": 0.5255439550758243, + "map50_95": 0.4823908978773399, + "map50": 0.6436635942062905, + "map75": 0.5227631511898091, "small_objects": { - "map50_95": 0.22517464073284388, - "map50": 0.36161616082579096, - "map75": 0.23804608448553247 + "map50_95": 0.20627144126452443, + "map50": 0.3234195270645268, + "map75": 0.22433010779366858 }, "medium_objects": { - "map50_95": 0.4361924560564629, - "map50": 0.6090810274677496, - "map75": 0.48462339743332006 + "map50_95": 0.4504440067740434, + "map50": 0.6198672027100715, + "map75": 0.5028411865713277 }, "large_objects": { - "map50_95": 0.6273765615214366, - "map50": 0.7658607828593057, - "map75": 0.6792628879214015 + "map50_95": 0.6232514892633709, + "map50": 0.7530164274037385, + "map75": 0.6684965691185336 + }, + "iou_thresholds": [ + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.8999999999999999, + 0.95 + ] + }, + { + "metadata": { + "model": "yolo_nas_s", + "param_count": 19053888, + "run_date": "2024-09-11T14:55:31.676004+00:00" + }, + "map50_95": 0.43622945288618536, + "map50": 0.5995001368949364, + "map75": 0.4723807816255251, + "small_objects": { + "map50_95": 0.17054831954083777, + "map50": 0.2698303872724884, + "map75": 0.18250067396376432 + }, + "medium_objects": { + "map50_95": 0.3963692663207268, + "map50": 0.5649695419910257, + "map75": 0.4383779405933604 + }, + "large_objects": { + "map50_95": 0.580714409729971, + "map50": 0.723936892681672, + "map75": 0.6297402510946248 }, "iou_thresholds": [ 0.5, From 5330574332fb606dde646eec96768d5d99e0251d Mon Sep 17 00:00:00 2001 From: quangdungluong Date: Wed, 11 Sep 2024 23:18:17 +0700 Subject: [PATCH 2/6] chore: build static site --- static/aggregate_results.js | 495 ++++++++++++++---------------------- 1 file changed, 192 insertions(+), 303 deletions(-) diff --git a/static/aggregate_results.js b/static/aggregate_results.js index 21d681a..452351a 100644 --- a/static/aggregate_results.js +++ b/static/aggregate_results.js @@ -6,23 +6,23 @@ const results = [ "param_count": 43668288, "run_date": "2024-09-10T23:36:29.969043+00:00" }, - "map50_95": 0.5143792118069956, - "map50": 0.6824004061676368, - "map75": 0.5627853497461194, + "map50_95": 0.5113536598644233, + "map50": 0.6818119743607345, + "map75": 0.5532226083613143, "small_objects": { - "map50_95": 0.2564719395368623, - "map50": 0.3970434043826135, - "map75": 0.2797247747359929 + "map50_95": 0.2541488631661326, + "map50": 0.3941095872790704, + "map75": 0.2741537493729201 }, "medium_objects": { - "map50_95": 0.48297143947641724, - "map50": 0.65475608470514, - "map75": 0.5439932731257328 + "map50_95": 0.47955247178653887, + "map50": 0.6528704380513769, + "map75": 0.5408872552762262 }, "large_objects": { - "map50_95": 0.6501877214588737, - "map50": 0.7839027323721222, - "map75": 0.7070045634235113 + "map50_95": 0.6474371533634005, + "map50": 0.7813953355680889, + "map75": 0.69372895366936 }, "iou_thresholds": [ 0.5, @@ -44,23 +44,23 @@ const results = [ "param_count": 68200608, "run_date": "2024-09-10T23:52:10.141435+00:00" }, - "map50_95": 0.5322172579705878, - "map50": 0.7001374705335366, - "map75": 0.5797446524456603, + "map50_95": 0.5224729352634083, + "map50": 0.6928428439393775, + "map75": 0.5658418835396923, "small_objects": { - "map50_95": 0.27274637042720357, - "map50": 0.4222710720281591, - "map75": 0.2957131128311451 + "map50_95": 0.25598979356112733, + "map50": 0.39654255057136006, + "map75": 0.274085101372328 }, "medium_objects": { - "map50_95": 0.5004643874851188, - "map50": 0.6724499844328559, - "map75": 0.5629014808086344 + "map50_95": 0.48758668056752924, + "map50": 0.6618212906647764, + "map75": 0.5442969728329826 }, "large_objects": { - "map50_95": 0.6687752341231782, - "map50": 0.7987958798254203, - "map75": 0.7207387845010169 + "map50_95": 0.6571236038359161, + "map50": 0.7926417959505571, + "map75": 0.706921017993406 }, "iou_thresholds": [ 0.5, @@ -82,23 +82,23 @@ const results = [ "param_count": 25886080, "run_date": "2024-09-10T23:24:56.134038+00:00" }, - "map50_95": 0.3765088416879911, - "map50": 0.5260609848471127, - "map75": 0.4090723182347159, + "map50_95": 0.4850764482852273, + "map50": 0.6557558710184571, + "map75": 0.5236025818200577, "small_objects": { - "map50_95": 0.12016938940212646, - "map50": 0.20362997388940046, - "map75": 0.1268368748435595 + "map50_95": 0.21862758887116926, + "map50": 0.3504338545153029, + "map75": 0.2328800190227224 }, "medium_objects": { - "map50_95": 0.32276234193312875, - "map50": 0.4725625917155686, - "map75": 0.3594433755980026 + "map50_95": 0.4505731666771029, + "map50": 0.6262046682033715, + "map75": 0.504870169981875 }, "large_objects": { - "map50_95": 0.5221844566075309, - "map50": 0.6637191598604325, - "map75": 0.5712079164028042 + "map50_95": 0.6209877039148616, + "map50": 0.7622134508520726, + "map75": 0.6666446122269053 }, "iou_thresholds": [ 0.5, @@ -120,23 +120,23 @@ const results = [ "param_count": 11156544, "run_date": "2024-09-10T23:18:11.913012+00:00" }, - "map50_95": 0.5193880242072942, - "map50": 0.6877678233859065, - "map75": 0.5671532124246925, + "map50_95": 0.4349514933109458, + "map50": 0.6034741727406848, + "map75": 0.4663653734518539, "small_objects": { - "map50_95": 0.2584432484516646, - "map50": 0.40473751495371346, - "map75": 0.28128426987698835 + "map50_95": 0.17419739522901567, + "map50": 0.2832984180771835, + "map75": 0.1873310846587216 }, "medium_objects": { - "map50_95": 0.4859813785361321, - "map50": 0.6582199031259204, - "map75": 0.5477517431630547 + "map50_95": 0.39280326869155036, + "map50": 0.5620035052841122, + "map75": 0.43216501299799076 }, "large_objects": { - "map50_95": 0.6589157327805413, - "map50": 0.7926125584781836, - "map75": 0.714490451301635 + "map50_95": 0.574521564036771, + "map50": 0.7237101815423765, + "map75": 0.619344639907595 }, "iou_thresholds": [ 0.5, @@ -158,23 +158,23 @@ const results = [ "param_count": 3151904, "run_date": "2024-09-10T23:14:14.800283+00:00" }, - "map50_95": 0.4985133021605519, - "map50": 0.6674225563674506, - "map75": 0.5440483868841379, + "map50_95": 0.36107992040614717, + "map50": 0.5177048447680661, + "map75": 0.38832373033586254, "small_objects": { - "map50_95": 0.24321138515145269, - "map50": 0.3794609907449295, - "map75": 0.25837288385953333 + "map50_95": 0.1116102187695158, + "map50": 0.19653257553097706, + "map75": 0.11004165166136577 }, "medium_objects": { - "map50_95": 0.4666641326436146, - "map50": 0.6402552349229303, - "map75": 0.5266073132389671 + "map50_95": 0.31055786390355683, + "map50": 0.46337363140413484, + "map75": 0.3366583462842102 }, "large_objects": { - "map50_95": 0.6337068975510818, - "map50": 0.769311494847955, - "map75": 0.6872034980360436 + "map50_95": 0.4956530290107429, + "map50": 0.6444217871612856, + "map75": 0.5454911394754738 }, "iou_thresholds": [ 0.5, @@ -196,23 +196,23 @@ const results = [ "param_count": 49972332, "run_date": "2024-09-10T16:18:12.499365+00:00" }, - "map50_95": 0.45374339278894915, - "map50": 0.6202786326213161, - "map75": 0.49485980546052266, + "map50_95": 0.5148052539985135, + "map50": 0.6922127474679615, + "map75": 0.558997224099962, "small_objects": { - "map50_95": 0.18538386854372754, - "map50": 0.29935561442986514, - "map75": 0.19679687800521717 + "map50_95": 0.24458802092882242, + "map50": 0.38689885245134953, + "map75": 0.2630160546386321 }, "medium_objects": { - "map50_95": 0.40763577645735893, - "map50": 0.5762364117411901, - "map75": 0.45695369625119897 + "map50_95": 0.46672014734193157, + "map50": 0.6472867660110199, + "map75": 0.5228654414090166 }, "large_objects": { - "map50_95": 0.6015530294400476, - "map50": 0.7464275332834238, - "map75": 0.6560326855935931 + "map50_95": 0.6591424467339809, + "map50": 0.7966885597912683, + "map75": 0.7127943119656794 }, "iou_thresholds": [ 0.5, @@ -265,80 +265,6 @@ const results = [ 0.95 ] }, - { - "metadata": { - "model": "RT-DETRv1 (r18vd)", - "param_count": 21955472, - "run_date": "2024-09-10T16:11:52.358781+00:00" - }, - "map50_95": 0.4475030676297469, - "map50": 0.6156237779572045, - "map75": 0.48407456419504113, - "small_objects": { - "map50_95": 0.19035576042172211, - "map50": 0.3089236487784832, - "map75": 0.2030844478437464 - }, - "medium_objects": { - "map50_95": 0.3952415547608772, - "map50": 0.5612930297373171, - "map75": 0.442782433290807 - }, - "large_objects": { - "map50_95": 0.5894997261827915, - "map50": 0.7265982743927857, - "map75": 0.6411468239795821 - }, - "iou_thresholds": [ - 0.5, - 0.55, - 0.6, - 0.65, - 0.7, - 0.75, - 0.8, - 0.85, - 0.8999999999999999, - 0.95 - ] - }, - { - "metadata": { - "model": "RT-DETRv1 (r50vd)", - "param_count": 49972332, - "run_date": "2024-09-10T16:18:12.499365+00:00" - }, - "map50_95": 0.5148052539985135, - "map50": 0.6922127474679615, - "map75": 0.558997224099962, - "small_objects": { - "map50_95": 0.24458802092882242, - "map50": 0.38689885245134953, - "map75": 0.2630160546386321 - }, - "medium_objects": { - "map50_95": 0.46672014734193157, - "map50": 0.6472867660110199, - "map75": 0.5228654414090166 - }, - "large_objects": { - "map50_95": 0.6591424467339809, - "map50": 0.7966885597912683, - "map75": 0.7127943119656794 - }, - "iou_thresholds": [ - 0.5, - 0.55, - 0.6, - 0.65, - 0.7, - 0.75, - 0.8, - 0.85, - 0.8999999999999999, - 0.95 - ] - }, { "metadata": { "model": "RT-DETRv1 (r101vd)", @@ -384,23 +310,23 @@ const results = [ "param_count": 21955472, "run_date": "2024-09-10T16:11:52.358781+00:00" }, - "map50_95": 0.4850764482852273, - "map50": 0.6557558710184571, - "map75": 0.5236025818200577, + "map50_95": 0.4475030676297469, + "map50": 0.6156237779572045, + "map75": 0.48407456419504113, "small_objects": { - "map50_95": 0.21862758887116926, - "map50": 0.3504338545153029, - "map75": 0.2328800190227224 + "map50_95": 0.19035576042172211, + "map50": 0.3089236487784832, + "map75": 0.2030844478437464 }, "medium_objects": { - "map50_95": 0.4505731666771029, - "map50": 0.6262046682033715, - "map75": 0.504870169981875 + "map50_95": 0.3952415547608772, + "map50": 0.5612930297373171, + "map75": 0.442782433290807 }, "large_objects": { - "map50_95": 0.6209877039148616, - "map50": 0.7622134508520726, - "map75": 0.6666446122269053 + "map50_95": 0.5894997261827915, + "map50": 0.7265982743927857, + "map75": 0.6411468239795821 }, "iou_thresholds": [ 0.5, @@ -422,23 +348,23 @@ const results = [ "param_count": 8096880, "run_date": "2024-09-11T06:06:20.613349+00:00" }, - "map50_95": 0.4349514933109458, - "map50": 0.6034741727406848, - "map75": 0.4663653734518539, + "map50_95": 0.45374339278894915, + "map50": 0.6202786326213161, + "map75": 0.49485980546052266, "small_objects": { - "map50_95": 0.17419739522901567, - "map50": 0.2832984180771835, - "map75": 0.1873310846587216 + "map50_95": 0.18538386854372754, + "map50": 0.29935561442986514, + "map75": 0.19679687800521717 }, "medium_objects": { - "map50_95": 0.39280326869155036, - "map50": 0.5620035052841122, - "map75": 0.43216501299799076 + "map50_95": 0.40763577645735893, + "map50": 0.5762364117411901, + "map75": 0.45695369625119897 }, "large_objects": { - "map50_95": 0.574521564036771, - "map50": 0.7237101815423765, - "map75": 0.619344639907595 + "map50_95": 0.6015530294400476, + "map50": 0.7464275332834238, + "map75": 0.6560326855935931 }, "iou_thresholds": [ 0.5, @@ -460,23 +386,23 @@ const results = [ "param_count": 25839728, "run_date": "2024-09-11T06:40:36.736680+00:00" }, - "map50_95": 0.36107992040614717, - "map50": 0.5177048447680661, - "map75": 0.38832373033586254, + "map50_95": 0.5193880242072942, + "map50": 0.6877678233859065, + "map75": 0.5671532124246925, "small_objects": { - "map50_95": 0.1116102187695158, - "map50": 0.19653257553097706, - "map75": 0.11004165166136577 + "map50_95": 0.2584432484516646, + "map50": 0.40473751495371346, + "map75": 0.28128426987698835 }, "medium_objects": { - "map50_95": 0.31055786390355683, - "map50": 0.46337363140413484, - "map75": 0.3366583462842102 + "map50_95": 0.4859813785361321, + "map50": 0.6582199031259204, + "map75": 0.5477517431630547 }, "large_objects": { - "map50_95": 0.4956530290107429, - "map50": 0.6444217871612856, - "map75": 0.5454911394754738 + "map50_95": 0.6589157327805413, + "map50": 0.7926125584781836, + "map75": 0.714490451301635 }, "iou_thresholds": [ 0.5, @@ -498,23 +424,23 @@ const results = [ "param_count": 16543024, "run_date": "2024-09-11T06:15:13.254833+00:00" }, - "map50_95": 0.5224729352634083, - "map50": 0.6928428439393775, - "map75": 0.5658418835396923, + "map50_95": 0.4985133021605519, + "map50": 0.6674225563674506, + "map75": 0.5440483868841379, "small_objects": { - "map50_95": 0.25598979356112733, - "map50": 0.39654255057136006, - "map75": 0.274085101372328 + "map50_95": 0.24321138515145269, + "map50": 0.3794609907449295, + "map75": 0.25837288385953333 }, "medium_objects": { - "map50_95": 0.48758668056752924, - "map50": 0.6618212906647764, - "map75": 0.5442969728329826 + "map50_95": 0.4666641326436146, + "map50": 0.6402552349229303, + "map75": 0.5266073132389671 }, "large_objects": { - "map50_95": 0.6571236038359161, - "map50": 0.7926417959505571, - "map75": 0.706921017993406 + "map50_95": 0.6337068975510818, + "map50": 0.769311494847955, + "map75": 0.6872034980360436 }, "iou_thresholds": [ 0.5, @@ -536,23 +462,23 @@ const results = [ "param_count": 31738160, "run_date": "2024-09-11T06:58:14.018727+00:00" }, - "map50_95": 0.5113536598644233, - "map50": 0.6818119743607345, - "map75": 0.5532226083613143, + "map50_95": 0.5322172579705878, + "map50": 0.7001374705335366, + "map75": 0.5797446524456603, "small_objects": { - "map50_95": 0.2541488631661326, - "map50": 0.3941095872790704, - "map75": 0.2741537493729201 + "map50_95": 0.27274637042720357, + "map50": 0.4222710720281591, + "map75": 0.2957131128311451 }, "medium_objects": { - "map50_95": 0.47955247178653887, - "map50": 0.6528704380513769, - "map75": 0.5408872552762262 + "map50_95": 0.5004643874851188, + "map50": 0.6724499844328559, + "map75": 0.5629014808086344 }, "large_objects": { - "map50_95": 0.6474371533634005, - "map50": 0.7813953355680889, - "map75": 0.69372895366936 + "map50_95": 0.6687752341231782, + "map50": 0.7987958798254203, + "map75": 0.7207387845010169 }, "iou_thresholds": [ 0.5, @@ -574,23 +500,23 @@ const results = [ "param_count": 2762608, "run_date": "2024-09-11T06:00:26.460240+00:00" }, - "map50_95": 0.4867825440454518, - "map50": 0.6591044524702075, - "map75": 0.5255439550758243, + "map50_95": 0.3765088416879911, + "map50": 0.5260609848471127, + "map75": 0.4090723182347159, "small_objects": { - "map50_95": 0.22517464073284388, - "map50": 0.36161616082579096, - "map75": 0.23804608448553247 + "map50_95": 0.12016938940212646, + "map50": 0.20362997388940046, + "map75": 0.1268368748435595 }, "medium_objects": { - "map50_95": 0.4361924560564629, - "map50": 0.6090810274677496, - "map75": 0.48462339743332006 + "map50_95": 0.32276234193312875, + "map50": 0.4725625917155686, + "map75": 0.3594433755980026 }, "large_objects": { - "map50_95": 0.6273765615214366, - "map50": 0.7658607828593057, - "map75": 0.6792628879214015 + "map50_95": 0.5221844566075309, + "map50": 0.6637191598604325, + "map75": 0.5712079164028042 }, "iou_thresholds": [ 0.5, @@ -612,23 +538,23 @@ const results = [ "param_count": 20534512, "run_date": "2024-09-11T06:26:25.254425+00:00" }, - "map50_95": 0.4687995594739722, - "map50": 0.6370584442700049, - "map75": 0.506877566545174, + "map50_95": 0.5143792118069956, + "map50": 0.6824004061676368, + "map75": 0.5627853497461194, "small_objects": { - "map50_95": 0.21535478708745523, - "map50": 0.3395428793596645, - "map75": 0.22778600330760196 + "map50_95": 0.2564719395368623, + "map50": 0.3970434043826135, + "map75": 0.2797247747359929 }, "medium_objects": { - "map50_95": 0.41591835004016664, - "map50": 0.582710782161214, - "map75": 0.46111270923584724 + "map50_95": 0.48297143947641724, + "map50": 0.65475608470514, + "map75": 0.5439932731257328 }, "large_objects": { - "map50_95": 0.6075716348351401, - "map50": 0.7449516529285527, - "map75": 0.6571583698150867 + "map50_95": 0.6501877214588737, + "map50": 0.7839027323721222, + "map75": 0.7070045634235113 }, "iou_thresholds": [ 0.5, @@ -688,23 +614,23 @@ const results = [ "param_count": 21955472, "run_date": "2024-09-10T16:26:50.070114+00:00" }, - "map50_95": 0.524330135379127, - "map50": 0.7039527758683548, - "map75": 0.56780497182413, + "map50_95": 0.4687995594739722, + "map50": 0.6370584442700049, + "map75": 0.506877566545174, "small_objects": { - "map50_95": 0.24785492909149326, - "map50": 0.40066746534705366, - "map75": 0.2710220492958572 + "map50_95": 0.21535478708745523, + "map50": 0.3395428793596645, + "map75": 0.22778600330760196 }, "medium_objects": { - "map50_95": 0.47685835692588135, - "map50": 0.66254604001764, - "map75": 0.5350030177768078 + "map50_95": 0.41591835004016664, + "map50": 0.582710782161214, + "map75": 0.46111270923584724 }, "large_objects": { - "map50_95": 0.6758202302184949, - "map50": 0.8126341241234784, - "map75": 0.7285502979309773 + "map50_95": 0.6075716348351401, + "map50": 0.7449516529285527, + "map75": 0.6571583698150867 }, "iou_thresholds": [ 0.5, @@ -764,23 +690,23 @@ const results = [ "param_count": 92486124, "run_date": "2024-09-10T16:44:47.186839+00:00" }, - "map50_95": 0.47600776753268037, - "map50": 0.6371598084661176, - "map75": 0.5179628054543638, + "map50_95": 0.524330135379127, + "map50": 0.7039527758683548, + "map75": 0.56780497182413, "small_objects": { - "map50_95": 0.19989784197863897, - "map50": 0.315776115787245, - "map75": 0.21839721685059119 + "map50_95": 0.24785492909149326, + "map50": 0.40066746534705366, + "map75": 0.2710220492958572 }, "medium_objects": { - "map50_95": 0.4439337891094138, - "map50": 0.6124309996092572, - "map75": 0.4995806950166813 + "map50_95": 0.47685835692588135, + "map50": 0.66254604001764, + "map75": 0.5350030177768078 }, "large_objects": { - "map50_95": 0.6217121959458887, - "map50": 0.7532565846774951, - "map75": 0.6717871346846185 + "map50_95": 0.6758202302184949, + "map50": 0.8126341241234784, + "map75": 0.7285502979309773 }, "iou_thresholds": [ 0.5, @@ -802,60 +728,23 @@ const results = [ "param_count": 33212676, "run_date": "2024-09-10T16:30:46.673344+00:00" }, - "map50_95": 0.4823908978773399, - "map50": 0.6436635942062905, - "map75": 0.5227631511898091, - "small_objects": { - "map50_95": 0.20627144126452443, - "map50": 0.3234195270645268, - "map75": 0.22433010779366858 - }, - "medium_objects": { - "map50_95": 0.4504440067740434, - "map50": 0.6198672027100715, - "map75": 0.5028411865713277 - }, - "large_objects": { - "map50_95": 0.6232514892633709, - "map50": 0.7530164274037385, - "map75": 0.6684965691185336 - }, - "iou_thresholds": [ - 0.5, - 0.55, - 0.6, - 0.65, - 0.7, - 0.75, - 0.8, - 0.85, - 0.8999999999999999, - 0.95 - ] - }, - { - "metadata": { - "model": "yolo_nas_s", - "param_count": 19053888, - "run_date": "2024-09-11T14:55:31.676004+00:00" - }, - "map50_95": 0.43622945288618536, - "map50": 0.5995001368949364, - "map75": 0.4723807816255251, + "map50_95": 0.4867825440454518, + "map50": 0.6591044524702075, + "map75": 0.5255439550758243, "small_objects": { - "map50_95": 0.17054831954083777, - "map50": 0.2698303872724884, - "map75": 0.18250067396376432 + "map50_95": 0.22517464073284388, + "map50": 0.36161616082579096, + "map75": 0.23804608448553247 }, "medium_objects": { - "map50_95": 0.3963692663207268, - "map50": 0.5649695419910257, - "map75": 0.4383779405933604 + "map50_95": 0.4361924560564629, + "map50": 0.6090810274677496, + "map75": 0.48462339743332006 }, "large_objects": { - "map50_95": 0.580714409729971, - "map50": 0.723936892681672, - "map75": 0.6297402510946248 + "map50_95": 0.6273765615214366, + "map50": 0.7658607828593057, + "map75": 0.6792628879214015 }, "iou_thresholds": [ 0.5, From 98ae41c8292f8684986d79232fe962a244942b26 Mon Sep 17 00:00:00 2001 From: quangdungluong Date: Thu, 12 Sep 2024 08:20:29 +0700 Subject: [PATCH 3/6] feat: update LICENSE for yolo-nas --- models/object_detection/yolo-nas/results_yolo_nas_l.json | 1 + models/object_detection/yolo-nas/results_yolo_nas_m.json | 1 + models/object_detection/yolo-nas/results_yolo_nas_s.json | 1 + models/object_detection/yolo-nas/run.py | 7 ++++++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/models/object_detection/yolo-nas/results_yolo_nas_l.json b/models/object_detection/yolo-nas/results_yolo_nas_l.json index 725376d..6084850 100644 --- a/models/object_detection/yolo-nas/results_yolo_nas_l.json +++ b/models/object_detection/yolo-nas/results_yolo_nas_l.json @@ -1,6 +1,7 @@ { "metadata": { "model": "yolo_nas_l", + "license": "Apache-2.0", "param_count": 66976392, "run_date": "2024-09-11T15:34:44.524051+00:00" }, diff --git a/models/object_detection/yolo-nas/results_yolo_nas_m.json b/models/object_detection/yolo-nas/results_yolo_nas_m.json index 793af05..f3b9e95 100644 --- a/models/object_detection/yolo-nas/results_yolo_nas_m.json +++ b/models/object_detection/yolo-nas/results_yolo_nas_m.json @@ -1,6 +1,7 @@ { "metadata": { "model": "yolo_nas_m", + "license": "Apache-2.0", "param_count": 51182658, "run_date": "2024-09-11T15:13:41.408248+00:00" }, diff --git a/models/object_detection/yolo-nas/results_yolo_nas_s.json b/models/object_detection/yolo-nas/results_yolo_nas_s.json index fc9b902..1f52015 100644 --- a/models/object_detection/yolo-nas/results_yolo_nas_s.json +++ b/models/object_detection/yolo-nas/results_yolo_nas_s.json @@ -1,6 +1,7 @@ { "metadata": { "model": "yolo_nas_s", + "license": "Apache-2.0", "param_count": 19053888, "run_date": "2024-09-11T14:55:31.676004+00:00" }, diff --git a/models/object_detection/yolo-nas/run.py b/models/object_detection/yolo-nas/run.py index 16f785a..1ab9efe 100644 --- a/models/object_detection/yolo-nas/run.py +++ b/models/object_detection/yolo-nas/run.py @@ -17,6 +17,7 @@ write_result_json, ) +LICENSE = "Apache-2.0" MODEL_IDS = ["yolo_nas_s", "yolo_nas_m", "yolo_nas_l"] DATASET_DIR = "../../../data/coco-val-2017" CONFIDENCE_THRESHOLD = 0.001 @@ -76,7 +77,11 @@ def run( mAP_result = mAP_metric.update(predictions, targets).compute() write_result_json( - model_id=model_id, model_name=model_id, model=model, mAP_result=mAP_result + model_id=model_id, + model_name=model_id, + model=model, + mAP_result=mAP_result, + license_name=LICENSE, ) From d81c77f6a9cc795b5e5d5f7680eee281aeb86259 Mon Sep 17 00:00:00 2001 From: LinasKo Date: Thu, 12 Sep 2024 12:49:44 +0300 Subject: [PATCH 4/6] Rename YOLO-NAS results, temporarily prevent from appearing on leaderboard --- build_static_site.py | 2 +- models/object_detection/yolo-nas/run.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/build_static_site.py b/build_static_site.py index de5cca1..3770f30 100644 --- a/build_static_site.py +++ b/build_static_site.py @@ -2,7 +2,7 @@ from pathlib import Path # Temporarily disabled from appearing on the board, e.g. if there's still some issues -BLACKLIST = ["yolov9"] +BLACKLIST = ["yolov9", "yolo-nas"] results_list = [] for model_dir in Path("models/object_detection").iterdir(): diff --git a/models/object_detection/yolo-nas/run.py b/models/object_detection/yolo-nas/run.py index 1ab9efe..b7ffbf6 100644 --- a/models/object_detection/yolo-nas/run.py +++ b/models/object_detection/yolo-nas/run.py @@ -18,7 +18,17 @@ ) LICENSE = "Apache-2.0" -MODEL_IDS = ["yolo_nas_s", "yolo_nas_m", "yolo_nas_l"] +MODEL_DICT = { + "yolo_nas_s": { + "name": "YOLO-NAS S", + }, + "yolo_nas_m": { + "name": "YOLO-NAS M", + }, + "yolo_nas_l": { + "name": "YOLO-NAS L", + }, +} DATASET_DIR = "../../../data/coco-val-2017" CONFIDENCE_THRESHOLD = 0.001 @@ -48,10 +58,11 @@ def run( dataset: If provided, use this dataset for evaluation. Otherwise, load the dataset from the default directory. """ # noqa: E501 // docs if not model_ids: - model_ids = MODEL_IDS + model_ids = list(MODEL_DICT.keys()) for model_id in model_ids: print(f"\nEvaluating model: {model_id}") + model_values = MODEL_DICT[model_id] if skip_if_result_exists and result_json_already_exists(model_id): print(f"Skipping {model_id}. Result already exists!") @@ -78,7 +89,7 @@ def run( write_result_json( model_id=model_id, - model_name=model_id, + model_name=model_values["name"], model=model, mAP_result=mAP_result, license_name=LICENSE, From 3183c3a56d0b7201d172041daec996b51c70a65e Mon Sep 17 00:00:00 2001 From: LinasKo Date: Thu, 12 Sep 2024 13:15:18 +0300 Subject: [PATCH 5/6] requirements.txt: increase supervision requirement, remove tqdm pin --- models/object_detection/yolo-nas/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/object_detection/yolo-nas/requirements.txt b/models/object_detection/yolo-nas/requirements.txt index 51e8e3a..0d37b7e 100644 --- a/models/object_detection/yolo-nas/requirements.txt +++ b/models/object_detection/yolo-nas/requirements.txt @@ -1,4 +1,4 @@ super_gradients==3.7.1 -supervision==0.23.0 +supervision>=0.24.0rc1 torch==2.2.2 -tqdm==4.66.4 +tqdm From d95299972ca3c030663ee5f6ffc210045db19471 Mon Sep 17 00:00:00 2001 From: quangdungluong Date: Thu, 12 Sep 2024 22:33:45 +0700 Subject: [PATCH 6/6] feat: update model params and rerun results --- .../yolo-nas/results_yolo_nas_l.json | 28 +++++++++---------- .../yolo-nas/results_yolo_nas_m.json | 28 +++++++++---------- .../yolo-nas/results_yolo_nas_s.json | 28 +++++++++---------- models/object_detection/yolo-nas/run.py | 6 ++-- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/models/object_detection/yolo-nas/results_yolo_nas_l.json b/models/object_detection/yolo-nas/results_yolo_nas_l.json index 6084850..4f4f643 100644 --- a/models/object_detection/yolo-nas/results_yolo_nas_l.json +++ b/models/object_detection/yolo-nas/results_yolo_nas_l.json @@ -1,27 +1,27 @@ { "metadata": { - "model": "yolo_nas_l", + "model": "YOLO-NAS L", "license": "Apache-2.0", "param_count": 66976392, - "run_date": "2024-09-11T15:34:44.524051+00:00" + "run_date": "2024-09-12T15:24:51.884717+00:00" }, - "map50_95": 0.4823908978773399, - "map50": 0.6436635942062905, - "map75": 0.5227631511898091, + "map50_95": 0.48298094432764754, + "map50": 0.6395166289719741, + "map75": 0.5280721995716876, "small_objects": { - "map50_95": 0.20627144126452443, - "map50": 0.3234195270645268, - "map75": 0.22433010779366858 + "map50_95": 0.20388303052343654, + "map50": 0.3176009687794093, + "map75": 0.22339241590855352 }, "medium_objects": { - "map50_95": 0.4504440067740434, - "map50": 0.6198672027100715, - "map75": 0.5028411865713277 + "map50_95": 0.44532990793587546, + "map50": 0.6110120117547352, + "map75": 0.5011779178048194 }, "large_objects": { - "map50_95": 0.6232514892633709, - "map50": 0.7530164274037385, - "map75": 0.6684965691185336 + "map50_95": 0.6240832744737295, + "map50": 0.751358076433659, + "map75": 0.6747371906034554 }, "iou_thresholds": [ 0.5, diff --git a/models/object_detection/yolo-nas/results_yolo_nas_m.json b/models/object_detection/yolo-nas/results_yolo_nas_m.json index f3b9e95..2dee79b 100644 --- a/models/object_detection/yolo-nas/results_yolo_nas_m.json +++ b/models/object_detection/yolo-nas/results_yolo_nas_m.json @@ -1,27 +1,27 @@ { "metadata": { - "model": "yolo_nas_m", + "model": "YOLO-NAS M", "license": "Apache-2.0", "param_count": 51182658, - "run_date": "2024-09-11T15:13:41.408248+00:00" + "run_date": "2024-09-12T15:03:37.531535+00:00" }, - "map50_95": 0.47600776753268037, - "map50": 0.6371598084661176, - "map75": 0.5179628054543638, + "map50_95": 0.4766481832173873, + "map50": 0.632842568125929, + "map75": 0.5229364873051732, "small_objects": { - "map50_95": 0.19989784197863897, - "map50": 0.315776115787245, - "map75": 0.21839721685059119 + "map50_95": 0.198911024541053, + "map50": 0.31075430842955604, + "map75": 0.21979759840215746 }, "medium_objects": { - "map50_95": 0.4439337891094138, - "map50": 0.6124309996092572, - "map75": 0.4995806950166813 + "map50_95": 0.43936196228345464, + "map50": 0.6034758757860981, + "map75": 0.49902495607127173 }, "large_objects": { - "map50_95": 0.6217121959458887, - "map50": 0.7532565846774951, - "map75": 0.6717871346846185 + "map50_95": 0.6237642850164452, + "map50": 0.7525691527759143, + "map75": 0.6784723077498078 }, "iou_thresholds": [ 0.5, diff --git a/models/object_detection/yolo-nas/results_yolo_nas_s.json b/models/object_detection/yolo-nas/results_yolo_nas_s.json index 1f52015..0b17f10 100644 --- a/models/object_detection/yolo-nas/results_yolo_nas_s.json +++ b/models/object_detection/yolo-nas/results_yolo_nas_s.json @@ -1,27 +1,27 @@ { "metadata": { - "model": "yolo_nas_s", + "model": "YOLO-NAS S", "license": "Apache-2.0", "param_count": 19053888, - "run_date": "2024-09-11T14:55:31.676004+00:00" + "run_date": "2024-09-12T14:45:02.840446+00:00" }, - "map50_95": 0.43622945288618536, - "map50": 0.5995001368949364, - "map75": 0.4723807816255251, + "map50_95": 0.4369354384266927, + "map50": 0.5950857479290619, + "map75": 0.47826919036030613, "small_objects": { - "map50_95": 0.17054831954083777, - "map50": 0.2698303872724884, - "map75": 0.18250067396376432 + "map50_95": 0.1682506254047077, + "map50": 0.2634323893258659, + "map75": 0.18181972444412323 }, "medium_objects": { - "map50_95": 0.3963692663207268, - "map50": 0.5649695419910257, - "map75": 0.4383779405933604 + "map50_95": 0.39388058360539213, + "map50": 0.5587017365344934, + "map75": 0.43967107626652585 }, "large_objects": { - "map50_95": 0.580714409729971, - "map50": 0.723936892681672, - "map75": 0.6297402510946248 + "map50_95": 0.5820850233048388, + "map50": 0.7216236849916268, + "map75": 0.6364020637637353 }, "iou_thresholds": [ 0.5, diff --git a/models/object_detection/yolo-nas/run.py b/models/object_detection/yolo-nas/run.py index b7ffbf6..4bd9e5f 100644 --- a/models/object_detection/yolo-nas/run.py +++ b/models/object_detection/yolo-nas/run.py @@ -35,8 +35,10 @@ def run_on_image(model, image) -> sv.Detections: model_params = dict( - iou=0.6, - conf=0.001, + conf=0.01, + iou=0.7, + nms_top_k=1000, + max_predictions=300, ) result = model.predict(image, **model_params) detections = sv.Detections.from_yolo_nas(result)