Error occurs when trying to train Object Detection with COCO datasets

[FirokOtaku]

I'm trying to train a Object Detection model with COCO dataset. I created a training config train-coco.py following documents here. Most code were copy from document and were replaced original Learner/Dataset/LearnerConfig/... with ObjectDetection ones.

# Define ClassConfig
print('Define ClassConfig')
from rastervision.core.data import ClassConfig

class_config = ClassConfig(
    names=['1', '2'],
    colors=['red', 'green'],
    null_class='1'
)

import albumentations as A
from rastervision.pytorch_learner import ObjectDetectionImageDataset

print('Loading Dataset')

ds = ObjectDetectionImageDataset(
    img_dir='./coco-dataset/images',
    annotation_uri='./coco-dataset/coco.json',
    transform=A.Resize(1024, 1024),
)

# Define model
# Use a light-weight panoptic FPN model with a ResNet-18 backbone.
print('Define model')

import torch

model = torch.hub.load(
    'AdeelH/pytorch-fpn:0.3',
    'make_fpn_resnet',
    name='resnet18',
    fpn_type='panoptic',
    num_classes=len(class_config),
    fpn_channels=128,
    in_channels=3,
    out_size=(256, 256),
    pretrained=True)



# DataConfig – Configure data-related hyper-parameters
print('DataConfig – Configure data-related hyper-parameters')

# from rastervision.pytorch_learner import SemanticSegmentationGeoDataConfig
from rastervision.pytorch_learner import ObjectDetectionDataConfig

data_cfg = ObjectDetectionDataConfig(
    # class_names=class_config.names,
    # class_colors=class_config.colors,
    # num_workers=0, # increase to use multi-processing
)

# SolverConfig – Configure the loss, optimizer, and scheduler(s)
print('SolverConfig – Configure the loss, optimizer, and scheduler(s)')

from rastervision.pytorch_learner import SolverConfig

solver_cfg = SolverConfig(
    batch_sz=8,
    lr=3e-2,
)

# LearnerConfig – Combine DataConfig, SolverConfig (and optionally, ModelConfig)
print('LearnerConfig – Combine DataConfig, SolverConfig (and optionally, ModelConfig)')

# from rastervision.pytorch_learner import SemanticSegmentationLearnerConfig
from rastervision.pytorch_learner import ObjectDetectionLearnerConfig

# learner_cfg = SemanticSegmentationLearnerConfig(data=data_cfg, solver=solver_cfg)
learner_cfg = ObjectDetectionLearnerConfig(data=data_cfg, solver=solver_cfg)

# Initialize Learner
print('Initialize Learner')

# from rastervision.pytorch_learner import SemanticSegmentationLearner
from rastervision.pytorch_learner import ObjectDetectionLearner

learner = ObjectDetectionLearner(
    cfg=learner_cfg,
    output_dir='./train-demo/',
    model=model,
    train_ds=ds,
    valid_ds=ds,
    training=True,
)
learner.log_data_stats()

# Train – Learner.train()
print('Train – Learner.train()')
learner.train(epochs=12)

# Examine predictions
print('Examine predictions')
learner.plot_predictions(split='valid', show=True)

# Save as a model-bundle
print('Save as a model-bundle')
learner.save_model_bundle()

That raises error when instantiate ObjectDetectionLearnerConfig:

(rastervision) t:\workspace-rastervision>python train-coco.py
Define ClassConfig
Loading Dataset
Define model
Using cache found in C:\Users\firok/.cache\torch\hub\AdeelH_pytorch-fpn_0.3
DataConfig – Configure data-related hyper-parameters
SolverConfig – Configure the loss, optimizer, and scheduler(s)
LearnerConfig – Combine DataConfig, SolverConfig (and optionally, ModelConfig)
Traceback (most recent call last):
  File "t:\workspace-rastervision\train-coco.py", line 71, in <module>
    learner_cfg = ObjectDetectionLearnerConfig(data=data_cfg, solver=solver_cfg)
  File "pydantic\main.py", line 341, in pydantic.main.BaseModel.__init__
pydantic.error_wrappers.ValidationError: 2 validation errors for ObjectDetectionLearnerConfig
data -> type_hint
  unexpected value; permitted: 'object_detection_image_data' (type=value_error.const; given=object_detection_data; permitted=('object_detection_image_data',))
data -> type_hint
  unexpected value; permitted: 'object_detection_geo_data' (type=value_error.const; given=object_detection_data; permitted=('object_detection_geo_data',))

I don't quite understand where things went wrong. Am I using wrong APIs?

[AdeelH]

You need to use ObjectDetectionImageDataConfig instead of ObjectDetectionDataConfig.

Also, the model you are using in your code is a semantic segmentation model, not an object detection model. You can use ObjectDetectionModelConfig instead. E.g. just pass model=ObjectDetectionModelConfig(backbone=Backbone.resnet18) to ObjectDetectionLearnerConfig. This will create a FasterRCNN model with a ResNet18 backbone.

[FirokOtaku]

New train-coco.py is here:

train-coco.py

from rastervision.core.data import ClassConfig

class_config = ClassConfig(
    names=['1', '2'],
    colors=['red', 'green'],
    null_class='1'
)

import albumentations as A
from rastervision.pytorch_learner import ObjectDetectionImageDataset

ds = ObjectDetectionImageDataset(
    img_dir='./coco-dataset/images',
    annotation_uri='./coco-dataset/coco.json',
    transform=A.Resize(1024, 1024),
)

from rastervision.pytorch_learner import ObjectDetectionModelConfig
from rastervision.pytorch_learner import Backbone
model = ObjectDetectionModelConfig(backbone=Backbone.resnet18).build(in_channels=3, num_classes=2, img_sz=256)

from rastervision.pytorch_learner import ObjectDetectionImageDataConfig
from rastervision.pytorch_learner import SolverConfig
from rastervision.pytorch_learner import ObjectDetectionLearnerConfig
data_cfg = ObjectDetectionImageDataConfig()
solver_cfg = SolverConfig(batch_sz=8, lr=3e-2)
learner_cfg = ObjectDetectionLearnerConfig(data=data_cfg, solver=solver_cfg)


from rastervision.pytorch_learner import ObjectDetectionLearner
learner = ObjectDetectionLearner(
    cfg=learner_cfg,
    output_dir='./train-demo/',
    model=model,
    train_ds=ds,
    valid_ds=ds,
    training=True,
)

learner.log_data_stats()

learner.train(epochs=12)

learner.plot_predictions(split='valid', show=True)

learner.save_model_bundle()

This time a new error rasied:

An attempt has been made to start a new process before the
current process has finished its bootstrapping phase.

This probably means that you are not using fork to start your
child processes and you have forgotten to use the proper idiom
in the main module:

Through TaskMgr I can see there is a python.exe process with RAM and VRAM allocated on it. But it just freeze and do nothing.

This might be some very basic problem but I have no idea what should I do next...

full log

(rastervision) t:\workspace-rastervision>python train-coco.py
2024-01-11 14:29:22:rastervision.pytorch_learner.learner: INFO - train_ds: 796 items
2024-01-11 14:29:22:rastervision.pytorch_learner.learner: INFO - valid_ds: 796 items
2024-01-11 14:29:22:rastervision.pytorch_learner.learner: INFO - epoch: 0
Training:   0%|                                                                                                                    | 0/100 [00:00<?, ?it/s]2024-01-11 14:29:29:rastervision.pytorch_learner.learner: INFO - train_ds: 796 items
2024-01-11 14:29:29:rastervision.pytorch_learner.learner: INFO - valid_ds: 796 items
2024-01-11 14:29:29:rastervision.pytorch_learner.learner: INFO - epoch: 0
Training:   0%|                                                                                                                    | 0/100 [00:00<?, ?it/s]Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\spawn.py", line 125, in _main
    prepare(preparation_data)
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\spawn.py", line 236, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
    main_content = runpy.run_path(main_path,
  File "C:\Users\firok\.conda\envs\rastervision\lib\runpy.py", line 289, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "C:\Users\firok\.conda\envs\rastervision\lib\runpy.py", line 96, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "C:\Users\firok\.conda\envs\rastervision\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "t:\workspace-rastervision\train-coco.py", line 44, in <module>
    learner.train(epochs=12)
  File "C:\Users\firok\.conda\envs\rastervision\lib\site-packages\rastervision\pytorch_learner\learner.py", line 391, in train
    train_metrics = self.train_epoch(
  File "C:\Users\firok\.conda\envs\rastervision\lib\site-packages\rastervision\pytorch_learner\learner.py", line 409, in train_epoch
    with tqdm(self.train_dl, desc='Training') as bar:
  File "C:\Users\firok\.conda\envs\rastervision\lib\site-packages\tqdm\asyncio.py", line 33, in __init__
    self.iterable_iterator = iter(iterable)
  File "C:\Users\firok\.conda\envs\rastervision\lib\site-packages\torch\utils\data\dataloader.py", line 438, in __iter__
    return self._get_iterator()
  File "C:\Users\firok\.conda\envs\rastervision\lib\site-packages\torch\utils\data\dataloader.py", line 386, in _get_iterator
    return _MultiProcessingDataLoaderIter(self)
  File "C:\Users\firok\.conda\envs\rastervision\lib\site-packages\torch\utils\data\dataloader.py", line 1039, in __init__
    w.start()
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\process.py", line 121, in start
    self._popen = self._Popen(self)
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\context.py", line 336, in _Popen
    return Popen(process_obj)
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\popen_spawn_win32.py", line 45, in __init__
    prep_data = spawn.get_preparation_data(process_obj._name)
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\spawn.py", line 154, in get_preparation_data
    _check_not_importing_main()
  File "C:\Users\firok\.conda\envs\rastervision\lib\multiprocessing\spawn.py", line 134, in _check_not_importing_main
    raise RuntimeError('''
RuntimeError:
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.
Training:   0%|                                                                                                                    | 0/100 [00:00<?, ?it/s]

[AdeelH]

Looks like this is a Windows issue. This seems to be the recommended workaround: https://pytorch.org/docs/stable/notes/windows.html#multiprocessing-error-without-if-clause-protection.