From c3d89386ccfda71d0c1862619c428567d424fb44 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Fri, 6 Oct 2023 19:51:39 -0400 Subject: [PATCH 01/19] adding image training endpoint --- dlp-cli | 2 +- training/training/core/dataset.py | 39 ++++++++++++++++ training/training/routes/image/__init__.py | 5 ++ training/training/routes/image/image.py | 53 ++++++++++++++++++++++ training/training/routes/image/schemas.py | 22 +++++++++ training/training/urls.py | 2 + 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 training/training/routes/image/__init__.py create mode 100644 training/training/routes/image/image.py create mode 100644 training/training/routes/image/schemas.py diff --git a/dlp-cli b/dlp-cli index 2ed81dc17..f1dd624f2 160000 --- a/dlp-cli +++ b/dlp-cli @@ -1 +1 @@ -Subproject commit 2ed81dc1703ae0eeab6417e0f1ac91aaf2dfc133 +Subproject commit f1dd624f2e36c9f0bd0ada4cf5aac3611e566996 diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index f7a5348a9..4dc9b139f 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -12,6 +12,9 @@ import torch from torch.utils.data import Dataset from torch.autograd import Variable +from torchvision import datasets, transforms +from torch.util.data import DataLoader, random_split +from enum import Enum class TrainTestDatasetCreator(ABC): @@ -98,3 +101,39 @@ def getCategoryList(self) -> list[str]: if self._category_list is None: raise Exception("Category list not available") return self._category_list + +class DefaultImageDatasets(Enum): + MNIST = "MNIST" + FASHION_MNIST = "FashionMNIST" + KMNIST = "KMNIST" +class ImageDefaultDatasetCreator(TrainTestDatasetCreator): + + def __init__( + self, + dataset_name:str, + train_tarnsform:None, + test_transform:None, + batch_size: int = 32, + shuffle: bool = True, + ): + if dataset_name not in DefaultImageDatasets.__members__: + raise Exception(f"The {dataset_name} file does not currently exist in our inventory. Please submit a request to the contributors of the repository") + self.train_transform = train_tarnsform or transforms.Compose([transforms.toTensor()]) + self.test_transform = test_transform or transforms.Compose([transforms.toTensor()]) + self.batch_size = batch_size + self.shuffle = shuffle + + self.train_set = datasets.__dict__[dataset_name](root='./backend/image_data_uploads', train=True, download=True, transform=self.train_transform) + self.test_set = datasets.__dict__[dataset_name](root='./backend/image_data_uploads', train=False, download=True, transform=self.test_transform) + + + @classmethod + def fromDefault(cls, dataset_name: str, train_transform=None, test_transform=None, batch_size: int = 32, shuffle: bool = True): + return cls(dataset_name, train_transform, test_transform, batch_size, shuffle) + def createTrainDataset(self) -> DataLoader: + return DataLoader(self.train_set, batch_size = self.batch_size, shuffle = self.shuffle, drop_last=True) + + def createTestDataset(self): + return DataLoader(self.test_set, batch_size = self.batch_size, shuffle = self.shuffle, drop_last=True) + def getCategoryList(self) -> list[str]: + return self.train_set.classes if hasattr(self.train_set, "classes") else [] \ No newline at end of file diff --git a/training/training/routes/image/__init__.py b/training/training/routes/image/__init__.py new file mode 100644 index 000000000..d24687888 --- /dev/null +++ b/training/training/routes/image/__init__.py @@ -0,0 +1,5 @@ +from training.routes.image.image import router + + +def get_image_router(): + return router diff --git a/training/training/routes/image/image.py b/training/training/routes/image/image.py new file mode 100644 index 000000000..a4f6babb8 --- /dev/null +++ b/training/training/routes/image/image.py @@ -0,0 +1,53 @@ +from typing import Literal, Optional +from django.http import HttpRequest +from ninja import Router, Schema +from training.core.criterion import getCriterionHandler +from training.core.dl_model import DLModel +from training.core.datasets import ImageDefaultDatasetCreator +from torch.utils.data import DataLoader +from training.core.optimizer import getOptimizer +from training.core.trainer import ClassificationTrainer +from training.routes.image.schemas import ImageParams +from training.core.authenticator import FirebaseAuth + +router = Router() + +@router.post("", auth=FirebaseAuth()) +def imageTrain(request: HttpRequest, imageParams: ImageParams): + if imageParams.default: + dataCreator = ImageDefaultDatasetCreator.fromDefault( + imageParams.default, imageParams.test_size, imageParams.shuffle + ) + train_loader = DataLoader( + dataCreator.createTrainDataset(), + batch_size=imageParams.batch_size, + shuffle=False, + drop_last=True, + ) + + test_loader = DataLoader( + dataCreator.createTestDataset(), + batch_size=imageParams.batch_size, + shuffle=False, + drop_last=True, + ) + + model = DLModel.fromLayerParamsList(imageParams.user_arch) + optimizer = getOptimizer(model, imageParams.optimizer_name, 0.05) + criterionHandler = getCriterionHandler(imageParams.criterion) + if imageParams.problem_type == "CLASSIFICATION": + trainer = ClassificationTrainer( + train_loader, + test_loader, + model, + optimizer, + criterionHandler, + imageParams.epochs, + dataCreator.getCategoryList(), + ) + for epoch_result in trainer: + print(epoch_result) + print(trainer.labels_last_epoch, trainer.y_pred_last_epoch) + print(trainer.generate_confusion_matrix()) + print(trainer.generate_AUC_ROC_CURVE()) + return trainer.generate_AUC_ROC_CURVE() \ No newline at end of file diff --git a/training/training/routes/image/schemas.py b/training/training/routes/image/schemas.py new file mode 100644 index 000000000..8d2ae92d6 --- /dev/null +++ b/training/training/routes/image/schemas.py @@ -0,0 +1,22 @@ +from typing import Any, Literal, Optional +from ninja import Schema + + +class LayerParams(Schema): + value: str + parameters: list[Any] + + +class ImageParams(Schema): + target: str + features: list[str] + name: str + problem_type: Literal["CLASSIFICATION"] + default: Optional[str] + criterion: str + optimizer_name: str + shuffle: bool + epochs: int + test_size: float + batch_size: int + user_arch: list[LayerParams] diff --git a/training/training/urls.py b/training/training/urls.py index c79e13107..8242b66be 100644 --- a/training/training/urls.py +++ b/training/training/urls.py @@ -21,6 +21,7 @@ from ninja import NinjaAPI, Schema from training.routes.datasets.default import get_default_datasets_router from training.routes.tabular import get_tabular_router +from training.routes.image import get_image_router api = NinjaAPI() @@ -32,6 +33,7 @@ def test(request: HttpRequest): api.add_router("/datasets/default/", get_default_datasets_router()) api.add_router("/tabular", get_tabular_router()) +api.add_router("/image", get_image_router()) urlpatterns = [ path("admin/", admin.site.urls), From 91f56bf867e74c2c753d212f95bd7381914e96e5 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sat, 7 Oct 2023 00:02:05 +0000 Subject: [PATCH 02/19] :art: Format Python code with psf/black --- training/training/core/dataset.py | 70 +++++++++++++++++------ training/training/routes/image/image.py | 3 +- training/training/routes/image/schemas.py | 2 +- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 4dc9b139f..021a2eccd 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -102,38 +102,74 @@ def getCategoryList(self) -> list[str]: raise Exception("Category list not available") return self._category_list + class DefaultImageDatasets(Enum): MNIST = "MNIST" FASHION_MNIST = "FashionMNIST" KMNIST = "KMNIST" -class ImageDefaultDatasetCreator(TrainTestDatasetCreator): + +class ImageDefaultDatasetCreator(TrainTestDatasetCreator): def __init__( - self, - dataset_name:str, - train_tarnsform:None, - test_transform:None, - batch_size: int = 32, + self, + dataset_name: str, + train_tarnsform: None, + test_transform: None, + batch_size: int = 32, shuffle: bool = True, ): if dataset_name not in DefaultImageDatasets.__members__: - raise Exception(f"The {dataset_name} file does not currently exist in our inventory. Please submit a request to the contributors of the repository") - self.train_transform = train_tarnsform or transforms.Compose([transforms.toTensor()]) - self.test_transform = test_transform or transforms.Compose([transforms.toTensor()]) + raise Exception( + f"The {dataset_name} file does not currently exist in our inventory. Please submit a request to the contributors of the repository" + ) + self.train_transform = train_tarnsform or transforms.Compose( + [transforms.toTensor()] + ) + self.test_transform = test_transform or transforms.Compose( + [transforms.toTensor()] + ) self.batch_size = batch_size self.shuffle = shuffle - self.train_set = datasets.__dict__[dataset_name](root='./backend/image_data_uploads', train=True, download=True, transform=self.train_transform) - self.test_set = datasets.__dict__[dataset_name](root='./backend/image_data_uploads', train=False, download=True, transform=self.test_transform) - + self.train_set = datasets.__dict__[dataset_name]( + root="./backend/image_data_uploads", + train=True, + download=True, + transform=self.train_transform, + ) + self.test_set = datasets.__dict__[dataset_name]( + root="./backend/image_data_uploads", + train=False, + download=True, + transform=self.test_transform, + ) @classmethod - def fromDefault(cls, dataset_name: str, train_transform=None, test_transform=None, batch_size: int = 32, shuffle: bool = True): + def fromDefault( + cls, + dataset_name: str, + train_transform=None, + test_transform=None, + batch_size: int = 32, + shuffle: bool = True, + ): return cls(dataset_name, train_transform, test_transform, batch_size, shuffle) + def createTrainDataset(self) -> DataLoader: - return DataLoader(self.train_set, batch_size = self.batch_size, shuffle = self.shuffle, drop_last=True) - + return DataLoader( + self.train_set, + batch_size=self.batch_size, + shuffle=self.shuffle, + drop_last=True, + ) + def createTestDataset(self): - return DataLoader(self.test_set, batch_size = self.batch_size, shuffle = self.shuffle, drop_last=True) + return DataLoader( + self.test_set, + batch_size=self.batch_size, + shuffle=self.shuffle, + drop_last=True, + ) + def getCategoryList(self) -> list[str]: - return self.train_set.classes if hasattr(self.train_set, "classes") else [] \ No newline at end of file + return self.train_set.classes if hasattr(self.train_set, "classes") else [] diff --git a/training/training/routes/image/image.py b/training/training/routes/image/image.py index a4f6babb8..13aae045e 100644 --- a/training/training/routes/image/image.py +++ b/training/training/routes/image/image.py @@ -12,6 +12,7 @@ router = Router() + @router.post("", auth=FirebaseAuth()) def imageTrain(request: HttpRequest, imageParams: ImageParams): if imageParams.default: @@ -50,4 +51,4 @@ def imageTrain(request: HttpRequest, imageParams: ImageParams): print(trainer.labels_last_epoch, trainer.y_pred_last_epoch) print(trainer.generate_confusion_matrix()) print(trainer.generate_AUC_ROC_CURVE()) - return trainer.generate_AUC_ROC_CURVE() \ No newline at end of file + return trainer.generate_AUC_ROC_CURVE() diff --git a/training/training/routes/image/schemas.py b/training/training/routes/image/schemas.py index 8d2ae92d6..12d37c7bc 100644 --- a/training/training/routes/image/schemas.py +++ b/training/training/routes/image/schemas.py @@ -13,7 +13,7 @@ class ImageParams(Schema): name: str problem_type: Literal["CLASSIFICATION"] default: Optional[str] - criterion: str + criterion: str optimizer_name: str shuffle: bool epochs: int From d7ce98db65b3c12811416c19e6a91a3c4d4e6a73 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Fri, 6 Oct 2023 20:38:44 -0400 Subject: [PATCH 03/19] importing image for dl_model.py --- training/training/core/dl_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/training/training/core/dl_model.py b/training/training/core/dl_model.py index d35d0ff17..29c57026a 100644 --- a/training/training/core/dl_model.py +++ b/training/training/core/dl_model.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from training.routes.tabular.schemas import LayerParams +from training.routes.image.schemas import LayerParams class DLModel(nn.Module): From cd8f85048ac6cd7b8a43bee6872ef245018e8ebd Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Fri, 6 Oct 2023 20:47:19 -0400 Subject: [PATCH 04/19] fixing return typing and typos --- training/training/core/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 021a2eccd..af0d3dd5f 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -113,7 +113,7 @@ class ImageDefaultDatasetCreator(TrainTestDatasetCreator): def __init__( self, dataset_name: str, - train_tarnsform: None, + train_transform: None, test_transform: None, batch_size: int = 32, shuffle: bool = True, @@ -122,7 +122,7 @@ def __init__( raise Exception( f"The {dataset_name} file does not currently exist in our inventory. Please submit a request to the contributors of the repository" ) - self.train_transform = train_tarnsform or transforms.Compose( + self.train_transform = train_transform or transforms.Compose( [transforms.toTensor()] ) self.test_transform = test_transform or transforms.Compose( @@ -163,7 +163,7 @@ def createTrainDataset(self) -> DataLoader: drop_last=True, ) - def createTestDataset(self): + def createTestDataset(self) -> DataLoader: return DataLoader( self.test_set, batch_size=self.batch_size, From a5d0c0b48e3712d87de1c30b270c7832fd98af18 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sat, 7 Oct 2023 13:34:14 -0400 Subject: [PATCH 05/19] optimizing imports and adding cifar dataset --- training/training/core/dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index af0d3dd5f..08c27a5d7 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -13,7 +13,7 @@ from torch.utils.data import Dataset from torch.autograd import Variable from torchvision import datasets, transforms -from torch.util.data import DataLoader, random_split +from torch.utils.data import DataLoader from enum import Enum @@ -107,6 +107,7 @@ class DefaultImageDatasets(Enum): MNIST = "MNIST" FASHION_MNIST = "FashionMNIST" KMNIST = "KMNIST" + CIFAR = "CIFAR10" class ImageDefaultDatasetCreator(TrainTestDatasetCreator): From 9e03ea0ffe4a6961da93ff32dc7b7336e21a8998 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sat, 7 Oct 2023 14:58:27 -0400 Subject: [PATCH 06/19] deleting local downloaded data --- training/training/core/dataset.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 08c27a5d7..4efe314a0 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -15,6 +15,8 @@ from torchvision import datasets, transforms from torch.utils.data import DataLoader from enum import Enum +import os +import shutil class TrainTestDatasetCreator(ABC): @@ -123,6 +125,9 @@ def __init__( raise Exception( f"The {dataset_name} file does not currently exist in our inventory. Please submit a request to the contributors of the repository" ) + + self.dataset_dir = './training/image_data_uploads' + self.train_transform = train_transform or transforms.Compose( [transforms.toTensor()] ) @@ -132,14 +137,17 @@ def __init__( self.batch_size = batch_size self.shuffle = shuffle + # Ensure the directory exists + os.makedirs(self.dataset_dir, exist_ok=True) + self.train_set = datasets.__dict__[dataset_name]( - root="./backend/image_data_uploads", + root=self.dataset_dir, train=True, download=True, transform=self.train_transform, ) self.test_set = datasets.__dict__[dataset_name]( - root="./backend/image_data_uploads", + root=self.dataset_dir, train=False, download=True, transform=self.test_transform, @@ -155,22 +163,30 @@ def fromDefault( shuffle: bool = True, ): return cls(dataset_name, train_transform, test_transform, batch_size, shuffle) + + def delete_datasets_from_directory(self): + if os.path.exists(self.dataset_dir): + shutil.rmtree(self.dataset_dir) def createTrainDataset(self) -> DataLoader: - return DataLoader( + train_loader = DataLoader( self.train_set, batch_size=self.batch_size, shuffle=self.shuffle, drop_last=True, ) + self.delete_datasets_from_directory() + return train_loader def createTestDataset(self) -> DataLoader: - return DataLoader( + test_loader = DataLoader( self.test_set, batch_size=self.batch_size, shuffle=self.shuffle, drop_last=True, ) + self.delete_datasets_from_directory() + return test_loader def getCategoryList(self) -> list[str]: return self.train_set.classes if hasattr(self.train_set, "classes") else [] From b3e82b96db6df9cbc426d8ac341961c48d0ab5b8 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 12 Oct 2023 20:16:04 -0400 Subject: [PATCH 07/19] adding return typing --- training/training/core/dataset.py | 2 +- training/training/routes/image/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 4efe314a0..6a28b9c8f 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -161,7 +161,7 @@ def fromDefault( test_transform=None, batch_size: int = 32, shuffle: bool = True, - ): + ) -> "ImageDefaultDatasetCreator": return cls(dataset_name, train_transform, test_transform, batch_size, shuffle) def delete_datasets_from_directory(self): diff --git a/training/training/routes/image/__init__.py b/training/training/routes/image/__init__.py index d24687888..89149b64b 100644 --- a/training/training/routes/image/__init__.py +++ b/training/training/routes/image/__init__.py @@ -1,5 +1,5 @@ from training.routes.image.image import router -def get_image_router(): +def get_image_router() -> router: return router From f9f1edb2dbf96d007ab6a46de0a9a6d469ad68ca Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Fri, 13 Oct 2023 00:23:41 +0000 Subject: [PATCH 08/19] :art: Format Python code with psf/black --- training/training/core/dataset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 6a28b9c8f..d3bdc42fd 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -125,8 +125,8 @@ def __init__( raise Exception( f"The {dataset_name} file does not currently exist in our inventory. Please submit a request to the contributors of the repository" ) - - self.dataset_dir = './training/image_data_uploads' + + self.dataset_dir = "./training/image_data_uploads" self.train_transform = train_transform or transforms.Compose( [transforms.toTensor()] @@ -137,7 +137,7 @@ def __init__( self.batch_size = batch_size self.shuffle = shuffle - # Ensure the directory exists + # Ensure the directory exists os.makedirs(self.dataset_dir, exist_ok=True) self.train_set = datasets.__dict__[dataset_name]( @@ -163,13 +163,13 @@ def fromDefault( shuffle: bool = True, ) -> "ImageDefaultDatasetCreator": return cls(dataset_name, train_transform, test_transform, batch_size, shuffle) - + def delete_datasets_from_directory(self): if os.path.exists(self.dataset_dir): shutil.rmtree(self.dataset_dir) def createTrainDataset(self) -> DataLoader: - train_loader = DataLoader( + train_loader = DataLoader( self.train_set, batch_size=self.batch_size, shuffle=self.shuffle, From ce4351ffc83133c37a45c5415b526e2c77912501 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 12 Oct 2023 20:25:06 -0400 Subject: [PATCH 09/19] fixing comment --- training/training/core/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 6a28b9c8f..8f575b8f8 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -20,7 +20,7 @@ class TrainTestDatasetCreator(ABC): - "Creator that creates train and test PyTorch datasets" + "Creator that creates train and test PyTorch datasets from a given dataset" @abstractmethod def createTrainDataset(self) -> Dataset: From 64645d63e7b9ce824332131df30bfd90baa87277 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 12 Oct 2023 20:34:17 -0400 Subject: [PATCH 10/19] adding dummy keyword arg --- training/training/core/dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 8673633fa..196dcf45c 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -120,6 +120,7 @@ def __init__( test_transform: None, batch_size: int = 32, shuffle: bool = True, + hello: str = "world" ): if dataset_name not in DefaultImageDatasets.__members__: raise Exception( From 5185cdaeca94df61d305d85d06cd3719f8aa25d7 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Fri, 13 Oct 2023 00:35:04 +0000 Subject: [PATCH 11/19] :art: Format Python code with psf/black --- training/training/core/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 196dcf45c..922cc4c16 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -120,7 +120,7 @@ def __init__( test_transform: None, batch_size: int = 32, shuffle: bool = True, - hello: str = "world" + hello: str = "world", ): if dataset_name not in DefaultImageDatasets.__members__: raise Exception( From 30f5dd2efaba49a217be397223c0edbd3dff43a7 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 12 Oct 2023 20:53:12 -0400 Subject: [PATCH 12/19] added logging --- training/training/core/dataset.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index 196dcf45c..c26d30c0f 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -120,7 +120,6 @@ def __init__( test_transform: None, batch_size: int = 32, shuffle: bool = True, - hello: str = "world" ): if dataset_name not in DefaultImageDatasets.__members__: raise Exception( @@ -141,6 +140,7 @@ def __init__( # Ensure the directory exists os.makedirs(self.dataset_dir, exist_ok=True) + # Load the datasets self.train_set = datasets.__dict__[dataset_name]( root=self.dataset_dir, train=True, @@ -167,7 +167,11 @@ def fromDefault( def delete_datasets_from_directory(self): if os.path.exists(self.dataset_dir): - shutil.rmtree(self.dataset_dir) + try: + shutil.rmtree(self.dataset_dir) + print(f"Successfully deleted {self.dataset_dir}") + except Exception as e: + print(f"Failed to delete {self.dataset_dir} with error: {e}") def createTrainDataset(self) -> DataLoader: train_loader = DataLoader( @@ -176,7 +180,7 @@ def createTrainDataset(self) -> DataLoader: shuffle=self.shuffle, drop_last=True, ) - self.delete_datasets_from_directory() + self.delete_datasets_from_directory() # Delete datasets after loading return train_loader def createTestDataset(self) -> DataLoader: @@ -186,7 +190,7 @@ def createTestDataset(self) -> DataLoader: shuffle=self.shuffle, drop_last=True, ) - self.delete_datasets_from_directory() + self.delete_datasets_from_directory() # Delete datasets after loading return test_loader def getCategoryList(self) -> list[str]: From 5cbbaf06e960ac144d085fcaef4149bc6c467c40 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 12 Oct 2023 22:54:29 -0400 Subject: [PATCH 13/19] testing endpoint --- training/training/core/dataset.py | 10 +++--- training/training/core/dl_model.py | 2 ++ .../routes/datasets/default/__init__.py | 5 --- training/training/routes/image/__init__.py | 5 --- training/training/routes/image/image.py | 32 +++++++++++-------- training/training/routes/image/schemas.py | 4 +-- training/training/urls.py | 16 ++++++---- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index c26d30c0f..fa1bf6259 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -127,20 +127,22 @@ def __init__( ) self.dataset_dir = "./training/image_data_uploads" - self.train_transform = train_transform or transforms.Compose( - [transforms.toTensor()] + [transforms.ToTensor()] ) + self.test_transform = test_transform or transforms.Compose( - [transforms.toTensor()] + [transforms.ToTensor()] ) self.batch_size = batch_size self.shuffle = shuffle # Ensure the directory exists os.makedirs(self.dataset_dir, exist_ok=True) - + print(f'train transform: {train_transform}') + print(f'test transform: {test_transform}') # Load the datasets + self.train_set = datasets.__dict__[dataset_name]( root=self.dataset_dir, train=True, diff --git a/training/training/core/dl_model.py b/training/training/core/dl_model.py index 29c57026a..eb9625326 100644 --- a/training/training/core/dl_model.py +++ b/training/training/core/dl_model.py @@ -16,6 +16,8 @@ class DLModel(nn.Module): "SOFTMAX": nn.Softmax, "SIGMOID": nn.Sigmoid, "LOGSOFTMAX": nn.LogSoftmax, + "CONV2D": nn.Conv2d, + "DROPOUT": nn.Dropout } def __init__(self, layer_list: list[nn.Module]): diff --git a/training/training/routes/datasets/default/__init__.py b/training/training/routes/datasets/default/__init__.py index 2a6f5e11b..e69de29bb 100644 --- a/training/training/routes/datasets/default/__init__.py +++ b/training/training/routes/datasets/default/__init__.py @@ -1,5 +0,0 @@ -from training.routes.datasets.default.columns import router - - -def get_default_datasets_router(): - return router diff --git a/training/training/routes/image/__init__.py b/training/training/routes/image/__init__.py index 89149b64b..e69de29bb 100644 --- a/training/training/routes/image/__init__.py +++ b/training/training/routes/image/__init__.py @@ -1,5 +0,0 @@ -from training.routes.image.image import router - - -def get_image_router() -> router: - return router diff --git a/training/training/routes/image/image.py b/training/training/routes/image/image.py index 13aae045e..0e74a01d5 100644 --- a/training/training/routes/image/image.py +++ b/training/training/routes/image/image.py @@ -3,7 +3,7 @@ from ninja import Router, Schema from training.core.criterion import getCriterionHandler from training.core.dl_model import DLModel -from training.core.datasets import ImageDefaultDatasetCreator +from training.core.dataset import ImageDefaultDatasetCreator from torch.utils.data import DataLoader from training.core.optimizer import getOptimizer from training.core.trainer import ClassificationTrainer @@ -17,23 +17,27 @@ def imageTrain(request: HttpRequest, imageParams: ImageParams): if imageParams.default: dataCreator = ImageDefaultDatasetCreator.fromDefault( - imageParams.default, imageParams.test_size, imageParams.shuffle - ) - train_loader = DataLoader( - dataCreator.createTrainDataset(), - batch_size=imageParams.batch_size, - shuffle=False, - drop_last=True, + imageParams.default ) + print(vars(dataCreator)) + train_loader = dataCreator.createTrainDataset() + test_loader = dataCreator.createTestDataset() + # train_loader = DataLoader( + # dataCreator.createTrainDataset(), + # batch_size=imageParams.batch_size, + # shuffle=False, + # drop_last=True, + # ) - test_loader = DataLoader( - dataCreator.createTestDataset(), - batch_size=imageParams.batch_size, - shuffle=False, - drop_last=True, - ) + # test_loader = DataLoader( + # dataCreator.createTestDataset(), + # batch_size=imageParams.batch_size, + # shuffle=False, + # drop_last=True, + # ) model = DLModel.fromLayerParamsList(imageParams.user_arch) + print(f'model is: {model}') optimizer = getOptimizer(model, imageParams.optimizer_name, 0.05) criterionHandler = getCriterionHandler(imageParams.criterion) if imageParams.problem_type == "CLASSIFICATION": diff --git a/training/training/routes/image/schemas.py b/training/training/routes/image/schemas.py index 12d37c7bc..1a07b518a 100644 --- a/training/training/routes/image/schemas.py +++ b/training/training/routes/image/schemas.py @@ -8,8 +8,8 @@ class LayerParams(Schema): class ImageParams(Schema): - target: str - features: list[str] + # target: str + # features: list[str] name: str problem_type: Literal["CLASSIFICATION"] default: Optional[str] diff --git a/training/training/urls.py b/training/training/urls.py index 8242b66be..2e6e6c048 100644 --- a/training/training/urls.py +++ b/training/training/urls.py @@ -19,9 +19,13 @@ from django.http import HttpRequest from django.urls import path from ninja import NinjaAPI, Schema -from training.routes.datasets.default import get_default_datasets_router -from training.routes.tabular import get_tabular_router -from training.routes.image import get_image_router + +from training.routes.datasets.default.columns import router as default_dataset_router +from training.routes.tabular.tabular import router as tabular_router +from training.routes.image.image import router as image_router +# from training.routes.datasets.default import get_default_datasets_router +# from training.routes.tabular import get_tabular_router +# from training.routes.image import image_router api = NinjaAPI() @@ -31,9 +35,9 @@ def test(request: HttpRequest): return 200, {"result": "200 Backend surface test successful"} -api.add_router("/datasets/default/", get_default_datasets_router()) -api.add_router("/tabular", get_tabular_router()) -api.add_router("/image", get_image_router()) +api.add_router("/datasets/default/", default_dataset_router) +api.add_router("/tabular", tabular_router) +api.add_router("/image", image_router) urlpatterns = [ path("admin/", admin.site.urls), From 6a35554d81c88dfc4f43a512acddd3f56094ca26 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 26 Oct 2023 21:44:41 -0400 Subject: [PATCH 14/19] adding maxpool2d and flatten --- training/training/core/dl_model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/training/training/core/dl_model.py b/training/training/core/dl_model.py index eb9625326..3ff846a8c 100644 --- a/training/training/core/dl_model.py +++ b/training/training/core/dl_model.py @@ -17,7 +17,9 @@ class DLModel(nn.Module): "SIGMOID": nn.Sigmoid, "LOGSOFTMAX": nn.LogSoftmax, "CONV2D": nn.Conv2d, - "DROPOUT": nn.Dropout + "DROPOUT": nn.Dropout, + "MAXPOOL2D": nn.MaxPool2d, + "FLATTEN": nn.Flatten } def __init__(self, layer_list: list[nn.Module]): From 2922549ba05e28bd3982c42c29dd107027d30f32 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Thu, 26 Oct 2023 22:46:58 -0400 Subject: [PATCH 15/19] updating criterion --- training/training/core/criterion.py | 2 +- training/training/core/trainer.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/training/training/core/criterion.py b/training/training/core/criterion.py index 781f46f2a..dac0100b0 100644 --- a/training/training/core/criterion.py +++ b/training/training/core/criterion.py @@ -46,7 +46,7 @@ class CELossHandler(CriterionHandler): def _compute_loss(self, output, labels): output = torch.reshape( output, - (output.shape[0], output.shape[2]), + (output.shape[0], output.shape[-1]), ) labels = labels.squeeze_() return nn.CrossEntropyLoss(reduction="mean")(output, labels.long()) diff --git a/training/training/core/trainer.py b/training/training/core/trainer.py index 02ce1567f..9682835a5 100644 --- a/training/training/core/trainer.py +++ b/training/training/core/trainer.py @@ -62,6 +62,10 @@ def _train_step(self, inputs: torch.Tensor, labels: torch.Tensor): self.optimizer.zero_grad() # zero out gradient for each batch self.model.forward(inputs) # make prediction on input self._outputs: torch.Tensor = self.model(inputs) # make prediction on input + print('MODEL FORWARD PASS DONE!!!!') + print(f'output dim: {self._outputs.shape}') + print(f'label dim: {labels.shape}') + print(f'loss function used: {self.criterionHandler}') loss = self.criterionHandler.compute_loss(self._outputs, labels) loss.backward() # backpropagation self.optimizer.step() # adjust optimizer weights @@ -200,6 +204,8 @@ def _train_init(self): super()._train_init() def _train_step(self, inputs: torch.Tensor, labels: torch.Tensor): + print(f'input shape: {inputs.shape}') + print(f'labels shape: {labels.shape}') super()._train_step(inputs, labels) self._train_correct += self.compute_correct(self._outputs, labels) From dbc97d1e50eb02925d2af486fdaff821a32774d2 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sat, 18 Nov 2023 16:30:44 -0800 Subject: [PATCH 16/19] removing print statements --- training/training/core/trainer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/training/training/core/trainer.py b/training/training/core/trainer.py index 9682835a5..6f33102b6 100644 --- a/training/training/core/trainer.py +++ b/training/training/core/trainer.py @@ -204,8 +204,6 @@ def _train_init(self): super()._train_init() def _train_step(self, inputs: torch.Tensor, labels: torch.Tensor): - print(f'input shape: {inputs.shape}') - print(f'labels shape: {labels.shape}') super()._train_step(inputs, labels) self._train_correct += self.compute_correct(self._outputs, labels) From 530888445a74e6778c5efc28ed70625bda8ec134 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sun, 19 Nov 2023 12:47:12 -0800 Subject: [PATCH 17/19] fixing cifar typo --- training/training/core/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/training/core/dataset.py b/training/training/core/dataset.py index fa1bf6259..520f94dc6 100644 --- a/training/training/core/dataset.py +++ b/training/training/core/dataset.py @@ -109,7 +109,7 @@ class DefaultImageDatasets(Enum): MNIST = "MNIST" FASHION_MNIST = "FashionMNIST" KMNIST = "KMNIST" - CIFAR = "CIFAR10" + CIFAR10 = "CIFAR10" class ImageDefaultDatasetCreator(TrainTestDatasetCreator): From 3af1d654a515efaee963693e4fbb37d95980e872 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sun, 19 Nov 2023 14:00:17 -0800 Subject: [PATCH 18/19] removed unused code --- training/training/urls.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/training/training/urls.py b/training/training/urls.py index 2e6e6c048..0a120abb3 100644 --- a/training/training/urls.py +++ b/training/training/urls.py @@ -23,9 +23,6 @@ from training.routes.datasets.default.columns import router as default_dataset_router from training.routes.tabular.tabular import router as tabular_router from training.routes.image.image import router as image_router -# from training.routes.datasets.default import get_default_datasets_router -# from training.routes.tabular import get_tabular_router -# from training.routes.image import image_router api = NinjaAPI() From f2d6f5b596cef51119f14191639bdb0c333545f2 Mon Sep 17 00:00:00 2001 From: codingwithsurya Date: Sun, 19 Nov 2023 14:10:16 -0800 Subject: [PATCH 19/19] solving nits --- training/training/core/trainer.py | 4 ---- training/training/routes/image/image.py | 14 -------------- 2 files changed, 18 deletions(-) diff --git a/training/training/core/trainer.py b/training/training/core/trainer.py index 6f33102b6..02ce1567f 100644 --- a/training/training/core/trainer.py +++ b/training/training/core/trainer.py @@ -62,10 +62,6 @@ def _train_step(self, inputs: torch.Tensor, labels: torch.Tensor): self.optimizer.zero_grad() # zero out gradient for each batch self.model.forward(inputs) # make prediction on input self._outputs: torch.Tensor = self.model(inputs) # make prediction on input - print('MODEL FORWARD PASS DONE!!!!') - print(f'output dim: {self._outputs.shape}') - print(f'label dim: {labels.shape}') - print(f'loss function used: {self.criterionHandler}') loss = self.criterionHandler.compute_loss(self._outputs, labels) loss.backward() # backpropagation self.optimizer.step() # adjust optimizer weights diff --git a/training/training/routes/image/image.py b/training/training/routes/image/image.py index 0e74a01d5..65e36e762 100644 --- a/training/training/routes/image/image.py +++ b/training/training/routes/image/image.py @@ -22,20 +22,6 @@ def imageTrain(request: HttpRequest, imageParams: ImageParams): print(vars(dataCreator)) train_loader = dataCreator.createTrainDataset() test_loader = dataCreator.createTestDataset() - # train_loader = DataLoader( - # dataCreator.createTrainDataset(), - # batch_size=imageParams.batch_size, - # shuffle=False, - # drop_last=True, - # ) - - # test_loader = DataLoader( - # dataCreator.createTestDataset(), - # batch_size=imageParams.batch_size, - # shuffle=False, - # drop_last=True, - # ) - model = DLModel.fromLayerParamsList(imageParams.user_arch) print(f'model is: {model}') optimizer = getOptimizer(model, imageParams.optimizer_name, 0.05)