Evaluation_ResNet18_3D_videos.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from distutils import util\n",
    "\n",
    "from temporal_transform import TemporalRandomCrop\n",
    "from resnet3d import generate_model\n",
    "from ucf101 import get_ucf_dataset\n",
    "from hmdb51 import get_hmdb_dataset\n",
    "from logger import Logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = Logger(show = True, html_output = True, config_file = \"config.txt\")\n",
    "\n",
    "torch.manual_seed(1)\n",
    "random.seed(1)\n",
    "\n",
    "frame_size = logger.config_dict['frame_resize']\n",
    "spatial_transform = transforms.Compose([transforms.Resize((frame_size, frame_size)), transforms.ToTensor()])\n",
    "logger.log(\"Frames resized to {}x{}\".format(frame_size, frame_size))\n",
    "\n",
    "sampling_method_str = logger.config_dict['sampling_method']\n",
    "if \"rand\" in sampling_method_str:\n",
    "    crop_size = int(sampling_method_str.replace(\"rand\", \"\"))\n",
    "    sampling_method = TemporalRandomCrop(size = crop_size)\n",
    "    logger.log(\"Sampling strategy selecting {} consecutives frames from a random index\".format(\n",
    "      crop_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_path = os.path.join(logger.data_folder, \n",
    "                          logger.config_dict['video_folder'], logger.config_dict['frame_folder'])\n",
    "annotation_path = logger.get_data_file(logger.config_dict['annotation_file'])\n",
    "batch_size  = logger.config_dict['batch_size']\n",
    "num_workers = logger.config_dict['num_workers']\n",
    "dataset_type = logger.config_dict['dataset_type']\n",
    "\n",
    "if dataset_type == \"ucf101\":\n",
    "    train_dataset = get_ucf_dataset(video_path, annotation_path, \"training\", sampling_method, \n",
    "                                    spatial_transform, temporal_transform = \"None\",\n",
    "                                    stack_clip = True, is_simclr_transform = False, \n",
    "                                    apply_same_per_clip = True)\n",
    "    test_dataset = get_ucf_dataset(video_path, annotation_path, \"validation\", sampling_method, \n",
    "                                   spatial_transform, temporal_transform = \"None\",\n",
    "                                   stack_clip = True, is_simclr_transform = False, \n",
    "                                   apply_same_per_clip = True)\n",
    "elif dataset_type == \"hmdb51\":\n",
    "    train_dataset = get_hmdb_dataset(video_path, annotation_path, \"training\", sampling_method, \n",
    "                                     spatial_transform, temporal_transform = \"None\",\n",
    "                                     stack_clip = True, is_simclr_transform = False, \n",
    "                                     apply_same_per_clip = True)\n",
    "    test_dataset = get_hmdb_dataset(video_path, annotation_path, \"validation\", sampling_method, \n",
    "                                    spatial_transform, temporal_transform = \"None\",\n",
    "                                    stack_clip = True, is_simclr_transform = False, \n",
    "                                    apply_same_per_clip = True)\n",
    "\n",
    "train_loader  = DataLoader(train_dataset, batch_size = batch_size, shuffle = False, drop_last = False, \n",
    "                           num_workers = num_workers)\n",
    "logger.log(\"{} Train data loaded with {} clips\".format(dataset_type.upper(), len(train_dataset)))\n",
    "\n",
    "test_loader = DataLoader(test_dataset, batch_size = batch_size, shuffle = False, drop_last = False,\n",
    "                         num_workers = num_workers)\n",
    "logger.log(\"{} Test data loaded with {} clips\".format(dataset_type.upper(), len(test_dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = logger.config_dict['base_convnet']\n",
    "output_dim = logger.config_dict['simclr_out_dim']\n",
    "model = generate_model(model_depth = 18, add_projection_layers = True, projection_dim = output_dim)\n",
    "logger.log(\"Model {} 3D simCLR loaded {}\".format(base_model, model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_epoch = logger.config_dict['model_checkpoint_epoch']\n",
    "checkpoint_file  = logger.config_dict['model_checkpoint_file']\n",
    "\n",
    "use_kinet = bool(util.strtobool(logger.config_dict['use_kinet']))\n",
    "\n",
    "if use_kinet:\n",
    "    logger.log(\"Loading kinetics pretrained model weights at epoch {} from {}\".format(\n",
    "      checkpoint_epoch, checkpoint_file))\n",
    "    checkpoint = torch.load(logger.get_model_file(checkpoint_file))\n",
    "    msg = model.load_state_dict(checkpoint['state_dict'], strict = False)\n",
    "    assert set(msg.missing_keys) == {\"fc1.weight\", \"fc1.bias\", \"fc2.weight\", \"fc2.bias\"}\n",
    "    checkpoint_epoch = 0\n",
    "else:    \n",
    "    training_batch_size = int(checkpoint_file.split(\"_\")[1].replace(\"b\", \"\"))\n",
    "    logger.log(\"Loading simCLR model weights at epoch {} with {} training batchsize from {}\".format(\n",
    "        checkpoint_epoch, training_batch_size, checkpoint_file))\n",
    "    checkpoint = torch.load(logger.get_model_file(checkpoint_file))\n",
    "\n",
    "    try:\n",
    "        model.load_state_dict(checkpoint['model'])\n",
    "    except:\n",
    "        model = nn.DataParallel(model)\n",
    "        model.load_state_dict(checkpoint['model'])\n",
    "\n",
    "    temporal_transform_str = logger.config_dict['temporal_transform_type']\n",
    "    step = logger.config_dict['temporal_transform_step']\n",
    "\n",
    "    if temporal_transform_str != \"None\":\n",
    "        logger.log(\"Using {} as temporal transform with step {}\".format(temporal_transform_str, step))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_avail = torch.cuda.is_available()\n",
    "logger.log(\"GPU available: {}\".format(gpu_avail))\n",
    "if gpu_avail:\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.eval()\n",
    "logger.log(\"Generating SimCLR features from training data ...\")\n",
    "X_train_feature = []\n",
    "y_train = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for i, (inputs, labels) in tqdm(enumerate(train_loader)):\n",
    "\n",
    "        #print(inputs.size())\n",
    "\n",
    "        if gpu_avail:\n",
    "            inputs = inputs.cuda()\n",
    "\n",
    "        features, _ = model(inputs)\n",
    "        X_train_feature.extend(features.detach().cpu().numpy())\n",
    "        y_train.extend(labels.numpy())\n",
    "    \n",
    "X_train_feature = np.array(X_train_feature)\n",
    "logger.log(\"Train features of shape: {}\".format(X_train_feature.shape), show_time = True)\n",
    "\n",
    "'''\n",
    "train_feats_filename  = base_model + \"_\" + str(training_batch_size) + \"b_\" + str(frame_size) + \"f_\" + str(temporal_shift)\n",
    "train_feats_filename += \"s_\" + \"x_train_feats\" + \"_\" + dataset_type + \"_\" + time_crop_type + \"_3d.npy\"\n",
    "\n",
    "train_feats_filename = logger.get_output_file(train_feats_filename)\n",
    "logger.log(\"Saving training data simCLR features at {}\".format(train_feats_filename))\n",
    "np.save(train_feats_filename, X_train_feature)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.eval()\n",
    "logger.log(\"Generating SimCLR features from testing data ...\")\n",
    "X_test_feature = []\n",
    "y_test = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for i, (inputs, labels) in tqdm(enumerate(test_loader)):\n",
    "        if gpu_avail:\n",
    "            inputs = inputs.cuda()\n",
    "\n",
    "        features, _ = model(inputs)\n",
    "        X_test_feature.extend(features.detach().cpu().numpy())\n",
    "        y_test.extend(labels.numpy())\n",
    "\n",
    "X_test_feature = np.array(X_test_feature)\n",
    "logger.log(\"Test features of shape: {}\".format(X_test_feature.shape), show_time = True)\n",
    "\n",
    "'''\n",
    "test_feats_filename  = base_model + \"_\" + str(training_batch_size) + \"b_\" + str(frame_size) + \"f_\" + str(temporal_shift) \n",
    "test_feats_filename += \"s_\" + \"x_test_feats\" + \"_\" + dataset_type + \"_\" + time_crop_type + \"_3d.npy\"\n",
    "\n",
    "test_feats_filename = logger.get_output_file(test_feats_filename)\n",
    "logger.log(\"Saving testing data simCLR features at {}\".format(test_feats_filename))\n",
    "np.save(test_feats_filename, X_test_feature)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import preprocessing\n",
    "\n",
    "clf = LogisticRegression(random_state=0, max_iter=2000, solver='lbfgs', C=1.0, n_jobs = -1)\n",
    "scaler = preprocessing.StandardScaler()\n",
    "scaler.fit(X_train_feature)\n",
    "\n",
    "logger.log(\"Start training LogisticRegression on SimCLR features ...\")\n",
    "clf.fit(scaler.transform(X_train_feature), y_train)\n",
    "logger.log(\"Finished training\", show_time = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_batch_size = None\n",
    "logger.log(\"SimCLR with {} batchsize feature evaluation on {}\".format(training_batch_size, dataset_type))\n",
    "logger.log(\"Train score: {:.4f}\".format(clf.score(scaler.transform(X_train_feature), y_train)))\n",
    "logger.log(\"Test score: {:.4f}\".format(clf.score(scaler.transform(X_test_feature), y_test)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}