get-system-info.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/fastai/courses/dl2_2017\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "fastai_root_path = '~/clouderizer/fast.ai'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data  fastai  out\r\n"
     ]
    }
   ],
   "source": [
    "!ls {fastai_root_path}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Clouderizer's Workspace Data Dir:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dogscats  dogscats.zip\r\n"
     ]
    }
   ],
   "source": [
    "!ls {fastai_root_path}/data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset for dl1 cours dir:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models\tsample\ttest1  train  valid\r\n"
     ]
    }
   ],
   "source": [
    "!ls data/dogscats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Clouderizer's Workspace Code Dir (scripts, Jupyter Notebooks):**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CODE-OF-CONDUCT.md\t   environment-old.yml\tMANIFEST.in\t  setup.cfg\r\n",
      "courses\t\t\t   environment.yml\tpytest.ini\t  setup.py\r\n",
      "docs\t\t\t   fastai\t\tREADME.md\t  tests\r\n",
      "environment-cpu.yml\t   LICENSE\t\tREADME.rst\t  tutorials\r\n",
      "environment-nopytorch.yml  MANIFEST\t\trequirements.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls {fastai_root_path}/fastai"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Clouderizer's Workspace Output Dir (project output like TensorBoard event data, etc):**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls {fastai_root_path}/out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\t     normalization_mode2.py  pytorch-tut.ipynb\tutils2.py\r\n",
      "DCGAN.ipynb  __pycache__\t     torch_utils.py\twgan-pytorch.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls ~/clouderizer/fast.ai/fastai/courses/dl1/dl2_2017/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check System Info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CPU and GPU Utilization and Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gputil\n",
      "  Downloading https://files.pythonhosted.org/packages/45/99/837428d26b47ebd6b66d6e1b180e98ec4a557767a93a81a02ea9d6242611/GPUtil-1.3.0.tar.gz\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from gputil) (1.14.3)\n",
      "Building wheels for collected packages: gputil\n",
      "  Running setup.py bdist_wheel for gputil ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /content/.cache/pip/wheels/17/0f/04/b79c006972335e35472c0b835ed52bfc0815258d409f560108\n",
      "Successfully built gputil\n",
      "\u001b[31mplotnine 0.3.0 has requirement scipy>=1.0.0, but you'll have scipy 0.19.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mkaggle-cli 0.12.13 has requirement lxml<4.1,>=4.0.0, but you'll have lxml 4.2.1 which is incompatible.\u001b[0m\n",
      "Installing collected packages: gputil\n",
      "Successfully installed gputil-1.3.0\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (5.4.5)\n",
      "\u001b[31mplotnine 0.3.0 has requirement scipy>=1.0.0, but you'll have scipy 0.19.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mkaggle-cli 0.12.13 has requirement lxml<4.1,>=4.0.0, but you'll have lxml 4.2.1 which is incompatible.\u001b[0m\n",
      "Collecting humanize\n",
      "  Downloading https://files.pythonhosted.org/packages/8c/e0/e512e4ac6d091fc990bbe13f9e0378f34cf6eecd1c6c268c9e598dcf5bb9/humanize-0.5.1.tar.gz\n",
      "Building wheels for collected packages: humanize\n",
      "  Running setup.py bdist_wheel for humanize ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /content/.cache/pip/wheels/69/86/6c/f8b8593bc273ec4b0c653d3827f7482bb2001a2781a73b7f44\n",
      "Successfully built humanize\n",
      "\u001b[31mplotnine 0.3.0 has requirement scipy>=1.0.0, but you'll have scipy 0.19.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mkaggle-cli 0.12.13 has requirement lxml<4.1,>=4.0.0, but you'll have lxml 4.2.1 which is incompatible.\u001b[0m\n",
      "Installing collected packages: humanize\n",
      "Successfully installed humanize-0.5.1\n"
     ]
    }
   ],
   "source": [
    "# memory footprint support libraries/code\n",
    "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
    "!pip install gputil\n",
    "!pip install psutil\n",
    "!pip install humanize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import psutil\n",
    "import humanize\n",
    "import os\n",
    "import GPUtil as GPU\n",
    "\n",
    "def printm():\n",
    "    GPUs = GPU.getGPUs()\n",
    "    # XXX: only one GPU on Colab and isn’t guaranteed\n",
    "    gpu = GPUs[0]\n",
    "\n",
    "    process = psutil.Process(os.getpid())\n",
    "    print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" I Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n",
    "    print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gen RAM Free: 12.1 GB  I Proc size: 105.5 MB\n",
      "GPU RAM Free: 11438MB | Used: 1MB | Util   0% | Total 11439MB\n"
     ]
    }
   ],
   "source": [
    "printm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check OS Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linux 1fcfa717dfdd 4.4.111+ #1 SMP Tue Mar 20 23:18:55 PDT 2018 x86_64 x86_64 x86_64 GNU/Linux\r\n"
     ]
    }
   ],
   "source": [
    "!uname -a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LESSOPEN=| /usr/bin/lesspipe %s\r\n",
      "TMUX=/tmp/tmux-1000/default,2312,0\r\n",
      "MAIL=/var/mail/ubuntu\r\n",
      "SSH_CLIENT=103.254.153.99 48482 22\r\n",
      "USER=ubuntu\r\n",
      "MPLBACKEND=module://ipykernel.pylab.backend_inline\r\n",
      "LD_LIBRARY_PATH=/usr/local/cuda-9.1/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-9.1/lib64:/usr/local/cuda/extras/CUPTI/lib64\r\n",
      "SHLVL=3\r\n",
      "HOME=/home/ubuntu\r\n",
      "SSH_TTY=/dev/pts/1\r\n",
      "CONDA_PS1_BACKUP=\\[\\e]0;\\u@\\h: \\w\\a\\]${debian_chroot:+($debian_chroot)}\\u@\\h:\\w\\$ \r\n",
      "PAGER=cat\r\n",
      "LOGNAME=ubuntu\r\n",
      "_=/home/ubuntu/anaconda3/envs/fastai/bin/jupyter\r\n",
      "EVENT_NOEPOLL=1\r\n",
      "XDG_SESSION_ID=4\r\n",
      "TERM=xterm-color\r\n",
      "CONDA_PATH_BACKUP=/home/ubuntu/anaconda3/bin:/usr/local/cuda-9.1/bin:/home/ubuntu/bin:/home/ubuntu/.local/bin:/home/ubuntu/anaconda3/bin:/usr/local/cuda-9.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/snap/bin\r\n",
      "PATH=/home/ubuntu/anaconda3/envs/fastai/bin:/home/ubuntu/bin:/home/ubuntu/.local/bin:/home/ubuntu/anaconda3/envs/fastai/bin:/home/ubuntu/anaconda3/bin:/usr/local/cuda-9.1/bin:/home/ubuntu/bin:/home/ubuntu/.local/bin:/home/ubuntu/anaconda3/bin:/usr/local/cuda-9.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/snap/bin\r\n",
      "XDG_RUNTIME_DIR=/run/user/1000\r\n",
      "LANG=en_US.UTF-8\r\n",
      "LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:\r\n",
      "SHELL=/bin/bash\r\n",
      "CONDA_DEFAULT_ENV=fastai\r\n",
      "LESSCLOSE=/usr/bin/lesspipe %s %s\r\n",
      "GIT_PAGER=cat\r\n",
      "PWD=/home/ubuntu/fastai/courses/dl2_2017\r\n",
      "CLICOLOR=1\r\n",
      "SSH_CONNECTION=103.254.153.99 48482 10.0.0.14 22\r\n",
      "JPY_PARENT_PID=2374\r\n",
      "CONDA_PREFIX=/home/ubuntu/anaconda3/envs/fastai\r\n",
      "TMUX_PANE=%0\r\n"
     ]
    }
   ],
   "source": [
    "!printenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No LSB modules are available.\r\n",
      "Distributor ID:\tUbuntu\r\n",
      "Description:\tUbuntu 16.04.1 LTS\r\n",
      "Release:\t16.04\r\n",
      "Codename:\txenial\r\n"
     ]
    }
   ],
   "source": [
    "!lsb_release -a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls dl2_2017/wgan_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mDEPRECATION: The default format will switch to columns in the future. You can use --format=(legacy|columns) (or define a format=(legacy|columns) in your pip.conf under the [list] section) to disable this warning.\u001b[0m\n",
      "torch (0.3.1.post2)\n",
      "torchtext (0.2.3)\n",
      "torchvision (0.2.1)\n",
      "\u001b[33mYou are using pip version 9.0.3, however version 10.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip list | grep -i \"torch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ls: cannot access 'dl2_2017/data': No such file or directory\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh dl2_2017/data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Process LSUN Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert LSUN LMDB to JPG files using the python scripts provided by them. Their docs: https://github.com/fyu/lsun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl2_2017_root = '~/clouderizer/fast.ai/fastai/courses/dl1/dl2_2017'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the scripts folder in the `dl2_2017` folder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "%mkdir {dl2_2017_root}/lsun_scripts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the scripts. We will be using the fast.ai modified version of the scripts that support Python 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100   158  100   158    0     0    158      0  0:00:01 --:--:--  0:00:01   637\n",
      "100  1641  100  1641    0     0   1641      0  0:00:01 --:--:--  0:00:01  1641\n"
     ]
    }
   ],
   "source": [
    "!curl -L https://github.com/fastai/fastai/raw/master/courses/dl2/lsun_scripts/lsun-data.py -o {dl2_2017_root}/lsun_scripts/lsun-data.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat {dl2_2017_root}/lsun_scripts/lsun-data.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bedroom_train_lmdb.zip\r\n"
     ]
    }
   ],
   "source": [
    "!ls ~/lsun_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process model weights (*.pth) in wgan_samples dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  wgan_samples/drive-download-20180515T095129Z-001.zip\n",
      "  inflating: netD_epoch_0_geniter_9000.pth  \n",
      "  inflating: netG_epoch_0_geniter_9000.pth  \n"
     ]
    }
   ],
   "source": [
    "!unzip wgan_samples/drive-download-20180515T095129Z-001.zip"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}