Data_Sci_Questions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Popular Data Science Questions\n",
    "Our goal in this project is to use Data Science Stack Exchange to determine what content a data company should create, based on interest by subject."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploring the Data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8839 entries, 0 to 8838\n",
      "Data columns (total 7 columns):\n",
      "Id               8839 non-null int64\n",
      "CreationDate     8839 non-null datetime64[ns]\n",
      "Score            8839 non-null int64\n",
      "ViewCount        8839 non-null int64\n",
      "Tags             8839 non-null object\n",
      "AnswerCount      8839 non-null int64\n",
      "FavoriteCount    1407 non-null float64\n",
      "dtypes: datetime64[ns](1), float64(1), int64(4), object(1)\n",
      "memory usage: 483.5+ KB\n"
     ]
    }
   ],
   "source": [
    "questions = pd.read_csv(\"2019_questions.csv\", parse_dates=[\"CreationDate\"])\n",
    "questions.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id               8839\n",
       "CreationDate     8839\n",
       "Score            8839\n",
       "ViewCount        8839\n",
       "Tags             8839\n",
       "AnswerCount      8839\n",
       "FavoriteCount    1407\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only the FavoriteCount column has missing values.  Also FavoriteCount column could be stored as an int64, not float64."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning the Data\n",
    "Filling in FavoriteCount with 0s if there is an NaN value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id               8839\n",
       "CreationDate     8839\n",
       "Score            8839\n",
       "ViewCount        8839\n",
       "Tags             8839\n",
       "AnswerCount      8839\n",
       "FavoriteCount    8839\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions.fillna(value={\"FavoriteCount\": 0}, inplace=True)\n",
    "questions[\"FavoriteCount\"] = questions[\"FavoriteCount\"].astype(int)\n",
    "questions.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>ViewCount</th>\n",
       "      <th>Tags</th>\n",
       "      <th>AnswerCount</th>\n",
       "      <th>FavoriteCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3216</th>\n",
       "      <td>48056</td>\n",
       "      <td>2019-03-26 22:07:29</td>\n",
       "      <td>1</td>\n",
       "      <td>139</td>\n",
       "      <td>[orange, orange3]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7187</th>\n",
       "      <td>43699</td>\n",
       "      <td>2019-01-09 01:15:04</td>\n",
       "      <td>0</td>\n",
       "      <td>514</td>\n",
       "      <td>[machine-learning, neural-network, deep-learni...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Id        CreationDate  Score  ViewCount  \\\n",
       "3216  48056 2019-03-26 22:07:29      1        139   \n",
       "7187  43699 2019-01-09 01:15:04      0        514   \n",
       "\n",
       "                                                   Tags  AnswerCount  \\\n",
       "3216                                  [orange, orange3]            1   \n",
       "7187  [machine-learning, neural-network, deep-learni...            2   \n",
       "\n",
       "      FavoriteCount  \n",
       "3216              1  \n",
       "7187              0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Making the tags data easier to work with:\n",
    "\n",
    "questions[\"Tags\"] = questions[\"Tags\"].str.replace(\"^<|>$\", \"\").str.split(\"><\")\n",
    "questions.sample(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Most Popular Used and Viewed Tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>multilabel-classification</th>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>regression</th>\n",
       "      <td>347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data-transfer</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orange</th>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linux</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Count\n",
       "multilabel-classification     92\n",
       "regression                   347\n",
       "data-transfer                  1\n",
       "orange                        64\n",
       "linux                          5"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_freq={}\n",
    "for tags in questions['Tags']:\n",
    "    for tag in tags:\n",
    "        tag_freq[tag]= tag_freq.get(tag, 0)+1\n",
    "        \n",
    "tag_count= pd.DataFrame.from_dict(tag_freq, orient='index')\n",
    "tag_count.rename(columns={0: \"Count\"}, inplace=True)\n",
    "tag_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>feature-engineering</th>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xgboost</th>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pytorch</th>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linear-regression</th>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data-science-model</th>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reinforcement-learning</th>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature-selection</th>\n",
       "      <td>209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image-classification</th>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data</th>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data-mining</th>\n",
       "      <td>217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>machine-learning-model</th>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>statistics</th>\n",
       "      <td>234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clustering</th>\n",
       "      <td>257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>predictive-modeling</th>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r</th>\n",
       "      <td>268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <td>340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>regression</th>\n",
       "      <td>347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pandas</th>\n",
       "      <td>354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lstm</th>\n",
       "      <td>402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time-series</th>\n",
       "      <td>466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn</th>\n",
       "      <td>489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlp</th>\n",
       "      <td>493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scikit-learn</th>\n",
       "      <td>540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tensorflow</th>\n",
       "      <td>584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>classification</th>\n",
       "      <td>685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keras</th>\n",
       "      <td>935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neural-network</th>\n",
       "      <td>1055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deep-learning</th>\n",
       "      <td>1220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>python</th>\n",
       "      <td>1814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>machine-learning</th>\n",
       "      <td>2693</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Count\n",
       "feature-engineering       163\n",
       "xgboost                   165\n",
       "pytorch                   175\n",
       "linear-regression         175\n",
       "data-science-model        186\n",
       "reinforcement-learning    203\n",
       "feature-selection         209\n",
       "image-classification      211\n",
       "data                      213\n",
       "data-mining               217\n",
       "machine-learning-model    224\n",
       "statistics                234\n",
       "clustering                257\n",
       "predictive-modeling       265\n",
       "r                         268\n",
       "dataset                   340\n",
       "regression                347\n",
       "pandas                    354\n",
       "lstm                      402\n",
       "time-series               466\n",
       "cnn                       489\n",
       "nlp                       493\n",
       "scikit-learn              540\n",
       "tensorflow                584\n",
       "classification            685\n",
       "keras                     935\n",
       "neural-network           1055\n",
       "deep-learning            1220\n",
       "python                   1814\n",
       "machine-learning         2693"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "most_common= tag_count.sort_values(by=\"Count\").tail(30)\n",
    "most_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fc13a63a7b8>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "most_common.plot(kind='barh', figsize= (16,8))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}