mechanism-of-action-exploration.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Mechanism of Action Exploration\n_By Nick Brooks, September 2020_\n\n#**Content:** <br>\n- Descriptive Statistics\n- Top Correlations"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport time\nimport math\nimport itertools\nfrom itertools import combinations\nimport re\nfrom collections import Counter\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nsns.set_style(\"whitegrid\")\nnotebookstart = time.time()\npd.options.display.max_colwidth = 500\npd.options.display.max_rows = 999\npd.options.display.width = 300\npd.options.display.max_columns = 100","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def big_count_plotter(plot_df, plt_set, columns, figsize, hue = None,\n                      custom_palette = sns.color_palette(\"Paired\", 15), top_n = 15):\n    \"\"\"\n    Iteratively Plot all categorical columns\n    Has category pre-processing - remove whitespace, lower, title, and takes first 30 characters.\n    \"\"\"\n    rows = math.ceil(len(plt_set)/columns)\n    n_plots = rows*columns\n    f,ax = plt.subplots(rows, columns, figsize = figsize)\n    for i in range(0,n_plots):\n        ax = plt.subplot(rows, columns, i+1)\n        if i < len(plt_set):\n            c_col = plt_set[i]\n            plt_tmp = plot_df.loc[plot_df[c_col].notnull(),c_col]\\\n                .astype(str).str.lower().str.strip()\\\n                .str.title().apply(lambda x: x[:30])\n            plot_order = plt_tmp.value_counts().index[:top_n]\n            if hue:\n                sns.countplot(y = plt_tmp, ax = ax, hue = hue, order = plot_order, palette = custom_palette)\n            else:\n                sns.countplot(y = plt_tmp, ax = ax, order = plot_order, palette = custom_palette)\n            ax.set_title(\"{} - {} Missing\".format(c_col.title(), plot_df[c_col].isnull().sum()))\n            ax.set_ylabel(\"{} Categories\".format(c_col.title()))\n            ax.set_xlabel(\"Count\")\n        else:\n            ax.axis('off')\n\n    plt.tight_layout(pad=1)\n    \n    \ndef big_boxplotter(plot_df, plt_set, columns, figsize, hue = None, plottype='kde',\n                   custom_palette = sns.color_palette(\"Dark2\", 15), quantile = .99):\n    rows = math.ceil(len(plt_set)/columns)\n    n_plots = rows*columns\n    f,ax = plt.subplots(rows, columns, figsize = figsize)\n    palette = itertools.cycle(custom_palette)\n    for i in range(0,n_plots):\n        ax = plt.subplot(rows, columns, i+1)\n        if i < len(plt_set):\n            cont_col = plt_set[i]\n            if hue:\n                plt_tmp = plot_df.loc[(plot_df[cont_col].notnull()) & \n                                          (plot_df[cont_col] < plot_df[cont_col].quantile(quantile)),\n                                      [cont_col, hue]]\n                if plottype == 'box':\n                    sns.boxplot(data=plt_tmp, x=cont_col, y=hue, color = next(palette), ax=ax)\n                    ax.set_ylabel(\"Categories\")\n                elif plottype == 'kde':\n                    for h in plt_tmp.dropna()[hue].value_counts()[:5].index:\n                        c = next(palette)\n                        sns.distplot(plt_tmp.loc[plt_tmp[hue] == h,cont_col], bins=10, kde=True, ax=ax,\n                                     kde_kws={\"color\": c, \"lw\": 2, \"label\":h}, color=c)\n                    ax.set_ylabel(\"Density Occurence\")\n            else:\n                plt_tmp = plot_df.loc[(plot_df[cont_col].notnull()) &\n                                          (plot_df[cont_col] < plot_df[cont_col].quantile(quantile)),\n                                      cont_col].astype(float)\n                if plottype == 'box':\n                    sns.boxplot(plt_tmp, color = next(palette), ax=ax)\n                    ax.set_ylabel(\"Categories\")\n                elif plottype == 'kde':\n                    sns.distplot(plt_tmp, bins=10, kde=True, ax=ax,\n                        kde_kws={\"color\": \"k\", \"lw\": 2}, color=next(palette))\n                    ax.set_ylabel(\"Density Occurence\")\n            ax.set_title(\"{} - {:.0f} Missing - {:.2f} Max\".format(cont_col.title(),\n                plot_df[cont_col].isnull().sum(), plot_df[cont_col].max()))\n            ax.set_xlabel(\"Value\")\n            \n        else:\n            ax.axis('off')\n\n    plt.tight_layout(pad=1)\n    \n    \ndef rank_correlations(df, figsize=(12,20), n_charts = 18, polyorder = 2, custom_palette = sns.color_palette(\"Paired\", 5)):\n    # Rank Correlations\n    palette = itertools.cycle(custom_palette)\n    continuous_rankedcorr = (df\n                             .corr()\n                             .unstack()\n                             .drop_duplicates().reset_index())\n    continuous_rankedcorr.columns = [\"f1\",\"f2\",\"Correlation Coefficient\"]\n    continuous_rankedcorr['abs_cor'] = abs(continuous_rankedcorr[\"Correlation Coefficient\"])\n    continuous_rankedcorr.sort_values(by='abs_cor', ascending=False, inplace=True)\n\n    # Plot Top Correlations\n    top_corr = [(x,y,cor) for x,y,cor in list(continuous_rankedcorr.iloc[:, :3].values) if x != y]\n    f, axes = plt.subplots(int(n_charts/3),3, figsize=figsize, sharex=False, sharey=False)\n    row = 0\n    col = 0\n    for (x,y, cor) in top_corr[:n_charts]:\n        if col == 3:\n            col = 0\n            row += 1\n        g = sns.regplot(x=x, y=y, data=df, order=polyorder, ax = axes[row,col], color=next(palette))\n        axes[row,col].set_title('{} and {}'.format(x, y))\n        axes[row,col].text(0.18, 0.93,\"Cor Coef: {:.2f}\".format(cor),\n                           ha='center', va='center', transform=axes[row,col].transAxes)\n        col += 1\n    plt.tight_layout(pad=0)\n    plt.show()\n    \n    \n# Data Exploration\ndef custom_describe(df, value_count_n = 5):\n    \"\"\"\n    Custom Describe Function - More Tailored to categorical type variables..\n    \"\"\"\n    unique_count = []\n    for x in df.columns:\n        unique_values_count = df[x].nunique()\n        value_count = df[x].value_counts().iloc[:5]\n\n        value_count_list = []\n        value_count_string = []\n        \n        for vc_i in range(0,value_count_n):\n            value_count_string += [\"ValCount {}\".format(vc_i+1),\n                                   \"Occ\"]\n            if vc_i <= unique_values_count - 1:\n                value_count_list.append(value_count.index[vc_i])\n                value_count_list.append(value_count.iloc[vc_i])\n            else:\n                value_count_list.append(np.nan)\n                value_count_list.append(np.nan)\n        \n        unique_count.append([x,\n                             unique_values_count,\n                             df[x].isnull().sum(),\n                             df[x].dtypes] + value_count_list)\n        \n    print(\"Dataframe Dimension: {} Rows, {} Columns\".format(*df.shape))\n    return pd.DataFrame(unique_count,\n            columns=[\"Column\",\"Unique\",\"Missing\",\"dtype\"\n                    ] + value_count_string\n                       ).set_index(\"Column\")\n\nprint(\"Helper Functions Ready\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Load"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/lish-moa/train_features.csv\")\nprint(\"Train DF Shape: {} Rows, {} Columns\".format(*df.shape))\n\nanon_cols = [x for x in df.columns if x not in ['sig_id','cp_type','cp_time','cp_dose']]\ncategorical_cols = [\"cp_dose\",\"cp_type\"]\ncontinuous_cols = [\"cp_time\"] + anon_cols\n\nlabels = pd.read_csv(\"/kaggle/input/lish-moa/train_targets_scored.csv\")\nprint(\"Labels DF Shape: {} Rows, {} Columns\".format(*labels.shape))\n\nlabel_names = [x for x in labels.columns if x not in \"sig_id\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explore Labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"melt_labels = pd.melt(labels, id_vars='sig_id', value_vars=label_names)\nmelt_labels = melt_labels.loc[melt_labels.value != 0].drop(\"value\", axis=1)\nmelt_labels.rename(columns={\"variable\":\"label\"}, inplace=True)\n\nf, ax = plt.subplots(figsize=[10,10])\nsns.countplot(y=melt_labels[\"label\"],\n              order=melt_labels[\"label\"].value_counts().index[:40],\n              palette=sns.color_palette(\"Paired\", 15),\n              ax=ax)\nax.set_title(\"Label Count Plot\")\nax.set_ylabel(\"Labels\")\nax.set_xlabel(\"Counts\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of active MoA Count:\\n{}\".format(\n    labels.sum(axis=1).value_counts().sort_index().to_dict()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_tokens = [item for sublist in [x.split(\"_\") for x in label_names] for item in sublist]\nnonPunct = re.compile('.*[A-Za-z0-9].*')  # must contain a letter or digit\nfiltered = [w for w in label_tokens if nonPunct.match(w)]\ncounts = Counter(filtered)\nlabel_keywords_pd = pd.Series(filtered)\n\nf, ax = plt.subplots(figsize=[10,6])\nsns.countplot(y=label_keywords_pd,\n              order=label_keywords_pd.value_counts().index[:25],\n              palette=sns.color_palette(\"Paired\", 15),\n              ax=ax\n             )\nax.set_title(\"Label Keywords\")\nax.set_ylabel(\"Label Keywords\")\nax.set_xlabel(\"Counts\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explore Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"big_count_plotter(plot_df = df,\n                  plt_set = categorical_cols,\n                  columns = 2,\n                  figsize = [10,5],\n                  custom_palette = sns.color_palette(\"Paired\", 15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Continuous Variables\")\ndisplay(df[continuous_cols].describe().T.sample(50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rank_correlations(df = df.loc[:,continuous_cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Script Complete - Runtime: {:.2f} Minutes\".format((time.time() - notebookstart) / 60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}