top-spotify-tracks-2019-eda.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Top Spotify Tracks 2019 EDA\n_By Nick Brooks, March 2020_\n\n**Goal:** <br>\nExplore the top spotify songs of 2019 worldwide. Refine my batch EDA functions.."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport time\nimport math\nimport itertools\nfrom wordcloud import WordCloud\n\nimport nltk\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nSIA = SentimentIntensityAnalyzer()\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nsns.set_style(\"whitegrid\")\nnotebookstart = time.time()\npd.options.display.max_colwidth = 500","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def meta_text_features(df, col):\n    df[col] = df[col].astype(str)\n    df[col + '_num_words'] = df[col].apply(lambda comment: len(comment.split())) # Count number of Words\n    df[col + '_num_unique_words'] = df[col].apply(lambda comment: len(set(w for w in comment.split())))\n    df[col + '_words_vs_unique'] = df[col+'_num_unique_words'] / df[col+'_num_words'] * 100 # Count Unique Words\n    if col == \"text\":\n        df[col+\"_vader_Compound\"]= df[col].apply(lambda x:SIA.polarity_scores(x)['compound'])\n\n    return df\n\ndef big_count_plotter(plot_df, plt_set, columns, figsize, hue = None,\n                      custom_palette = sns.color_palette(\"Paired\", 15), top_n = 15):\n    \"\"\"\n    Iteratively Plot all categorical columns\n    Has category pre-processing - remove whitespace, lower, title, and takes first 30 characters.\n    \"\"\"\n    rows = math.ceil(len(plt_set)/columns)\n    n_plots = rows*columns\n    f,ax = plt.subplots(rows, columns, figsize = figsize)\n    for i in range(0,n_plots):\n        ax = plt.subplot(rows, columns, i+1)\n        if i < len(plt_set):\n            c_col = plt_set[i]\n            plt_tmp = plot_df.loc[plot_df[c_col].notnull(),c_col]\\\n                .astype(str).str.lower().str.strip()\\\n                .str.title().apply(lambda x: x[:30])\n            plot_order = plt_tmp.value_counts().index[:top_n]\n            if hue:\n                sns.countplot(y = plt_tmp, ax = ax, hue = hue, order = plot_order, palette = custom_palette)\n            else:\n                sns.countplot(y = plt_tmp, ax = ax, order = plot_order, palette = custom_palette)\n            ax.set_title(\"{} - {} Missing\".format(c_col.title(), plot_df[c_col].isnull().sum()))\n            ax.set_ylabel(\"{} Categories\".format(c_col.title()))\n            ax.set_xlabel(\"Count\")\n        else:\n            ax.axis('off')\n\n    plt.tight_layout(pad=1)\n    \n    \ndef big_boxplotter(plot_df, plt_set, columns, figsize, hue = None, plottype='kde',\n                   custom_palette = sns.color_palette(\"Dark2\", 15), quantile = .99):\n    rows = math.ceil(len(plt_set)/columns)\n    n_plots = rows*columns\n    f,ax = plt.subplots(rows, columns, figsize = figsize)\n    palette = itertools.cycle(custom_palette)\n    for i in range(0,n_plots):\n        ax = plt.subplot(rows, columns, i+1)\n        if i < len(plt_set):\n            cont_col = plt_set[i]\n            if hue:\n                plt_tmp = plot_df.loc[(plot_df[cont_col].notnull()) & \n                                          (plot_df[cont_col] < plot_df[cont_col].quantile(quantile)),\n                                      [cont_col, hue]]\n                if plottype == 'box':\n                    sns.boxplot(data=plt_tmp, x=cont_col, y=hue, color = next(palette), ax=ax)\n                elif plottype == 'kde':\n                    for h in plt_tmp.dropna()[hue].value_counts()[:5].index:\n                        c = next(palette)\n                        sns.distplot(plt_tmp.loc[plt_tmp[hue] == h,cont_col], bins=10, kde=True, ax=ax,\n                                     kde_kws={\"color\": c, \"lw\": 2, \"label\":h}, color=c)\n            else:\n                plt_tmp = plot_df.loc[(plot_df[cont_col].notnull()) &\n                                          (plot_df[cont_col] < plot_df[cont_col].quantile(quantile)),\n                                      cont_col].astype(float)\n                if plottype == 'box':\n                    sns.boxplot(plt_tmp, color = next(palette), ax=ax)\n                elif plottype == 'kde':\n                    sns.distplot(plt_tmp, bins=10, kde=True, ax=ax,\n                        kde_kws={\"color\": \"k\", \"lw\": 2}, color=next(palette))\n            ax.set_title(\"{} - {} Missing - {} Max\".format(cont_col.title(),\n                plot_df[cont_col].isnull().sum(), plot_df[cont_col].max()))\n            ax.set_ylabel(\"Box\")\n            ax.set_xlabel(\"Count\")\n            \n        else:\n            ax.axis('off')\n\n    plt.tight_layout(pad=1)\n    \ndef big_word_cloud(plot_df, plt_set, columns, figsize, cmap = \"plasma\"):\n    \"\"\"\n    Iteratively Plot WordClouds\n    \"\"\"\n    rows = math.ceil(len(plt_set)/columns)\n    n_plots = rows*columns\n    f,ax = plt.subplots(rows, columns, figsize = figsize)\n    for i in range(0,n_plots):\n        ax = plt.subplot(rows, columns, i+1)\n        if i < len(plt_set):\n            str_col = plt_set[i]\n            string = \" \".join(plot_df.loc[plot_df[str_col].notnull(),str_col]\\\n                              .astype(str).str.lower().str.replace(\"none\", \"\").str.title())\n            string += 'EMPTY'\n            ax = plt.subplot(rows, 2, i+1)\n            plot_cloud(string, ax, title = \"{} - {} Missing\".format(\n                str_col.title(), plot_df[str_col].isnull().sum()), cmap = cmap)\n        else:\n            ax.axis('off')\n    plt.tight_layout(pad=0)\n    \ndef plot_cloud(string, ax, title = \"WordCloud\", cmap = \"plasma\"):\n    wordcloud = WordCloud(width=800, height=500,\n                          collocations=True,\n                          background_color=\"black\",\n                          max_words = 100,\n                          colormap=cmap\n                ).generate(string)\n\n    ax.imshow(wordcloud, interpolation='bilinear')\n    ax.set_title(title,  fontsize=18)\n    ax.axis('off')\n    \n    \ndef rank_correlations(df, figsize=(12,20), n_charts = 18, polyorder = 2, custom_palette = sns.color_palette(\"Paired\", 5)):\n    # Rank Correlations\n    palette = itertools.cycle(custom_palette)\n    continuous_rankedcorr = (df\n                             .corr()\n                             .unstack()\n                             .drop_duplicates().reset_index())\n    continuous_rankedcorr.columns = [\"f1\",\"f2\",\"Correlation Coefficient\"]\n    continuous_rankedcorr['abs_cor'] = abs(continuous_rankedcorr[\"Correlation Coefficient\"])\n    continuous_rankedcorr.sort_values(by='abs_cor', ascending=False, inplace=True)\n\n    # Plot Top Correlations\n    top_corr = [(x,y,cor) for x,y,cor in list(continuous_rankedcorr.iloc[:, :3].values) if x != y]\n    f, axes = plt.subplots(int(n_charts/3),3, figsize=figsize, sharex=False, sharey=False)\n    row = 0\n    col = 0\n    for (x,y, cor) in top_corr[:n_charts]:\n        if col == 3:\n            col = 0\n            row += 1\n        g = sns.regplot(x=x, y=y, data=df, order=polyorder, ax = axes[row,col], color=next(palette))\n        axes[row,col].set_title('{} and {}'.format(x, y))\n        axes[row,col].text(0.18, 0.93,\"Cor Coef: {}\".format(str(round(cor,2))),\n                           ha='center', va='center', transform=axes[row,col].transAxes)\n        col += 1\n    plt.tight_layout(pad=0)\n    plt.show()\n    \n    \n# Data Exploration\ndef custom_describe(df, value_count_n = 5):\n    \"\"\"\n    Custom Describe Function - More Tailored to categorical type variables..\n    \"\"\"\n    unique_count = []\n    for x in df.columns:\n        unique_values_count = df[x].nunique()\n        value_count = df[x].value_counts().iloc[:5]\n\n        value_count_list = []\n        value_count_string = []\n        \n        for vc_i in range(0,value_count_n):\n            value_count_string += [\"ValCount {}\".format(vc_i+1),\n                                   \"Occ\"]\n            if vc_i <= unique_values_count - 1:\n                value_count_list.append(value_count.index[vc_i])\n                value_count_list.append(value_count.iloc[vc_i])\n            else:\n                value_count_list.append(np.nan)\n                value_count_list.append(np.nan)\n        \n        unique_count.append([x,\n                             unique_values_count,\n                             df[x].isnull().sum(),\n                             df[x].dtypes] + value_count_list)\n        \n    print(\"Dataframe Dimension: {} Rows, {} Columns\".format(*df.shape))\n    return pd.DataFrame(unique_count,\n            columns=[\"Column\",\"Unique\",\"Missing\",\"dtype\"\n                    ] + value_count_string\n                       ).set_index(\"Column\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"column_mapping = {\n    'Track.Name': \"Track\",\n    'Artist.Name': \"Artist\",\n    'Beats.Per.Minute': 'BPM',\n    'Loudness..dB..': 'Loudness',\n    'Valence.': 'Valence',\n    'Length.': 'Length',\n    'Acousticness..': 'Acousticness',\n    'Speechiness.': 'Speechiness'\n}\n\ndf = pd.read_csv(\"/kaggle/input/top50spotify2019/top50.csv\", encoding='ISO-8859-1').iloc[:,1:].rename(column_mapping, axis = 1)\nprint(\"DF Shape: {} Rows, {} Columns\".format(*df.shape))\n\ndf['Genre'] = df['Genre'].str.title()\n\ndisplay(df.sort_values(by='Popularity', ascending=False).iloc[:7])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"custom_describe(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(figsize = [6,7])\nsns.countplot(y=df[\"Genre\"], palette = \"plasma\", order = df[\"Genre\"].value_counts().index, ax=ax)\nax.set_xlabel(\"Occurence\")\nax.set_ylabel(\"Gere\")\nax.set_title(\"Genre Counts\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"continuous_cols = [\n    'BPM',\n    'Energy',\n    'Danceability',\n    'Loudness',\n    'Liveness',\n    'Valence',\n    'Length',\n    'Acousticness',\n    'Speechiness',\n    'Popularity'\n]\n\n\nbig_boxplotter(plot_df = df,\n               plt_set = continuous_cols,\n               hue = None,\n               columns = 2,\n               figsize = [12,18],\n               quantile = .98)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"continuous_cols = [\n    'BPM',\n    'Energy',\n    'Danceability',\n    'Loudness',\n    'Liveness',\n    'Valence',\n    'Length',\n    'Acousticness',\n    'Speechiness',\n    'Popularity'\n]\n\nplot_tmp = df.loc[df.Genre.isin(['Dance Pop', 'Latin']), :]\nbig_boxplotter(plot_df = df,\n               plt_set = continuous_cols,\n               hue = 'Genre',\n               plottype= 'box',\n               columns = 2,\n               figsize = [15,25],\n               quantile = .98)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot Correlation Matrix\nf, ax = plt.subplots(figsize=[9,6])\nax = sns.heatmap(df[continuous_cols].corr(), \n                 annot=True, fmt=\".2f\",\n                 vmin=-1, vmax=1,\n                 cbar_kws={'label': 'Correlation Coefficient'})\nax.set_title(\"Continuous Variable Correlation Matrix\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rank_correlations(df = df.loc[:,continuous_cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"All Tracks\")\nbig_word_cloud(df,\n               plt_set = ['Track'],\n               columns = 1,\n               cmap='Spectral',\n               figsize = [15,15])\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Script Complete - Runtime: {:.2f} Minutes\".format((time.time() - notebookstart) / 60))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}