ieee-fraud-feature-engineering-and-eda.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# IEEE Fraud Feature Engineering and EDA\n_By Nick Brooks_\n\n- V1 - 20/08/2019 - First Commit <br>\n\n**Aim:** <br>\nBuild Features for Credit Card Fraud Project"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/arnocandel/python-datatable\n# more information: http://github.com/h2oai/datatable\n!pip install --upgrade https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/pydatatable/0.8.0.dev115/x86_64-centos7/datatable-0.8.0.dev115-cp36-cp36m-linux_x86_64.whl\n    \n# Latest Pandas version\n!pip install -q 'pandas==0.25' --force-reinstall","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datatable as dt\nimport pandas as pd\nprint(\"DataTable version:\", dt.__version__)\nprint(\"Pandas version:\", pd.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nimport os\nfrom contextlib import contextmanager\nimport gc; gc.enable()\nimport pprint\nimport time\n\nimport datetime\nimport csv\nimport random\n\nfrom sklearn import preprocessing\n\n# Viz\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"Define DF Schema..\")\n\ntarget_var = 'isFraud'\n\nemails = {'gmail': 'google',\n'att.net': 'att',\n'twc.com': 'spectrum',\n'scranton.edu': 'other',\n'optonline.net': 'other',\n'hotmail.co.uk': 'microsoft',\n'comcast.net': 'other',\n'yahoo.com.mx': 'yahoo',\n'yahoo.fr': 'yahoo',\n'yahoo.es': 'yahoo',\n'charter.net': 'spectrum',\n'live.com': 'microsoft',\n'aim.com': 'aol',\n'hotmail.de': 'microsoft',\n'centurylink.net': 'centurylink',\n'gmail.com': 'google',\n'me.com': 'apple',\n'earthlink.net': 'other',\n'gmx.de': 'other',\n'web.de': 'other',\n'cfl.rr.com': 'other',\n'hotmail.com': 'microsoft',\n'protonmail.com': 'other',\n'hotmail.fr': 'microsoft',\n'windstream.net': 'other',\n'outlook.es': 'microsoft',\n'yahoo.co.jp': 'yahoo',\n'yahoo.de': 'yahoo',\n'servicios-ta.com': 'other',\n'netzero.net': 'other',\n'suddenlink.net': 'other',\n'roadrunner.com': 'other',\n'sc.rr.com': 'other',\n'live.fr': 'microsoft',\n'verizon.net': 'yahoo',\n'msn.com': 'microsoft',\n'q.com': 'centurylink',\n'prodigy.net.mx': 'att',\n'frontier.com': 'yahoo',\n'anonymous.com': 'other',\n'rocketmail.com': 'yahoo',\n'sbcglobal.net': 'att',\n'frontiernet.net': 'yahoo',\n'ymail.com': 'yahoo',\n'outlook.com': 'microsoft',\n'mail.com': 'other',\n'bellsouth.net': 'other',\n'embarqmail.com': 'centurylink',\n'cableone.net': 'other',\n'hotmail.es': 'microsoft',\n'mac.com': 'apple',\n'yahoo.co.uk': 'yahoo',\n'netzero.com': 'other',\n'yahoo.com': 'yahoo',\n'live.com.mx': 'microsoft',\n'ptd.net': 'other',\n'cox.net': 'other',\n'aol.com': 'aol',\n'juno.com': 'other',\n'icloud.com': 'apple'}\n\n\nus_emails = ['gmail', 'net', 'edu']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    print(\"\\nMemeory Usage Before:\")\n    print(df.info(memory_usage = 'deep'))\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    \n    print(\"Memeory Usage After:\")\n    print(df.info(memory_usage = 'deep'))\n    return df\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes'.format(name, round((time.time() - t0)/60,2)))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def fraud_aggregate_function(dataframe):\n    # Credit https://www.kaggle.com/davidcairuz/feature-engineering-lightgbm-w-gpu\n    dataframe = (dataframe.assign(    \n        TransactionAmt_to_mean_card1 = (dataframe['TransactionAmt'] / dataframe.groupby(['card1'])['TransactionAmt'].transform('mean')),\n        TransactionAmt_to_std_card1 = (dataframe['TransactionAmt'] / dataframe.groupby(['card1'])['TransactionAmt'].transform('std')),\n        TransactionAmt_to_std_card4 = (dataframe['TransactionAmt'] / dataframe.groupby(['card4'])['TransactionAmt'].transform('std')),\n        TransactionAmt_to_mean_card4 = (dataframe['TransactionAmt'] / dataframe.groupby(['card4'])['TransactionAmt'].transform('mean')),\n\n        id_02_to_mean_card1 = (dataframe['id_02'] / dataframe.groupby(['card1'])['id_02'].transform('mean')),\n        id_02_to_std_card1 = (dataframe['id_02'] / dataframe.groupby(['card1'])['id_02'].transform('std')),\n        id_02_to_std_card4 = (dataframe['id_02'] / dataframe.groupby(['card4'])['id_02'].transform('std')),\n        id_02_to_mean_card4 = (dataframe['id_02'] / dataframe.groupby(['card4'])['id_02'].transform('mean')),\n\n        D15_to_mean_card1 = (dataframe['D15'] / dataframe.groupby(['card1'])['D15'].transform('mean')),\n        D15_to_std_card1 = (dataframe['D15'] / dataframe.groupby(['card1'])['D15'].transform('std')),\n        D15_to_mean_card4 = (dataframe['D15'] / dataframe.groupby(['card4'])['D15'].transform('mean')),\n        D15_to_std_card4 = (dataframe['D15'] / dataframe.groupby(['card4'])['D15'].transform('std')),\n        D15_to_mean_addr1 = (dataframe['D15'] / dataframe.groupby(['addr1'])['D15'].transform('mean')),\n        D15_to_std_addr1 = (dataframe['D15'] / dataframe.groupby(['addr1'])['D15'].transform('std')),\n\n        TransactionAmt_Log = np.log(dataframe['TransactionAmt'])\n    ))\n    # https://www.kaggle.com/c/ieee-fraud-detection/discussion/100499\n    for c in ['P_emaildomain', 'R_emaildomain']:\n        dataframe[c + '_bin'] = dataframe[c].map(emails)\n        dataframe[c + '_suffix'] = dataframe[c].map(lambda x: str(x).split('.')[-1])\n        dataframe[c + '_suffix'] = dataframe[c + '_suffix'].map(lambda x: x if str(x) not in us_emails else 'us')\n    \n    return dataframe\n    \n# Device Features\ndef id_split(dataframe):\n    # https://www.kaggle.com/davidcairuz/feature-engineering-lightgbm-w-gpu\n    dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]\n    dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]\n\n    dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]\n    dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]\n\n    dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]\n    dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]\n\n    dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]\n    dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]\n\n    dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]\n    dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]\n\n    dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'\n    dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'\n    dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'\n    dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'\n    dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'\n    dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'\n    dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'\n    dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'\n    dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'\n    dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'\n    dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'\n    dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'\n    dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'\n    dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'\n    dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'\n    dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'\n    dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'\n\n    dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts()[dataframe.device_name.value_counts() < 200].index), 'device_name'] = \"Others\"\n    dataframe['had_id'] = 1\n    gc.collect()\n    \n    return dataframe\n    \ndef fraud_preprocessing(debug = None):\n    print(\"Starting Pre-Processing..\")\n    with timer(\"Load Tables\"):\n        folder_path = '../input/ieee-fraud-detection/'\n        # parse all files\n        with timer(\"Read Tables\"):\n            # Read with DataTables (H20.ML)\n            train_identity = dt.fread(f'{folder_path}train_identity.csv')\n            test_identity = dt.fread(f'{folder_path}test_identity.csv')\n            train_transaction = dt.fread(f'{folder_path}train_transaction.csv')\n            test_transaction = dt.fread(f'{folder_path}test_transaction.csv')\n\n        # join frames\n        with timer(\"Join Identity Tables\"):\n            train_identity.key = 'TransactionID'\n            test_identity.key = 'TransactionID'\n            train = train_transaction[:, :, dt.join(train_identity)]\n            test = test_transaction[:, :, dt.join(test_identity)]\n\n        with timer(\"To Pandas\"):\n            if debug:\n                train = reduce_mem_usage(train.to_pandas().head(debug))\n                test = reduce_mem_usage(test.to_pandas().head(debug))\n            else:\n                train = reduce_mem_usage(train.to_pandas())\n                test = reduce_mem_usage(test.to_pandas())\n        \n        traindex = train.index\n        testdex = test.index\n        original_cols = train.columns\n        \n        # Get column groups\n        prefix_cols = {}\n        prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']\n        for i,p in enumerate(prefix):\n            prefix_cols[p] = [x for x in train.columns.tolist() if x.startswith(prefix[i])]\n        \n    with timer(\"Train/Test Split Feature Engineering\"):        \n        # Encoding - count encoding for both train and test\n        for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:\n            train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True).value_counts(dropna=False))\n            test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True).value_counts(dropna=False))\n\n        # Encoding - count encoding separately for train and test\n        for feature in ['id_01', 'id_31', 'id_33', 'id_36']:\n            train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))\n            test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))\n        \n        # Aggregated Features\n        train = fraud_aggregate_function(train)\n        test = fraud_aggregate_function(test)\n        \n        # Combine\n        y = train[target_var].copy()\n        df = pd.concat([train,test],axis = 0).reset_index()\n        del train, test\n        \n    with timer(\"Whole Feature Engineering\"):\n        START_DATE = '2017-12-01'\n        startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')    \n        df = df.assign(\n                # New feature - decimal part of the transaction amount\n                TransactionAmt_decimal = ((df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000).astype(int),\n\n                # Count encoding for card1 feature. \n                # Explained in this kernel: https://www.kaggle.com/nroman/eda-for-cis-fraud-detection\n                card1_count_full = df['card1'].map(df['card1'].value_counts(dropna=False)),\n\n                # https://www.kaggle.com/fchmiel/day-and-time-powerful-predictive-feature\n                Transaction_day_of_week = np.floor((df['TransactionDT'] / (3600 * 24) - 1) % 7),\n                Transaction_hour = np.floor(df['TransactionDT'] / 3600) % 24,\n\n                TransactionDT = df['TransactionDT'].apply(lambda x: (startdate + datetime.timedelta(seconds = x))),\n            )\n        df = df.assign(\n                # Time of Day\n                dow = df['TransactionDT'].dt.dayofweek,\n                year = df['TransactionDT'].dt.year,\n                month = df['TransactionDT'].dt.month,\n                hour = df['TransactionDT'].dt.hour,\n                day = df['TransactionDT'].dt.day,\n        \n                # All NaN\n                all_group_nan_sum = df.isnull().sum(axis=1) / df.shape[1],\n                all_group_0_count = (df == 0).astype(int).sum(axis=1) / (df.shape[1] - df.isnull().sum(axis=1))\n        )\n        \n        # Create Features based on anonymised prefix groups\n        for p in prefix_cols:\n            column_set = prefix_cols[p]\n            # Take NA count\n            df[p + \"group_nan_sum\"] = df[column_set].isnull().sum(axis=1) / df[column_set].shape[1]\n\n            # Take SUM/Mean if numeric\n            numeric_cols = [x for x in column_set if df[x].dtype != object]\n            if numeric_cols:\n                df[p + \"group_sum\"] = df[column_set].sum(axis=1)\n                df[p + \"group_mean\"] = df[column_set].mean(axis=1)\n                # Zero Count\n                df[p + \"group_0_count\"] = (df[column_set] == 0).astype(int).sum(axis=1) / (df[column_set].shape[1] - df[p + \"group_nan_sum\"])\n                \n    with timer(\"Rolling Features\"):\n        prefix = ['C', 'card']\n        \n        value = \"TransactionAmt\"\n        timevar = \"TransactionDT\"\n        \n        for window in ['12h', '5d']:\n            with timer(\"TimeFrame: {}\".format(window)):\n                for i, p in enumerate(prefix):\n                    for var in prefix_cols[p]:\n                        gb_var = [var]\n                        df = pd.merge(df, (df.set_index(timevar)\n                                           .sort_values(timevar)\n                                           .groupby(gb_var)\n                                           .rolling(window)[value].sum()\n                                           .rename(gb_var[0] + \"_AMT_\" + window + \"_sum\")\n                                           .reset_index()\n                                           .drop_duplicates([timevar] + gb_var)),\n                                      on= [timevar] + gb_var, how= 'left')\n\n                        df = pd.merge(df, (df.set_index(timevar)\n                                           .sort_values(timevar)\n                                           .groupby(gb_var)\n                                           .rolling(window)[value].mean()\n                                           .rename(gb_var[0] + \"_AMT_\" + window + \"_mean\")\n                                           .reset_index()\n                                           .drop_duplicates([timevar] + gb_var)),\n                                      on= [timevar] + gb_var, how= 'left')\n\n                        df = pd.merge(df, (df.set_index(timevar)\n                                           .sort_values(timevar)\n                                           .groupby(gb_var)\n                                           .rolling(window)[value].count()\n                                           .rename(gb_var[0] + \"_AMT_\" + window + \"_count\")\n                                           .reset_index()\n                                           .drop_duplicates([timevar] + gb_var)),\n                                      on= [timevar] + gb_var, how= 'left')\n\n    with timer(\"Label Encode\"):\n        categorical_cols = []\n        # Label Encoding\n        for f in df.columns:\n            if df[f].dtype=='object': \n                categorical_cols += [f]\n                lbl = preprocessing.LabelEncoder()\n                df[f] = lbl.fit_transform(df[f].astype(str))\n                \n                \n    df.fillna(-9,inplace=True)\n    df.set_index(\"TransactionID\",inplace=True)\n    \n    # One more memory reduction\n    df = reduce_mem_usage(df)\n    print(\"Total Shape: {} Rows, {} Columns\".format(*df.shape))\n    \n    return df, y, original_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = None # None for no debug, else number of rows\ndf, y, original_cols = fraud_preprocessing(debug = DEBUG)\ndf.loc[df.isFraud == 2, 'isFraud'] = np.nan","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"with timer(\"Write Table\"):\n    write_cols = [x for x in df.columns if x not in original_cols] + [target_var]\n    print(\"Writing {} Column\".format(len(write_cols)))\n    df.loc[df[target_var].notnull(),write_cols].to_csv(\"train_fraud_fe_nb.csv\", index = True)\n    df.loc[df[target_var].isnull(),write_cols].to_csv(\"test_fraud_fe_nb.csv\", index = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Features for EDA\ndf['yrmth'] = df.year.astype(str) + df.month.map(\"{:02}\".format)\ndf.loc[df.isFraud == 2, 'isFraud'] = np.nan\ndf['traintest'] = 'Test'\ndf.loc[df.isFraud.notnull(),'traintest'] = 'Train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Are there redundant Transaction IDs?\")\nprint(df.index.value_counts().value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train / Submission Time Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(2,2, figsize = [12,10])\nfor tt in ['Train','Test']:\n    df.loc[df.traintest == tt,['all_group_nan_sum','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').count().plot(label = tt, ax = ax[0,0])\n    df.loc[df.traintest == tt,['all_group_nan_sum','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = tt, ax = ax[1,0])\n    df.loc[df.traintest == tt,['all_group_0_count','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = tt, ax = ax[1,1])\n    df.loc[df.traintest == tt,['TransactionAmt','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = tt, ax = ax[0,1])\nax[0,0].set_title(\"Observation Count: Train/ Test\")\nax[0,0].set_ylabel(\"Count\")\nax[1,0].set_title(\"Average Number of Missing Values in Rows: Train/ Test\")\nax[1,0].set_ylabel(\"Percent of Rows Is Null\")\nax[0,1].set_title(\"Average Number of Missing Values in Rows: Train/ Test\")\nax[0,1].set_ylabel(\"Percent of Rows Is Null\")\nax[1,1].set_title(\"Average Number of Zero Values in Rows: Train/ Test\")\nax[1,1].set_ylabel(\"Percent of Rows Is Zero\")\n\nplt.tight_layout(pad=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Missing Values and Zeroes"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(1,2,figsize = [12,5])\n\nprefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']\nfor i, p in enumerate(prefix):\n    df.loc[df.traintest == tt,[p + \"group_nan_sum\",'TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = \"Missing\", ax = ax[0])\n    df.loc[df.traintest == tt,[p + \"group_0_count\",'TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = \"Zero\", ax = ax[1])\n\nax[0].get_legend().remove()\nax[1].legend(prefix,fontsize='large', loc='center left',bbox_to_anchor=(1, 0.5))\n\nax[0].set_title(\"Proportion of Data Missing by Column Group\")\nax[1].set_title(\"Proportion of Data Equal Zero by Column Group\")\nax[0].set_ylabel(\"Proportion Missing\")\nax[1].set_ylabel(\"Proportion Zero\")\n\nplt.tight_layout(pad=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Rolling Averages"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Univariate Exploration"},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = ['card1_count_full', 'card1','card2','card2_count_full']\nplot_df = df[cols + ['isFraud']]\n\nt_r,t_c = 2, 2\nf, axes = plt.subplots(t_r,t_c, figsize = [12,8],sharex=False, sharey=False)\nrow,col = 0,0\nfor c in cols:\n    if col == t_c:\n        col = 0\n        row += 1\n    sns.kdeplot(plot_df.loc[plot_df.isFraud == 0, c], shade = True, alpha = 0.6, color = 'black', ax = axes[row,col], label = 'Not Fraud')\n    sns.kdeplot(plot_df.loc[plot_df.isFraud == 1, c], shade = True, alpha = 0.6, color = 'lime', ax = axes[row,col], label = 'Fraud')\n    axes[row,col].set_title('{} and Fraud Distribution'.format(c.title()))\n    col+=1\n    \nplt.tight_layout(pad=0)\nplt.show()\ndel plot_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Missing Values Pattern\n# Hourly Pattern\n# Individual's susepticality to fraud (explore ID)\n# Is there a way to see how close various fraud claims are?\n\n# Create a CPU kernel where I can experiment with features and LOFO..\n# Smash all data together.","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}