Track1-MLPClassifier-predict-private.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cuDNN version 6021 on context None\n",
      "Mapped name None to device cuda: GeForce GTX TITAN X (0000:02:00.0)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import swifter\n",
    "from sklearn.model_selection import train_test_split\n",
    "import utils\n",
    "from hep_ml.nnet import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"./data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/idao_cb/lib/python3.6/site-packages/numpy/lib/arraysetops.py:522: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv(\"data/test_private_v3_track_1.csv.gz\",\n",
    "                   usecols=[utils.ID_COLUMN] + utils.SIMPLE_FEATURE_COLUMNS,\n",
    "                   index_col=utils.ID_COLUMN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ncl[0]', 'ncl[1]', 'ncl[2]', 'ncl[3]', 'avg_cs[0]', 'avg_cs[1]',\n",
       "       'avg_cs[2]', 'avg_cs[3]', 'ndof', 'MatchedHit_TYPE[0]',\n",
       "       'MatchedHit_TYPE[1]', 'MatchedHit_TYPE[2]', 'MatchedHit_TYPE[3]',\n",
       "       'MatchedHit_X[0]', 'MatchedHit_X[1]', 'MatchedHit_X[2]',\n",
       "       'MatchedHit_X[3]', 'MatchedHit_Y[0]', 'MatchedHit_Y[1]',\n",
       "       'MatchedHit_Y[2]', 'MatchedHit_Y[3]', 'MatchedHit_Z[0]',\n",
       "       'MatchedHit_Z[1]', 'MatchedHit_Z[2]', 'MatchedHit_Z[3]',\n",
       "       'MatchedHit_DX[0]', 'MatchedHit_DX[1]', 'MatchedHit_DX[2]',\n",
       "       'MatchedHit_DX[3]', 'MatchedHit_DY[0]', 'MatchedHit_DY[1]',\n",
       "       'MatchedHit_DY[2]', 'MatchedHit_DY[3]', 'MatchedHit_DZ[0]',\n",
       "       'MatchedHit_DZ[1]', 'MatchedHit_DZ[2]', 'MatchedHit_DZ[3]',\n",
       "       'MatchedHit_T[0]', 'MatchedHit_T[1]', 'MatchedHit_T[2]',\n",
       "       'MatchedHit_T[3]', 'MatchedHit_DT[0]', 'MatchedHit_DT[1]',\n",
       "       'MatchedHit_DT[2]', 'MatchedHit_DT[3]', 'Lextra_X[0]', 'Lextra_X[1]',\n",
       "       'Lextra_X[2]', 'Lextra_X[3]', 'Lextra_Y[0]', 'Lextra_Y[1]',\n",
       "       'Lextra_Y[2]', 'Lextra_Y[3]', 'NShared', 'Mextra_DX2[0]',\n",
       "       'Mextra_DX2[1]', 'Mextra_DX2[2]', 'Mextra_DX2[3]', 'Mextra_DY2[0]',\n",
       "       'Mextra_DY2[1]', 'Mextra_DY2[2]', 'Mextra_DY2[3]', 'FOI_hits_N', 'PT',\n",
       "       'P'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "func = utils.find_closest_hit_per_station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_test  = pd.read_hdf('closest_hits_features.test.filled.m9999.v1.private.hdf', 'key')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_test.columns  = [func.__name__ + '_' + str(c) if str(c).isnumeric() else c for c in dt_test.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.concat([test, dt_test], axis=1, copy=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1452188, 113)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Новые признаки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "NA_VALUE = -9999.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fillna(df):\n",
    "    \"\"\"\n",
    "    Пропуски только в 2 и 3 колонках\n",
    "    \"\"\"\n",
    "    \n",
    "    for axis in 'XYZ':\n",
    "        cols = ['MatchedHit_{}[{}]'.format(axis, i) for i in range(4)]\n",
    "        mask = np.isclose(df.loc[:, cols[2]], NA_VALUE)\n",
    "        mask = mask & ~np.isclose(df.loc[:, cols[3]], NA_VALUE)\n",
    "        \n",
    "        df.loc[mask, cols[2]] = df.loc[mask, [cols[1], cols[3]]].mean(axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(fillna, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = fillna(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Модифицируем matched_hit_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modify_matched_hit_type(df):\n",
    "    map_type = {2: 1, 0: 0, 1: -1}\n",
    "    cols = df.columns[df.columns.str.startswith('MatchedHit_TYPE')]\n",
    "    for col in cols:\n",
    "        df[col] = df[col].map(map_type)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(modify_matched_hit_type, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = modify_matched_hit_type(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Вычесть среднее из MatchedHit_Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MatchedHit_Z[0]    15270\n",
       "MatchedHit_Z[1]    16470\n",
       "MatchedHit_Z[2]    17670\n",
       "MatchedHit_Z[3]    18870\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matched_hit_z_cols = train.columns[train.columns.str.startswith(\"MatchedHit_Z\")]\n",
    "matched_hit_z_mean = pd.Series([15270, 16470, 17670, 18870], index=matched_hit_z_cols)\n",
    "matched_hit_z_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_unbiased_z(df):\n",
    "    for i in range(4):\n",
    "        distance = np.zeros(df.shape[0])\n",
    "    \n",
    "        col = f\"MatchedHit_Z[{i}]\"\n",
    "        mask = np.isclose(df.loc[:, col], NA_VALUE)\n",
    "        distance[~mask] = df.loc[~mask, col] - matched_hit_z_mean[col]\n",
    "        distance[mask] = 0\n",
    "    \n",
    "        df[col] = distance\n",
    "        \n",
    "        df[f\"MatchedHit_Z[{i}]_sign\"] = np.sign(df[col]).astype(np.int8)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(make_unbiased_z, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = make_unbiased_z(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_Z[0]</th>\n",
       "      <th>MatchedHit_Z[1]</th>\n",
       "      <th>MatchedHit_Z[2]</th>\n",
       "      <th>MatchedHit_Z[3]</th>\n",
       "      <th>MatchedHit_Z[0]_sign</th>\n",
       "      <th>MatchedHit_Z[1]_sign</th>\n",
       "      <th>MatchedHit_Z[2]_sign</th>\n",
       "      <th>MatchedHit_Z[3]_sign</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>136.950</td>\n",
       "      <td>144.400</td>\n",
       "      <td>138.152</td>\n",
       "      <td>147.201</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>129.885</td>\n",
       "      <td>133.830</td>\n",
       "      <td>126.380</td>\n",
       "      <td>-65.605</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-61.252</td>\n",
       "      <td>-53.578</td>\n",
       "      <td>-59.605</td>\n",
       "      <td>-50.332</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-71.983</td>\n",
       "      <td>-68.110</td>\n",
       "      <td>-75.630</td>\n",
       "      <td>-67.846</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-64.852</td>\n",
       "      <td>-60.414</td>\n",
       "      <td>-67.373</td>\n",
       "      <td>-59.027</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_Z[0]  MatchedHit_Z[1]  MatchedHit_Z[2]  MatchedHit_Z[3]  \\\n",
       "0          136.950          144.400          138.152          147.201   \n",
       "1          129.885          133.830          126.380          -65.605   \n",
       "2          -61.252          -53.578          -59.605          -50.332   \n",
       "3          -71.983          -68.110          -75.630          -67.846   \n",
       "4          -64.852          -60.414          -67.373          -59.027   \n",
       "\n",
       "   MatchedHit_Z[0]_sign  MatchedHit_Z[1]_sign  MatchedHit_Z[2]_sign  \\\n",
       "0                     1                     1                     1   \n",
       "1                     1                     1                     1   \n",
       "2                    -1                    -1                    -1   \n",
       "3                    -1                    -1                    -1   \n",
       "4                    -1                    -1                    -1   \n",
       "\n",
       "   MatchedHit_Z[3]_sign  \n",
       "0                     1  \n",
       "1                    -1  \n",
       "2                    -1  \n",
       "3                    -1  \n",
       "4                    -1  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_Z\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Расстояние до центра (+)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_polar_distances_matched_hit(df):\n",
    "    for i in range(4):\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "    \n",
    "        name = f\"MatchedHit_R[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = np.log(1.0 + np.power(df.loc[~mask, cols], 2).sum(axis=1))\n",
    "        \n",
    "        name = f\"MatchedHit_A[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = np.arctan2(df.loc[~mask, cols[1]], df.loc[~mask, cols[0]])\n",
    "\n",
    "    # намеренное переупорядочивание признаков\n",
    "    cols  = df.columns[df.columns.str.startswith(\"MatchedHit_R\")].tolist()\n",
    "    cols += df.columns[df.columns.str.startswith(\"MatchedHit_A\")].tolist()\n",
    "    cols  = df.columns[:-len(cols)].tolist() + cols\n",
    "    df = df[cols]\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(compute_polar_distances_matched_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = compute_polar_distances_matched_hit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_R[0]</th>\n",
       "      <th>MatchedHit_R[1]</th>\n",
       "      <th>MatchedHit_R[2]</th>\n",
       "      <th>MatchedHit_R[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12.930077</td>\n",
       "      <td>13.078763</td>\n",
       "      <td>13.207914</td>\n",
       "      <td>13.338887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16.332379</td>\n",
       "      <td>16.538395</td>\n",
       "      <td>16.718047</td>\n",
       "      <td>16.903495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.370883</td>\n",
       "      <td>14.524747</td>\n",
       "      <td>14.662623</td>\n",
       "      <td>14.794065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16.525674</td>\n",
       "      <td>16.678201</td>\n",
       "      <td>16.831457</td>\n",
       "      <td>16.963506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13.714256</td>\n",
       "      <td>13.840950</td>\n",
       "      <td>13.964308</td>\n",
       "      <td>14.099488</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_R[0]  MatchedHit_R[1]  MatchedHit_R[2]  MatchedHit_R[3]\n",
       "0        12.930077        13.078763        13.207914        13.338887\n",
       "1        16.332379        16.538395        16.718047        16.903495\n",
       "2        14.370883        14.524747        14.662623        14.794065\n",
       "3        16.525674        16.678201        16.831457        16.963506\n",
       "4        13.714256        13.840950        13.964308        14.099488"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_R\")].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_A[0]</th>\n",
       "      <th>MatchedHit_A[1]</th>\n",
       "      <th>MatchedHit_A[2]</th>\n",
       "      <th>MatchedHit_A[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.002782</td>\n",
       "      <td>1.002964</td>\n",
       "      <td>1.010349</td>\n",
       "      <td>1.008445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2.885018</td>\n",
       "      <td>-2.891988</td>\n",
       "      <td>-2.897091</td>\n",
       "      <td>-2.974431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.522447</td>\n",
       "      <td>1.503224</td>\n",
       "      <td>1.531503</td>\n",
       "      <td>1.530494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2.844808</td>\n",
       "      <td>-2.844651</td>\n",
       "      <td>-2.846558</td>\n",
       "      <td>-2.846975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.042894</td>\n",
       "      <td>2.019910</td>\n",
       "      <td>2.000288</td>\n",
       "      <td>2.000319</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_A[0]  MatchedHit_A[1]  MatchedHit_A[2]  MatchedHit_A[3]\n",
       "0         1.002782         1.002964         1.010349         1.008445\n",
       "1        -2.885018        -2.891988        -2.897091        -2.974431\n",
       "2         1.522447         1.503224         1.531503         1.530494\n",
       "3        -2.844808        -2.844651        -2.846558        -2.846975\n",
       "4         2.042894         2.019910         2.000288         2.000319"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_A\")].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_na(df):\n",
    "    df[df == NA_VALUE] = None\n",
    "    df.fillna(df.mean(), inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(replace_na, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = replace_na(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Дельты и отношения для радиусов (+)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ratio_radius_matched_hit(df):\n",
    "    for i, j in zip([0, 1, 2, 3], [1, 2, 3, 0]):\n",
    "        col_i = f\"MatchedHit_R[{i}]\"\n",
    "        col_j = f\"MatchedHit_R[{j}]\"\n",
    "        mask = np.isclose(df.loc[:, [col_i, col_j]], NA_VALUE).any(axis=1)\n",
    "\n",
    "        name = f\"MatchedHit_R[{j}/{i}]_delta\"\n",
    "        name_abs = name + '_abs'\n",
    "        df[name] = df[name_abs] = NA_VALUE\n",
    "        \n",
    "        df.loc[~mask, name] = df.loc[~mask, col_j] - df.loc[~mask, col_i]\n",
    "        df.loc[~mask, name_abs] = np.abs(df.loc[~mask, name])\n",
    "        \n",
    "        name = f\"MatchedHit_R[{j}/{i}]_frac\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = df.loc[~mask, col_j] / df.loc[~mask, col_i]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(ratio_radius_matched_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ratio_radius_matched_hit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_R[1/0]_delta</th>\n",
       "      <th>MatchedHit_R[1/0]_delta_abs</th>\n",
       "      <th>MatchedHit_R[1/0]_frac</th>\n",
       "      <th>MatchedHit_R[2/1]_delta</th>\n",
       "      <th>MatchedHit_R[2/1]_delta_abs</th>\n",
       "      <th>MatchedHit_R[2/1]_frac</th>\n",
       "      <th>MatchedHit_R[3/2]_delta</th>\n",
       "      <th>MatchedHit_R[3/2]_delta_abs</th>\n",
       "      <th>MatchedHit_R[3/2]_frac</th>\n",
       "      <th>MatchedHit_R[0/3]_delta</th>\n",
       "      <th>MatchedHit_R[0/3]_delta_abs</th>\n",
       "      <th>MatchedHit_R[0/3]_frac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.148686</td>\n",
       "      <td>0.148686</td>\n",
       "      <td>1.011499</td>\n",
       "      <td>0.129151</td>\n",
       "      <td>0.129151</td>\n",
       "      <td>1.009875</td>\n",
       "      <td>0.130973</td>\n",
       "      <td>0.130973</td>\n",
       "      <td>1.009916</td>\n",
       "      <td>-0.408810</td>\n",
       "      <td>0.408810</td>\n",
       "      <td>0.969352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.206016</td>\n",
       "      <td>0.206016</td>\n",
       "      <td>1.012614</td>\n",
       "      <td>0.179652</td>\n",
       "      <td>0.179652</td>\n",
       "      <td>1.010863</td>\n",
       "      <td>0.185449</td>\n",
       "      <td>0.185449</td>\n",
       "      <td>1.011093</td>\n",
       "      <td>-0.571117</td>\n",
       "      <td>0.571117</td>\n",
       "      <td>0.966213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.153863</td>\n",
       "      <td>0.153863</td>\n",
       "      <td>1.010707</td>\n",
       "      <td>0.137877</td>\n",
       "      <td>0.137877</td>\n",
       "      <td>1.009493</td>\n",
       "      <td>0.131441</td>\n",
       "      <td>0.131441</td>\n",
       "      <td>1.008964</td>\n",
       "      <td>-0.423181</td>\n",
       "      <td>0.423181</td>\n",
       "      <td>0.971395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.152526</td>\n",
       "      <td>0.152526</td>\n",
       "      <td>1.009230</td>\n",
       "      <td>0.153256</td>\n",
       "      <td>0.153256</td>\n",
       "      <td>1.009189</td>\n",
       "      <td>0.132049</td>\n",
       "      <td>0.132049</td>\n",
       "      <td>1.007845</td>\n",
       "      <td>-0.437831</td>\n",
       "      <td>0.437831</td>\n",
       "      <td>0.974190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.126694</td>\n",
       "      <td>0.126694</td>\n",
       "      <td>1.009238</td>\n",
       "      <td>0.123358</td>\n",
       "      <td>0.123358</td>\n",
       "      <td>1.008913</td>\n",
       "      <td>0.135179</td>\n",
       "      <td>0.135179</td>\n",
       "      <td>1.009680</td>\n",
       "      <td>-0.385231</td>\n",
       "      <td>0.385231</td>\n",
       "      <td>0.972678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_R[1/0]_delta  MatchedHit_R[1/0]_delta_abs  \\\n",
       "0                 0.148686                     0.148686   \n",
       "1                 0.206016                     0.206016   \n",
       "2                 0.153863                     0.153863   \n",
       "3                 0.152526                     0.152526   \n",
       "4                 0.126694                     0.126694   \n",
       "\n",
       "   MatchedHit_R[1/0]_frac  MatchedHit_R[2/1]_delta  \\\n",
       "0                1.011499                 0.129151   \n",
       "1                1.012614                 0.179652   \n",
       "2                1.010707                 0.137877   \n",
       "3                1.009230                 0.153256   \n",
       "4                1.009238                 0.123358   \n",
       "\n",
       "   MatchedHit_R[2/1]_delta_abs  MatchedHit_R[2/1]_frac  \\\n",
       "0                     0.129151                1.009875   \n",
       "1                     0.179652                1.010863   \n",
       "2                     0.137877                1.009493   \n",
       "3                     0.153256                1.009189   \n",
       "4                     0.123358                1.008913   \n",
       "\n",
       "   MatchedHit_R[3/2]_delta  MatchedHit_R[3/2]_delta_abs  \\\n",
       "0                 0.130973                     0.130973   \n",
       "1                 0.185449                     0.185449   \n",
       "2                 0.131441                     0.131441   \n",
       "3                 0.132049                     0.132049   \n",
       "4                 0.135179                     0.135179   \n",
       "\n",
       "   MatchedHit_R[3/2]_frac  MatchedHit_R[0/3]_delta  \\\n",
       "0                1.009916                -0.408810   \n",
       "1                1.011093                -0.571117   \n",
       "2                1.008964                -0.423181   \n",
       "3                1.007845                -0.437831   \n",
       "4                1.009680                -0.385231   \n",
       "\n",
       "   MatchedHit_R[0/3]_delta_abs  MatchedHit_R[0/3]_frac  \n",
       "0                     0.408810                0.969352  \n",
       "1                     0.571117                0.966213  \n",
       "2                     0.423181                0.971395  \n",
       "3                     0.437831                0.974190  \n",
       "4                     0.385231                0.972678  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = filter(lambda s: s.startswith(\"MatchedHit_R\") and (\"_delta\" in s or \"_frac\" in s), train.columns)\n",
    "train.loc[:, cols].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Расстояние между Lextra и MatchedHit (+)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distance_dist_lextra_matched(df):\n",
    "    distance_counts = np.zeros(df.shape[0], dtype=int)\n",
    "    \n",
    "    distance_se = np.zeros(df.shape[0])\n",
    "    distance_ae = np.zeros(df.shape[0])\n",
    "    \n",
    "    for i in range(4):\n",
    "        distance = np.zeros(df.shape[0])\n",
    "\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\", f\"Lextra_X[{i}]\", f\"Lextra_Y[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "\n",
    "        mse = df.loc[~mask, cols[:2]].values - df.loc[~mask, cols[2:]].values\n",
    "        mse = np.power(mse, 2.0).sum(axis=1)\n",
    "        \n",
    "        mae = df.loc[~mask, cols[:2]].values - df.loc[~mask, cols[2:]].values\n",
    "        mae = np.abs(mae).max(axis=1)\n",
    "\n",
    "        distance[~mask] = np.log(1.0 + mse)\n",
    "        distance[mask] = NA_VALUE\n",
    "        \n",
    "        distance_se[~mask] += mse\n",
    "        distance_ae[~mask] = np.maximum(distance_ae[~mask], mae)\n",
    "        distance_counts[~mask] += 1\n",
    "\n",
    "        df[f\"distance_LextraMatchedHit[{i}]\"] = distance\n",
    "\n",
    "#     distance_se /= distance_counts\n",
    "#     df[f\"distance_LextraMatchedHit_SE\"] = distance_se\n",
    "#     df[f\"distance_LextraMatchedHit_AE\"] = distance_ae\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(distance_dist_lextra_matched, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = distance_dist_lextra_matched(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distance_LextraMatchedHit[0]</th>\n",
       "      <th>distance_LextraMatchedHit[1]</th>\n",
       "      <th>distance_LextraMatchedHit[2]</th>\n",
       "      <th>distance_LextraMatchedHit[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.787645</td>\n",
       "      <td>6.930203</td>\n",
       "      <td>7.078433</td>\n",
       "      <td>7.381559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.315024</td>\n",
       "      <td>10.127164</td>\n",
       "      <td>10.571843</td>\n",
       "      <td>12.132834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.357462</td>\n",
       "      <td>6.408753</td>\n",
       "      <td>6.697971</td>\n",
       "      <td>7.287096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8.855256</td>\n",
       "      <td>9.224567</td>\n",
       "      <td>9.392788</td>\n",
       "      <td>9.816331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.986883</td>\n",
       "      <td>7.280281</td>\n",
       "      <td>7.445496</td>\n",
       "      <td>6.714575</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distance_LextraMatchedHit[0]  distance_LextraMatchedHit[1]  \\\n",
       "0                      6.787645                      6.930203   \n",
       "1                      9.315024                     10.127164   \n",
       "2                      4.357462                      6.408753   \n",
       "3                      8.855256                      9.224567   \n",
       "4                      6.986883                      7.280281   \n",
       "\n",
       "   distance_LextraMatchedHit[2]  distance_LextraMatchedHit[3]  \n",
       "0                      7.078433                      7.381559  \n",
       "1                     10.571843                     12.132834  \n",
       "2                      6.697971                      7.287096  \n",
       "3                      9.392788                      9.816331  \n",
       "4                      7.445496                      6.714575  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"distance_LextraMatchedHit\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Отношение дисперсий"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ratio_mextra_matched(df):    \n",
    "    for i in range(4):\n",
    "        cols = [f\"MatchedHit_DX[{i}]\", f\"MatchedHit_DY[{i}]\", f\"Mextra_DX2[{i}]\", f\"Mextra_DY2[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "\n",
    "        tmpX = df.loc[~mask, cols[::2]]\n",
    "        tmpY = df.loc[~mask, cols[1::2]]\n",
    "        \n",
    "        ratio = tmpX.iloc[:, 0] / np.sqrt(tmpX.iloc[:, 1]) + \\\n",
    "                tmpY.iloc[:, 0] / np.sqrt(tmpY.iloc[:, 1])\n",
    "\n",
    "        name = f\"ratio_MextraMatchedHit[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = ratio\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(ratio_mextra_matched, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ratio_mextra_matched(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ratio_MextraMatchedHit[0]</th>\n",
       "      <th>ratio_MextraMatchedHit[1]</th>\n",
       "      <th>ratio_MextraMatchedHit[2]</th>\n",
       "      <th>ratio_MextraMatchedHit[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.568041</td>\n",
       "      <td>5.452831</td>\n",
       "      <td>5.923010</td>\n",
       "      <td>4.591995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.836899</td>\n",
       "      <td>3.697998</td>\n",
       "      <td>3.890070</td>\n",
       "      <td>2.942780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15.471354</td>\n",
       "      <td>9.812719</td>\n",
       "      <td>10.688360</td>\n",
       "      <td>8.330187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.402639</td>\n",
       "      <td>6.001297</td>\n",
       "      <td>6.386638</td>\n",
       "      <td>4.884344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.857346</td>\n",
       "      <td>2.445352</td>\n",
       "      <td>2.635871</td>\n",
       "      <td>2.025441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ratio_MextraMatchedHit[0]  ratio_MextraMatchedHit[1]  \\\n",
       "0                   8.568041                   5.452831   \n",
       "1                   5.836899                   3.697998   \n",
       "2                  15.471354                   9.812719   \n",
       "3                   9.402639                   6.001297   \n",
       "4                   3.857346                   2.445352   \n",
       "\n",
       "   ratio_MextraMatchedHit[2]  ratio_MextraMatchedHit[3]  \n",
       "0                   5.923010                   4.591995  \n",
       "1                   3.890070                   2.942780  \n",
       "2                  10.688360                   8.330187  \n",
       "3                   6.386638                   4.884344  \n",
       "4                   2.635871                   2.025441  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"ratio_MextraMatchedHit\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Еще один момент"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def momentum(df):\n",
    "    mask = np.isclose(df.loc[:, ['P', 'PT']], NA_VALUE).any(axis=1)\n",
    "    df['PO'] = NA_VALUE\n",
    "    df.loc[~mask, 'PO'] = np.power(df.loc[~mask, 'P'], 2.0) - np.power(df.loc[~mask, 'PT'], 2.0)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(momentum, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = momentum(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>P</th>\n",
       "      <th>PT</th>\n",
       "      <th>PO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50268.359507</td>\n",
       "      <td>2589.074895</td>\n",
       "      <td>2.520205e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9136.633410</td>\n",
       "      <td>1214.306768</td>\n",
       "      <td>8.200353e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46241.344045</td>\n",
       "      <td>4021.866299</td>\n",
       "      <td>2.122086e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14855.922836</td>\n",
       "      <td>2900.460154</td>\n",
       "      <td>2.122858e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21442.937297</td>\n",
       "      <td>1915.277178</td>\n",
       "      <td>4.561313e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              P           PT            PO\n",
       "0  50268.359507  2589.074895  2.520205e+09\n",
       "1   9136.633410  1214.306768  8.200353e+07\n",
       "2  46241.344045  4021.866299  2.122086e+09\n",
       "3  14855.922836  2900.460154  2.122858e+08\n",
       "4  21442.937297  1915.277178  4.561313e+08"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, ['P', 'PT', 'PO']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Усредняем дисперсии"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_equals(df):\n",
    "    for n in [0, 1, 2, 3]:\n",
    "        df[f'Mextra_DXY2[{n}]_mean'] = sum([df[f'Mextra_D{k}2[{n}]'] for k in ('X', 'Y')])/2\n",
    "        df[f'MatchedHit_DXYZ[{n}]_mean'] = sum([df[f'MatchedHit_D{k}[{n}]'] for k in ('X', 'Y', 'Z')])/3\n",
    "        df.drop(columns=[f'Mextra_D{k}2[{n}]' for k in ('X', 'Y')], inplace=True)\n",
    "        df.drop(columns=[f'MatchedHit_D{k}[{n}]' for k in ('X', 'Y', 'Z')], inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(mean_equals, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = mean_equals(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Удаляем лишние колонки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import operator\n",
    "from functools import reduce\n",
    "\n",
    "def drop_columns(df):\n",
    "    mask = [df.columns.str.startswith('MatchedHit_X'),\n",
    "            df.columns.str.startswith('MatchedHit_Y'),\n",
    "            df.columns.str.startswith('Lextra_X'),\n",
    "            df.columns.str.startswith('Lextra_Y'),\n",
    "            df.columns.str.startswith('FOI_hits_N['),\n",
    "            df.columns.str.startswith('ndof'),\n",
    "            df.columns.str.startswith('MatchedHit_DT'),\n",
    "            df.columns.str.startswith('MatchedHit_T'),\n",
    "            df.columns.str.endswith('sign'),\n",
    "            df.columns.str.endswith('abs'),\n",
    "            df.columns.str.endswith('delta')]\n",
    "    mask = reduce(operator.or_, mask)\n",
    "    cols = df.columns[mask].tolist()\n",
    "    cols += df.columns[df.columns.str.startswith('find_closest_hit_per_station')].tolist()\n",
    "    \n",
    "    df.drop(columns=cols, inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(drop_columns, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = drop_columns(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предсказание модели"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I know this is incorrect. See it as a low-hanging fruit to beat the baseline. Catboost affirmatively [refused](https://github.com/catboost/catboost/pull/399) to support negative weights. At the same time, its evaluation is [extremly fast](https://catboost.ai/news/best-in-class-inference-and-a-ton-of-speedups), so we are using it as the timing benchmark for Track 2. Feel free to use a patched version with disabled negative weights check."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ncl[0]',\n",
       " 'ncl[1]',\n",
       " 'ncl[2]',\n",
       " 'ncl[3]',\n",
       " 'avg_cs[0]',\n",
       " 'avg_cs[1]',\n",
       " 'avg_cs[2]',\n",
       " 'avg_cs[3]',\n",
       " 'MatchedHit_Z[0]',\n",
       " 'MatchedHit_Z[1]',\n",
       " 'MatchedHit_Z[2]',\n",
       " 'MatchedHit_Z[3]',\n",
       " 'NShared',\n",
       " 'FOI_hits_N',\n",
       " 'PT',\n",
       " 'P',\n",
       " 'MatchedHit_R[0]',\n",
       " 'MatchedHit_R[1]',\n",
       " 'MatchedHit_R[2]',\n",
       " 'MatchedHit_R[3]',\n",
       " 'MatchedHit_A[0]',\n",
       " 'MatchedHit_A[1]',\n",
       " 'MatchedHit_A[2]',\n",
       " 'MatchedHit_A[3]',\n",
       " 'MatchedHit_R[1/0]_frac',\n",
       " 'MatchedHit_R[2/1]_frac',\n",
       " 'MatchedHit_R[3/2]_frac',\n",
       " 'MatchedHit_R[0/3]_frac',\n",
       " 'distance_LextraMatchedHit[0]',\n",
       " 'distance_LextraMatchedHit[1]',\n",
       " 'distance_LextraMatchedHit[2]',\n",
       " 'distance_LextraMatchedHit[3]',\n",
       " 'ratio_MextraMatchedHit[0]',\n",
       " 'ratio_MextraMatchedHit[1]',\n",
       " 'ratio_MextraMatchedHit[2]',\n",
       " 'ratio_MextraMatchedHit[3]',\n",
       " 'PO',\n",
       " 'Mextra_DXY2[0]_mean',\n",
       " 'MatchedHit_DXYZ[0]_mean',\n",
       " 'Mextra_DXY2[1]_mean',\n",
       " 'MatchedHit_DXYZ[1]_mean',\n",
       " 'Mextra_DXY2[2]_mean',\n",
       " 'MatchedHit_DXYZ[2]_mean',\n",
       " 'Mextra_DXY2[3]_mean',\n",
       " 'MatchedHit_DXYZ[3]_mean']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = list(filter(lambda s: s not in utils.TRAIN_COLUMNS, train.columns))\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Заглушка для scaler\n",
    "\n",
    "import copy\n",
    "\n",
    "from sklearn.base import TransformerMixin\n",
    "\n",
    "class IdentityScaler(TransformerMixin):\n",
    "    def __init__(self):        \n",
    "        pass\n",
    "    \n",
    "    def fit(self, X, y=None, **fit_params):        \n",
    "        return self\n",
    "        \n",
    "    def transform(self, X, y=None):\n",
    "        return X\n",
    "    \n",
    "    def fit_transform(self, X, y=None, **fit_params):\n",
    "        return self.transform(X, y)\n",
    "    \n",
    "    def get_params(self, deep=True):\n",
    "        return copy.deepcopy(self.__dict__) if deep else self.__dict__\n",
    "    \n",
    "    def set_params(self, **params):\n",
    "        self.__dict__.update(copy.deepcopy(params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "params = pickle.load(open('models/scaler_params.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in params.keys():\n",
    "    mean, std = params[col]\n",
    "    \n",
    "    scale_mean_std = lambda x: (x - mean) / std\n",
    "#     train[col], test[col] = map(scale_mean_std, [train[col], test[col]])\n",
    "    test[col] = scale_mean_std(test[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_g = [pickle.load(open('models/hepmlp_model_{}.pkl'.format(i), 'rb'))\n",
    "            for i in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = [model.predict_proba(test.loc[:, features].values)[:, 1] for model in models_g]\n",
    "predictions = np.asarray(predictions).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hepmlp_model_0</th>\n",
       "      <th>hepmlp_model_1</th>\n",
       "      <th>hepmlp_model_2</th>\n",
       "      <th>hepmlp_model_3</th>\n",
       "      <th>hepmlp_model_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.836028</td>\n",
       "      <td>0.759051</td>\n",
       "      <td>0.782133</td>\n",
       "      <td>0.749505</td>\n",
       "      <td>0.803459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.699055</td>\n",
       "      <td>0.644392</td>\n",
       "      <td>0.803113</td>\n",
       "      <td>0.756668</td>\n",
       "      <td>0.803238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.644136</td>\n",
       "      <td>0.676205</td>\n",
       "      <td>0.706104</td>\n",
       "      <td>0.424573</td>\n",
       "      <td>0.567216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.706382</td>\n",
       "      <td>0.749088</td>\n",
       "      <td>0.680390</td>\n",
       "      <td>0.745034</td>\n",
       "      <td>0.627922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.821929</td>\n",
       "      <td>0.864923</td>\n",
       "      <td>0.819402</td>\n",
       "      <td>0.744335</td>\n",
       "      <td>0.736508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   hepmlp_model_0  hepmlp_model_1  hepmlp_model_2  hepmlp_model_3  \\\n",
       "0        0.836028        0.759051        0.782133        0.749505   \n",
       "1        0.699055        0.644392        0.803113        0.756668   \n",
       "2        0.644136        0.676205        0.706104        0.424573   \n",
       "3        0.706382        0.749088        0.680390        0.745034   \n",
       "4        0.821929        0.864923        0.819402        0.744335   \n",
       "\n",
       "   hepmlp_model_4  \n",
       "0        0.803459  \n",
       "1        0.803238  \n",
       "2        0.567216  \n",
       "3        0.627922  \n",
       "4        0.736508  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred = pd.DataFrame(predictions,\n",
    "                       columns=[\"hepmlp_model_{}\".format(i) for i in range(predictions.shape[1])],\n",
    "                       index=test.index)\n",
    "df_pred.to_csv(\"preds/private_test_hepmlp_preds.csv\", sep=',', index=False)\n",
    "df_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}