Track1-CatBoostClassifier-predict-public.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import swifter\n",
    "from sklearn.model_selection import train_test_split\n",
    "import utils\n",
    "import catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"./data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/idao_cb/lib/python3.6/site-packages/numpy/lib/arraysetops.py:522: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    }
   ],
   "source": [
    "train, test = utils.load_data_csv(DATA_PATH, utils.SIMPLE_FEATURE_COLUMNS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ncl[0]', 'ncl[1]', 'ncl[2]', 'ncl[3]', 'avg_cs[0]', 'avg_cs[1]',\n",
       "       'avg_cs[2]', 'avg_cs[3]', 'ndof', 'MatchedHit_TYPE[0]',\n",
       "       'MatchedHit_TYPE[1]', 'MatchedHit_TYPE[2]', 'MatchedHit_TYPE[3]',\n",
       "       'MatchedHit_X[0]', 'MatchedHit_X[1]', 'MatchedHit_X[2]',\n",
       "       'MatchedHit_X[3]', 'MatchedHit_Y[0]', 'MatchedHit_Y[1]',\n",
       "       'MatchedHit_Y[2]', 'MatchedHit_Y[3]', 'MatchedHit_Z[0]',\n",
       "       'MatchedHit_Z[1]', 'MatchedHit_Z[2]', 'MatchedHit_Z[3]',\n",
       "       'MatchedHit_DX[0]', 'MatchedHit_DX[1]', 'MatchedHit_DX[2]',\n",
       "       'MatchedHit_DX[3]', 'MatchedHit_DY[0]', 'MatchedHit_DY[1]',\n",
       "       'MatchedHit_DY[2]', 'MatchedHit_DY[3]', 'MatchedHit_DZ[0]',\n",
       "       'MatchedHit_DZ[1]', 'MatchedHit_DZ[2]', 'MatchedHit_DZ[3]',\n",
       "       'MatchedHit_T[0]', 'MatchedHit_T[1]', 'MatchedHit_T[2]',\n",
       "       'MatchedHit_T[3]', 'MatchedHit_DT[0]', 'MatchedHit_DT[1]',\n",
       "       'MatchedHit_DT[2]', 'MatchedHit_DT[3]', 'Lextra_X[0]', 'Lextra_X[1]',\n",
       "       'Lextra_X[2]', 'Lextra_X[3]', 'Lextra_Y[0]', 'Lextra_Y[1]',\n",
       "       'Lextra_Y[2]', 'Lextra_Y[3]', 'NShared', 'Mextra_DX2[0]',\n",
       "       'Mextra_DX2[1]', 'Mextra_DX2[2]', 'Mextra_DX2[3]', 'Mextra_DY2[0]',\n",
       "       'Mextra_DY2[1]', 'Mextra_DY2[2]', 'Mextra_DY2[3]', 'FOI_hits_N', 'PT',\n",
       "       'P', 'label', 'weight'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "func = utils.find_closest_hit_per_station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dt_train = pd.read_hdf('closest_hits_features.train.filled.p1000.hdf', 'key')\n",
    "# dt_test  = pd.read_hdf('closest_hits_features.test.filled.p1000.hdf', 'key')\n",
    "\n",
    "dt_train = pd.read_hdf('closest_hits_features.train.filled.m9999.v1.hdf', 'key')\n",
    "dt_test  = pd.read_hdf('closest_hits_features.test.filled.m9999.v1.hdf', 'key')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_train.columns = [func.__name__ + '_' + str(c) if str(c).isnumeric() else c for c in dt_train.columns]\n",
    "dt_test.columns  = [func.__name__ + '_' + str(c) if str(c).isnumeric() else c for c in dt_test.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train, dt_train], axis=1, copy=False)\n",
    "test = pd.concat([test, dt_test], axis=1, copy=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5445705, 115), (726095, 113))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'label', 'weight'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(train.columns.tolist()).symmetric_difference(test.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Новые признаки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "NA_VALUE = -9999.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fillna(df):\n",
    "    \"\"\"\n",
    "    Пропуски только в 2 и 3 колонках\n",
    "    \"\"\"\n",
    "    \n",
    "    for axis in 'XYZ':\n",
    "        cols = ['MatchedHit_{}[{}]'.format(axis, i) for i in range(4)]\n",
    "        mask = np.isclose(df.loc[:, cols[2]], NA_VALUE)\n",
    "        mask = mask & ~np.isclose(df.loc[:, cols[3]], NA_VALUE)\n",
    "        \n",
    "        df.loc[mask, cols[2]] = df.loc[mask, [cols[1], cols[3]]].mean(axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(fillna, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = fillna(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Вычесть среднее из MatchedHit_Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MatchedHit_Z[0]    15256.8700\n",
       "MatchedHit_Z[1]    16363.0635\n",
       "MatchedHit_Z[2]    17555.9910\n",
       "MatchedHit_Z[3]    18764.2210\n",
       "dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matched_hit_z_cols = train.columns.str.startswith(\"MatchedHit_Z\")\n",
    "mask = np.isclose(train.loc[:, matched_hit_z_cols], NA_VALUE).any(axis=0)\n",
    "matched_hit_z_mean = train.loc[~mask, matched_hit_z_cols].mean(axis=0)\n",
    "matched_hit_z_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MatchedHit_Z[0]    15270\n",
       "MatchedHit_Z[1]    16470\n",
       "MatchedHit_Z[2]    17670\n",
       "MatchedHit_Z[3]    18870\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matched_hit_z_cols = train.columns[train.columns.str.startswith(\"MatchedHit_Z\")]\n",
    "matched_hit_z_mean = pd.Series([15270, 16470, 17670, 18870], index=matched_hit_z_cols)\n",
    "matched_hit_z_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_unbiased_z(df):\n",
    "    for i in range(4):\n",
    "        distance = np.zeros(df.shape[0])\n",
    "    \n",
    "        col = f\"MatchedHit_Z[{i}]\"\n",
    "        mask = np.isclose(df.loc[:, col], NA_VALUE)\n",
    "        distance[~mask] = df.loc[~mask, col] - matched_hit_z_mean[col]\n",
    "        distance[mask] = NA_VALUE\n",
    "    \n",
    "        df[col] = distance\n",
    "        \n",
    "        \"\"\"\n",
    "        df[f\"MatchedHit_Z[{i}]_sign\"] = np.sign(df[col]).astype(np.int8)\n",
    "        \"\"\"\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(make_unbiased_z, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = make_unbiased_z(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_Z[0]</th>\n",
       "      <th>MatchedHit_Z[1]</th>\n",
       "      <th>MatchedHit_Z[2]</th>\n",
       "      <th>MatchedHit_Z[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>126.441</td>\n",
       "      <td>-65.305</td>\n",
       "      <td>-72.188</td>\n",
       "      <td>-63.771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-152.701</td>\n",
       "      <td>-148.568</td>\n",
       "      <td>-155.830</td>\n",
       "      <td>-147.787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-157.764</td>\n",
       "      <td>-154.032</td>\n",
       "      <td>-161.697</td>\n",
       "      <td>-154.055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-150.160</td>\n",
       "      <td>-145.826</td>\n",
       "      <td>-152.887</td>\n",
       "      <td>56.592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-66.947</td>\n",
       "      <td>-62.676</td>\n",
       "      <td>-69.800</td>\n",
       "      <td>-61.620</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_Z[0]  MatchedHit_Z[1]  MatchedHit_Z[2]  MatchedHit_Z[3]\n",
       "0          126.441          -65.305          -72.188          -63.771\n",
       "1         -152.701         -148.568         -155.830         -147.787\n",
       "2         -157.764         -154.032         -161.697         -154.055\n",
       "3         -150.160         -145.826         -152.887           56.592\n",
       "4          -66.947          -62.676          -69.800          -61.620"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_Z\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Расстояние до центра (+)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_polar_distances_matched_hit(df):\n",
    "    for i in range(4):\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "    \n",
    "        name = f\"MatchedHit_R[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = np.log(1.0 + np.power(df.loc[~mask, cols], 2).sum(axis=1))\n",
    "        \n",
    "        name = f\"MatchedHit_A[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = np.arctan2(df.loc[~mask, cols[1]], df.loc[~mask, cols[0]])\n",
    "\n",
    "    # намеренное переупорядочивание признаков\n",
    "    cols  = df.columns[df.columns.str.startswith(\"MatchedHit_R\")].tolist()\n",
    "    cols += df.columns[df.columns.str.startswith(\"MatchedHit_A\")].tolist()\n",
    "    cols  = df.columns[:-len(cols)].tolist() + cols\n",
    "    df = df[cols]\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(compute_polar_distances_matched_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = compute_polar_distances_matched_hit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_R[0]</th>\n",
       "      <th>MatchedHit_R[1]</th>\n",
       "      <th>MatchedHit_R[2]</th>\n",
       "      <th>MatchedHit_R[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15.045311</td>\n",
       "      <td>15.028344</td>\n",
       "      <td>15.170549</td>\n",
       "      <td>15.302802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13.993879</td>\n",
       "      <td>14.148052</td>\n",
       "      <td>14.229975</td>\n",
       "      <td>14.363091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16.444550</td>\n",
       "      <td>16.597730</td>\n",
       "      <td>16.726557</td>\n",
       "      <td>16.858928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12.540267</td>\n",
       "      <td>12.700585</td>\n",
       "      <td>12.846768</td>\n",
       "      <td>13.174559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12.503433</td>\n",
       "      <td>12.698478</td>\n",
       "      <td>12.888758</td>\n",
       "      <td>13.025454</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_R[0]  MatchedHit_R[1]  MatchedHit_R[2]  MatchedHit_R[3]\n",
       "0        15.045311        15.028344        15.170549        15.302802\n",
       "1        13.993879        14.148052        14.229975        14.363091\n",
       "2        16.444550        16.597730        16.726557        16.858928\n",
       "3        12.540267        12.700585        12.846768        13.174559\n",
       "4        12.503433        12.698478        12.888758        13.025454"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_R\")].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_A[0]</th>\n",
       "      <th>MatchedHit_A[1]</th>\n",
       "      <th>MatchedHit_A[2]</th>\n",
       "      <th>MatchedHit_A[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.582870</td>\n",
       "      <td>-1.563293</td>\n",
       "      <td>-1.540319</td>\n",
       "      <td>-1.539549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2.936887</td>\n",
       "      <td>-2.936216</td>\n",
       "      <td>-2.930155</td>\n",
       "      <td>-2.931664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2.688974</td>\n",
       "      <td>-2.688812</td>\n",
       "      <td>-2.685729</td>\n",
       "      <td>-2.686143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.986865</td>\n",
       "      <td>1.999996</td>\n",
       "      <td>2.003493</td>\n",
       "      <td>1.986901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.606379</td>\n",
       "      <td>2.621242</td>\n",
       "      <td>2.634569</td>\n",
       "      <td>2.633024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_A[0]  MatchedHit_A[1]  MatchedHit_A[2]  MatchedHit_A[3]\n",
       "0        -1.582870        -1.563293        -1.540319        -1.539549\n",
       "1        -2.936887        -2.936216        -2.930155        -2.931664\n",
       "2        -2.688974        -2.688812        -2.685729        -2.686143\n",
       "3         1.986865         1.999996         2.003493         1.986901\n",
       "4         2.606379         2.621242         2.634569         2.633024"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_A\")].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_polar_distances_lextra(df):\n",
    "    for i in range(4):\n",
    "        cols = [f\"Lextra_X[{i}]\", f\"Lextra_Y[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "    \n",
    "        name = f\"Lextra_R[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = np.log(1.0 + np.power(df.loc[~mask, cols], 2).sum(axis=1))\n",
    "\n",
    "        \"\"\"\n",
    "        name = f\"Lextra_A[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = np.arctan2(df.loc[~mask, cols[1]], df.loc[~mask, cols[0]])\n",
    "        \"\"\"\n",
    "    \n",
    "    # намеренное переупорядочивание признаков\n",
    "    cols  = df.columns[df.columns.str.startswith(\"Lextra_R\")].tolist()\n",
    "    cols += df.columns[df.columns.str.startswith(\"Lextra_A\")].tolist()\n",
    "    cols  = df.columns[:-len(cols)].tolist() + cols\n",
    "    df = df[cols]\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(compute_polar_distances_lextra, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/idao_cb/lib/python3.6/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/root/anaconda3/envs/idao_cb/lib/python3.6/site-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "train = compute_polar_distances_lextra(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lextra_R[0]</th>\n",
       "      <th>Lextra_R[1]</th>\n",
       "      <th>Lextra_R[2]</th>\n",
       "      <th>Lextra_R[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.988222</td>\n",
       "      <td>15.137619</td>\n",
       "      <td>15.279196</td>\n",
       "      <td>15.413349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14.021024</td>\n",
       "      <td>14.156076</td>\n",
       "      <td>14.282589</td>\n",
       "      <td>14.401579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16.462706</td>\n",
       "      <td>16.625766</td>\n",
       "      <td>16.776542</td>\n",
       "      <td>16.916753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12.603785</td>\n",
       "      <td>12.783881</td>\n",
       "      <td>12.950539</td>\n",
       "      <td>13.105415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12.490319</td>\n",
       "      <td>12.668763</td>\n",
       "      <td>12.832673</td>\n",
       "      <td>12.984229</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lextra_R[0]  Lextra_R[1]  Lextra_R[2]  Lextra_R[3]\n",
       "0    14.988222    15.137619    15.279196    15.413349\n",
       "1    14.021024    14.156076    14.282589    14.401579\n",
       "2    16.462706    16.625766    16.776542    16.916753\n",
       "3    12.603785    12.783881    12.950539    13.105415\n",
       "4    12.490319    12.668763    12.832673    12.984229"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"Lextra_R\")].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: [0, 1, 2, 3, 4]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"Lextra_A\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Дельты для траекторий MatchedHit_X, MatchedHit_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def diff_matched_hit(df):\n",
    "    for axis in ['X', 'Y']:\n",
    "        for i, j in zip([0, 1, 2, 3], [1, 2, 3, 0]):\n",
    "            col_i = f\"MatchedHit_{axis}[{i}]\"\n",
    "            col_j = f\"MatchedHit_{axis}[{j}]\"\n",
    "            mask = np.isclose(df.loc[:, [col_i, col_j]], NA_VALUE).any(axis=1)\n",
    "\n",
    "            name = f\"MatchedHit_{axis}[{j}/{i}]_delta\"\n",
    "            df[name] = df[name + '_abs'] = NA_VALUE\n",
    "            \n",
    "            df.loc[~mask, name] = df.loc[~mask, col_j] - df.loc[~mask, col_i]\n",
    "            df.loc[~mask, name + '_abs'] = np.abs(df.loc[~mask, name])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(diff_matched_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = diff_matched_hit(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Дельты и отношения для радиусов (+)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ratio_radius_matched_hit(df):\n",
    "    for i, j in zip([0, 1, 2, 3], [1, 2, 3, 0]):\n",
    "        col_i = f\"MatchedHit_R[{i}]\"\n",
    "        col_j = f\"MatchedHit_R[{j}]\"\n",
    "        mask = np.isclose(df.loc[:, [col_i, col_j]], NA_VALUE).any(axis=1)\n",
    "\n",
    "        name = f\"MatchedHit_R[{j}/{i}]_delta\"\n",
    "        name_abs = name + '_abs'\n",
    "        df[name] = df[name_abs] = NA_VALUE\n",
    "        \n",
    "        df.loc[~mask, name] = df.loc[~mask, col_j] - df.loc[~mask, col_i]\n",
    "        df.loc[~mask, name_abs] = np.abs(df.loc[~mask, name])\n",
    "        \n",
    "        name = f\"MatchedHit_R[{j}/{i}]_frac\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = df.loc[~mask, col_j] / df.loc[~mask, col_i]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(ratio_radius_matched_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ratio_radius_matched_hit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_R[1/0]_delta</th>\n",
       "      <th>MatchedHit_R[1/0]_delta_abs</th>\n",
       "      <th>MatchedHit_R[1/0]_frac</th>\n",
       "      <th>MatchedHit_R[2/1]_delta</th>\n",
       "      <th>MatchedHit_R[2/1]_delta_abs</th>\n",
       "      <th>MatchedHit_R[2/1]_frac</th>\n",
       "      <th>MatchedHit_R[3/2]_delta</th>\n",
       "      <th>MatchedHit_R[3/2]_delta_abs</th>\n",
       "      <th>MatchedHit_R[3/2]_frac</th>\n",
       "      <th>MatchedHit_R[0/3]_delta</th>\n",
       "      <th>MatchedHit_R[0/3]_delta_abs</th>\n",
       "      <th>MatchedHit_R[0/3]_frac</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.016967</td>\n",
       "      <td>0.016967</td>\n",
       "      <td>0.998872</td>\n",
       "      <td>0.142205</td>\n",
       "      <td>0.142205</td>\n",
       "      <td>1.009462</td>\n",
       "      <td>0.132253</td>\n",
       "      <td>0.132253</td>\n",
       "      <td>1.008718</td>\n",
       "      <td>-0.257491</td>\n",
       "      <td>0.257491</td>\n",
       "      <td>0.983174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.154173</td>\n",
       "      <td>0.154173</td>\n",
       "      <td>1.011017</td>\n",
       "      <td>0.081923</td>\n",
       "      <td>0.081923</td>\n",
       "      <td>1.005790</td>\n",
       "      <td>0.133116</td>\n",
       "      <td>0.133116</td>\n",
       "      <td>1.009355</td>\n",
       "      <td>-0.369211</td>\n",
       "      <td>0.369211</td>\n",
       "      <td>0.974294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.153180</td>\n",
       "      <td>0.153180</td>\n",
       "      <td>1.009315</td>\n",
       "      <td>0.128827</td>\n",
       "      <td>0.128827</td>\n",
       "      <td>1.007762</td>\n",
       "      <td>0.132371</td>\n",
       "      <td>0.132371</td>\n",
       "      <td>1.007914</td>\n",
       "      <td>-0.414378</td>\n",
       "      <td>0.414378</td>\n",
       "      <td>0.975421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.160318</td>\n",
       "      <td>0.160318</td>\n",
       "      <td>1.012784</td>\n",
       "      <td>0.146183</td>\n",
       "      <td>0.146183</td>\n",
       "      <td>1.011510</td>\n",
       "      <td>0.327791</td>\n",
       "      <td>0.327791</td>\n",
       "      <td>1.025515</td>\n",
       "      <td>-0.634292</td>\n",
       "      <td>0.634292</td>\n",
       "      <td>0.951855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.195045</td>\n",
       "      <td>0.195045</td>\n",
       "      <td>1.015599</td>\n",
       "      <td>0.190280</td>\n",
       "      <td>0.190280</td>\n",
       "      <td>1.014984</td>\n",
       "      <td>0.136696</td>\n",
       "      <td>0.136696</td>\n",
       "      <td>1.010606</td>\n",
       "      <td>-0.522021</td>\n",
       "      <td>0.522021</td>\n",
       "      <td>0.959923</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_R[1/0]_delta  MatchedHit_R[1/0]_delta_abs  \\\n",
       "0                -0.016967                     0.016967   \n",
       "1                 0.154173                     0.154173   \n",
       "2                 0.153180                     0.153180   \n",
       "3                 0.160318                     0.160318   \n",
       "4                 0.195045                     0.195045   \n",
       "\n",
       "   MatchedHit_R[1/0]_frac  MatchedHit_R[2/1]_delta  \\\n",
       "0                0.998872                 0.142205   \n",
       "1                1.011017                 0.081923   \n",
       "2                1.009315                 0.128827   \n",
       "3                1.012784                 0.146183   \n",
       "4                1.015599                 0.190280   \n",
       "\n",
       "   MatchedHit_R[2/1]_delta_abs  MatchedHit_R[2/1]_frac  \\\n",
       "0                     0.142205                1.009462   \n",
       "1                     0.081923                1.005790   \n",
       "2                     0.128827                1.007762   \n",
       "3                     0.146183                1.011510   \n",
       "4                     0.190280                1.014984   \n",
       "\n",
       "   MatchedHit_R[3/2]_delta  MatchedHit_R[3/2]_delta_abs  \\\n",
       "0                 0.132253                     0.132253   \n",
       "1                 0.133116                     0.133116   \n",
       "2                 0.132371                     0.132371   \n",
       "3                 0.327791                     0.327791   \n",
       "4                 0.136696                     0.136696   \n",
       "\n",
       "   MatchedHit_R[3/2]_frac  MatchedHit_R[0/3]_delta  \\\n",
       "0                1.008718                -0.257491   \n",
       "1                1.009355                -0.369211   \n",
       "2                1.007914                -0.414378   \n",
       "3                1.025515                -0.634292   \n",
       "4                1.010606                -0.522021   \n",
       "\n",
       "   MatchedHit_R[0/3]_delta_abs  MatchedHit_R[0/3]_frac  \n",
       "0                     0.257491                0.983174  \n",
       "1                     0.369211                0.974294  \n",
       "2                     0.414378                0.975421  \n",
       "3                     0.634292                0.951855  \n",
       "4                     0.522021                0.959923  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = filter(lambda s: s.startswith(\"MatchedHit_R\") and (\"_delta\" in s or \"_frac\" in s), train.columns)\n",
    "train.loc[:, cols].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ratio_radius_lextra_hit(df):\n",
    "    for i, j in zip([0, 1, 2, 3], [1, 2, 3, 0]):\n",
    "        col_i = f\"Lextra_R[{i}]\"\n",
    "        col_j = f\"Lextra_R[{j}]\"\n",
    "        mask = np.isclose(df.loc[:, [col_i, col_j]], NA_VALUE).any(axis=1)\n",
    "\n",
    "        name = f\"Lextra_R[{j}/{i}]_delta\"\n",
    "        name_abs = name + '_abs'\n",
    "        df[name] = df[name_abs] = NA_VALUE\n",
    "        \n",
    "        df.loc[~mask, name] = df.loc[~mask, col_j] - df.loc[~mask, col_i]\n",
    "        df.loc[~mask, name_abs] = np.abs(df.loc[~mask, name])\n",
    "        \n",
    "#         name = f\"Lextra_R[{j}/{i}]_frac\"\n",
    "#         df[name] = NA_VALUE\n",
    "#         df.loc[~mask, name] = df.loc[~mask, col_j] / df.loc[~mask, col_i]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(ratio_radius_lextra_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ratio_radius_lextra_hit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lextra_R[1/0]_delta</th>\n",
       "      <th>Lextra_R[1/0]_delta_abs</th>\n",
       "      <th>Lextra_R[2/1]_delta</th>\n",
       "      <th>Lextra_R[2/1]_delta_abs</th>\n",
       "      <th>Lextra_R[3/2]_delta</th>\n",
       "      <th>Lextra_R[3/2]_delta_abs</th>\n",
       "      <th>Lextra_R[0/3]_delta</th>\n",
       "      <th>Lextra_R[0/3]_delta_abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.149397</td>\n",
       "      <td>0.149397</td>\n",
       "      <td>0.141576</td>\n",
       "      <td>0.141576</td>\n",
       "      <td>0.134154</td>\n",
       "      <td>0.134154</td>\n",
       "      <td>-0.425127</td>\n",
       "      <td>0.425127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.135052</td>\n",
       "      <td>0.135052</td>\n",
       "      <td>0.126513</td>\n",
       "      <td>0.126513</td>\n",
       "      <td>0.118990</td>\n",
       "      <td>0.118990</td>\n",
       "      <td>-0.380556</td>\n",
       "      <td>0.380556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.163060</td>\n",
       "      <td>0.163060</td>\n",
       "      <td>0.150776</td>\n",
       "      <td>0.150776</td>\n",
       "      <td>0.140211</td>\n",
       "      <td>0.140211</td>\n",
       "      <td>-0.454047</td>\n",
       "      <td>0.454047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.180096</td>\n",
       "      <td>0.180096</td>\n",
       "      <td>0.166658</td>\n",
       "      <td>0.166658</td>\n",
       "      <td>0.154875</td>\n",
       "      <td>0.154875</td>\n",
       "      <td>-0.501630</td>\n",
       "      <td>0.501630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.178443</td>\n",
       "      <td>0.178443</td>\n",
       "      <td>0.163911</td>\n",
       "      <td>0.163911</td>\n",
       "      <td>0.151556</td>\n",
       "      <td>0.151556</td>\n",
       "      <td>-0.493910</td>\n",
       "      <td>0.493910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Lextra_R[1/0]_delta  Lextra_R[1/0]_delta_abs  Lextra_R[2/1]_delta  \\\n",
       "0             0.149397                 0.149397             0.141576   \n",
       "1             0.135052                 0.135052             0.126513   \n",
       "2             0.163060                 0.163060             0.150776   \n",
       "3             0.180096                 0.180096             0.166658   \n",
       "4             0.178443                 0.178443             0.163911   \n",
       "\n",
       "   Lextra_R[2/1]_delta_abs  Lextra_R[3/2]_delta  Lextra_R[3/2]_delta_abs  \\\n",
       "0                 0.141576             0.134154                 0.134154   \n",
       "1                 0.126513             0.118990                 0.118990   \n",
       "2                 0.150776             0.140211                 0.140211   \n",
       "3                 0.166658             0.154875                 0.154875   \n",
       "4                 0.163911             0.151556                 0.151556   \n",
       "\n",
       "   Lextra_R[0/3]_delta  Lextra_R[0/3]_delta_abs  \n",
       "0            -0.425127                 0.425127  \n",
       "1            -0.380556                 0.380556  \n",
       "2            -0.454047                 0.454047  \n",
       "3            -0.501630                 0.501630  \n",
       "4            -0.493910                 0.493910  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = filter(lambda s: s.startswith(\"Lextra_R\") and (\"_delta\" in s or \"_frac\" in s), train.columns)\n",
    "train.loc[:, cols].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Расстояние между Lextra и MatchedHit (+)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distance_dist_lextra_matched(df):\n",
    "    distance_counts = np.zeros(df.shape[0], dtype=int)\n",
    "    \n",
    "    distance_se = np.zeros(df.shape[0])\n",
    "    distance_ae = np.zeros(df.shape[0])\n",
    "    \n",
    "    for i in range(4):\n",
    "        distance = np.zeros(df.shape[0])\n",
    "\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\", f\"Lextra_X[{i}]\", f\"Lextra_Y[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "\n",
    "        mse = df.loc[~mask, cols[:2]].values - df.loc[~mask, cols[2:]].values\n",
    "        mse = np.power(mse, 2.0).sum(axis=1)\n",
    "        \n",
    "        mae = df.loc[~mask, cols[:2]].values - df.loc[~mask, cols[2:]].values\n",
    "        mae = np.abs(mae).max(axis=1)\n",
    "\n",
    "        distance[~mask] = np.log(1.0 + mse)\n",
    "        distance[mask] = NA_VALUE\n",
    "        \n",
    "        distance_se[~mask] += mse\n",
    "        distance_ae[~mask] = np.maximum(distance_ae[~mask], mae)\n",
    "        distance_counts[~mask] += 1\n",
    "\n",
    "        df[f\"distance_LextraMatchedHit[{i}]\"] = distance\n",
    "\n",
    "    distance_se /= distance_counts\n",
    "    df[f\"distance_LextraMatchedHit_SE\"] = distance_se\n",
    "    df[f\"distance_LextraMatchedHit_AE\"] = distance_ae\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(distance_dist_lextra_matched, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = distance_dist_lextra_matched(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distance_LextraMatchedHit[0]</th>\n",
       "      <th>distance_LextraMatchedHit[1]</th>\n",
       "      <th>distance_LextraMatchedHit[2]</th>\n",
       "      <th>distance_LextraMatchedHit[3]</th>\n",
       "      <th>distance_LextraMatchedHit_SE</th>\n",
       "      <th>distance_LextraMatchedHit_AE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.905678</td>\n",
       "      <td>9.410019</td>\n",
       "      <td>9.709775</td>\n",
       "      <td>10.382541</td>\n",
       "      <td>15921.890431</td>\n",
       "      <td>141.38583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.149215</td>\n",
       "      <td>7.209501</td>\n",
       "      <td>7.683212</td>\n",
       "      <td>7.697716</td>\n",
       "      <td>1749.054697</td>\n",
       "      <td>44.06910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.281535</td>\n",
       "      <td>8.339558</td>\n",
       "      <td>9.620339</td>\n",
       "      <td>10.037943</td>\n",
       "      <td>10895.470803</td>\n",
       "      <td>151.18740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.786357</td>\n",
       "      <td>6.766075</td>\n",
       "      <td>7.648479</td>\n",
       "      <td>8.442012</td>\n",
       "      <td>1981.263462</td>\n",
       "      <td>50.28820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.865971</td>\n",
       "      <td>4.313771</td>\n",
       "      <td>5.827522</td>\n",
       "      <td>5.261681</td>\n",
       "      <td>155.152357</td>\n",
       "      <td>18.10010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distance_LextraMatchedHit[0]  distance_LextraMatchedHit[1]  \\\n",
       "0                      7.905678                      9.410019   \n",
       "1                      7.149215                      7.209501   \n",
       "2                      7.281535                      8.339558   \n",
       "3                      5.786357                      6.766075   \n",
       "4                      2.865971                      4.313771   \n",
       "\n",
       "   distance_LextraMatchedHit[2]  distance_LextraMatchedHit[3]  \\\n",
       "0                      9.709775                     10.382541   \n",
       "1                      7.683212                      7.697716   \n",
       "2                      9.620339                     10.037943   \n",
       "3                      7.648479                      8.442012   \n",
       "4                      5.827522                      5.261681   \n",
       "\n",
       "   distance_LextraMatchedHit_SE  distance_LextraMatchedHit_AE  \n",
       "0                  15921.890431                     141.38583  \n",
       "1                   1749.054697                      44.06910  \n",
       "2                  10895.470803                     151.18740  \n",
       "3                   1981.263462                      50.28820  \n",
       "4                    155.152357                      18.10010  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"distance_LextraMatchedHit\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Абсолютное и относительное смещение"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def offset_matched_hit(df):\n",
    "    offset_abs = np.sqrt(np.power(df.loc[:, [\"MatchedHit_X[0]\", f\"MatchedHit_Y[0]\"]], 2).sum(axis=1))\n",
    "    \n",
    "    for i, j in zip([0, 1, 2], [1, 2, 3]):\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\", f\"MatchedHit_X[{j}]\", f\"MatchedHit_Y[{j}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "        \n",
    "        mse = df.loc[~mask, cols[:2]].values - df.loc[~mask, cols[2:]].values\n",
    "        mse = np.sqrt(np.power(mse, 2.0).sum(axis=1))\n",
    "        offset_abs[~mask] += mse\n",
    "        \n",
    "    offset_from = df.loc[:, [\"MatchedHit_X[0]\", f\"MatchedHit_Y[0]\"]].values\n",
    "    offset_to = np.zeros(shape=(df.shape[0], 2))\n",
    "    \n",
    "    for i in range(4):\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "        offset_to[~mask] = df.loc[~mask, cols].values\n",
    "    \n",
    "    offset_rel = np.sqrt(np.power(offset_from - offset_to, 2.0).sum(axis=1))\n",
    "    \n",
    "    df[\"MatchedHit_offset_abs\"] = offset_abs\n",
    "    df[\"MatchedHit_offset_rel\"] = offset_rel\n",
    "    df[\"MatchedHit_offset_rel_div_abs\"] = offset_rel / offset_abs\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(offset_matched_hit, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = offset_matched_hit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MatchedHit_offset_abs</th>\n",
       "      <th>MatchedHit_offset_rel</th>\n",
       "      <th>MatchedHit_offset_rel_div_abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2165.393319</td>\n",
       "      <td>268.098049</td>\n",
       "      <td>0.123810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1315.500803</td>\n",
       "      <td>221.744884</td>\n",
       "      <td>0.168563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4580.357699</td>\n",
       "      <td>857.161787</td>\n",
       "      <td>0.187139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>726.999869</td>\n",
       "      <td>197.256114</td>\n",
       "      <td>0.271329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>674.842028</td>\n",
       "      <td>155.558226</td>\n",
       "      <td>0.230511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MatchedHit_offset_abs  MatchedHit_offset_rel  MatchedHit_offset_rel_div_abs\n",
       "0            2165.393319             268.098049                       0.123810\n",
       "1            1315.500803             221.744884                       0.168563\n",
       "2            4580.357699             857.161787                       0.187139\n",
       "3             726.999869             197.256114                       0.271329\n",
       "4             674.842028             155.558226                       0.230511"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"MatchedHit_offset_\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Отношение дисперсий"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ratio_mextra_matched(df):    \n",
    "    for i in range(4):\n",
    "        cols = [f\"MatchedHit_DX[{i}]\", f\"MatchedHit_DY[{i}]\", f\"Mextra_DX2[{i}]\", f\"Mextra_DY2[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "\n",
    "        tmpX = df.loc[~mask, cols[::2]]\n",
    "        tmpY = df.loc[~mask, cols[1::2]]\n",
    "        \n",
    "        ratio = tmpX.iloc[:, 0] / np.sqrt(tmpX.iloc[:, 1]) + \\\n",
    "                tmpY.iloc[:, 0] / np.sqrt(tmpY.iloc[:, 1])\n",
    "\n",
    "        name = f\"ratio_MextraMatchedHit[{i}]\"\n",
    "        df[name] = NA_VALUE\n",
    "        df.loc[~mask, name] = ratio\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(ratio_mextra_matched, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ratio_mextra_matched(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ratio_MextraMatchedHit[0]</th>\n",
       "      <th>ratio_MextraMatchedHit[1]</th>\n",
       "      <th>ratio_MextraMatchedHit[2]</th>\n",
       "      <th>ratio_MextraMatchedHit[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.180889</td>\n",
       "      <td>1.391761</td>\n",
       "      <td>1.516274</td>\n",
       "      <td>1.159187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.071184</td>\n",
       "      <td>5.123200</td>\n",
       "      <td>5.566834</td>\n",
       "      <td>4.322200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13.092113</td>\n",
       "      <td>8.331748</td>\n",
       "      <td>8.906781</td>\n",
       "      <td>6.837303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.210887</td>\n",
       "      <td>1.401526</td>\n",
       "      <td>1.520892</td>\n",
       "      <td>2.348935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.097199</td>\n",
       "      <td>3.263563</td>\n",
       "      <td>3.561245</td>\n",
       "      <td>2.762397</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ratio_MextraMatchedHit[0]  ratio_MextraMatchedHit[1]  \\\n",
       "0                   2.180889                   1.391761   \n",
       "1                   8.071184                   5.123200   \n",
       "2                  13.092113                   8.331748   \n",
       "3                   2.210887                   1.401526   \n",
       "4                   5.097199                   3.263563   \n",
       "\n",
       "   ratio_MextraMatchedHit[2]  ratio_MextraMatchedHit[3]  \n",
       "0                   1.516274                   1.159187  \n",
       "1                   5.566834                   4.322200  \n",
       "2                   8.906781                   6.837303  \n",
       "3                   1.520892                   2.348935  \n",
       "4                   3.561245                   2.762397  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith(\"ratio_MextraMatchedHit\")].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Левые и правые границы для Lextra (- не влезает в память)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigma_lextra_hit(df):\n",
    "    for axis in ['X', 'Y']:\n",
    "        for i in range(0, 4):\n",
    "            col_i = f\"Lextra_{axis}[{i}]\"\n",
    "            col_j = f\"Mextra_D{axis}2[{i}]\"\n",
    "            mask = np.isclose(df.loc[:, [col_i, col_j]], NA_VALUE).any(axis=1)\n",
    "\n",
    "            for col, factor in zip([f\"Lextra_{axis}[{i}]_minus_sigma2\",\n",
    "                                    f\"Lextra_{axis}[{i}]_minus_sigma\",\n",
    "                                    f\"Lextra_{axis}[{i}]_plus_sigma\",\n",
    "                                    f\"Lextra_{axis}[{i}]_plus_sigma2\"],\n",
    "                                   [-2, -1, 1, 2]):\n",
    "                df[col] = NA_VALUE\n",
    "                df.loc[~mask, col] = df.loc[~mask, col_i] + factor * np.sqrt(df.loc[~mask, col_j])\n",
    "        \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Число объектов в кластерах"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cluster_size(df):\n",
    "    for i in range(4):\n",
    "        name = f'cl_size[{i}]'\n",
    "        df[name] = NA_VALUE\n",
    "        \n",
    "        cols = [f'ncl[{i}]', f'avg_cs[{i}]']\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "        df.loc[~mask, name] = df.loc[~mask, cols[0]] * df.loc[~mask, cols[1]]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(cluster_size, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = cluster_size(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cl_size[0]</th>\n",
       "      <th>cl_size[1]</th>\n",
       "      <th>cl_size[2]</th>\n",
       "      <th>cl_size[3]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>117.000004</td>\n",
       "      <td>13.999999</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>116.000003</td>\n",
       "      <td>20.000002</td>\n",
       "      <td>16.000001</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>162.000000</td>\n",
       "      <td>70.000002</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>38.999997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>649.999989</td>\n",
       "      <td>118.000007</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>18.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35.999999</td>\n",
       "      <td>43.999999</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>14.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cl_size[0]  cl_size[1]  cl_size[2]  cl_size[3]\n",
       "0  117.000004   13.999999   15.000000   24.000000\n",
       "1  116.000003   20.000002   16.000001   24.000000\n",
       "2  162.000000   70.000002   23.000000   38.999997\n",
       "3  649.999989  118.000007   22.000000   18.000000\n",
       "4   35.999999   43.999999    8.000000   14.000000"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, train.columns.str.startswith('cl_size')].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Последняя площадка"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def station_latest(df):\n",
    "    station = np.zeros(df.shape[0], dtype=int)\n",
    "    \n",
    "    for i in range(4):\n",
    "        cols = [f\"MatchedHit_X[{i}]\", f\"MatchedHit_Y[{i}]\", f\"MatchedHit_Z[{i}]\"]\n",
    "        mask = np.isclose(df.loc[:, cols], NA_VALUE).any(axis=1)\n",
    "        station[~mask] += 1\n",
    "        \n",
    "    df[\"station_latest\"] = station\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(station_latest, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train.loc[:, \"station_latest\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.unique(train['station_latest'], return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Еще один момент"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def momentum(df):\n",
    "    mask = np.isclose(df.loc[:, ['P', 'PT']], NA_VALUE).any(axis=1)\n",
    "    df['PO'] = NA_VALUE\n",
    "    df.loc[~mask, 'PO'] = np.power(df.loc[~mask, 'P'], 2.0) - np.power(df.loc[~mask, 'PT'], 2.0)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(momentum, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = momentum(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>P</th>\n",
       "      <th>PT</th>\n",
       "      <th>PO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6884.502272</td>\n",
       "      <td>1174.484530</td>\n",
       "      <td>4.601696e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48769.526891</td>\n",
       "      <td>4344.639775</td>\n",
       "      <td>2.359591e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20296.048412</td>\n",
       "      <td>4109.805750</td>\n",
       "      <td>3.950391e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>26062.046122</td>\n",
       "      <td>940.242439</td>\n",
       "      <td>6.783462e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59872.343332</td>\n",
       "      <td>1386.377851</td>\n",
       "      <td>3.582775e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              P           PT            PO\n",
       "0   6884.502272  1174.484530  4.601696e+07\n",
       "1  48769.526891  4344.639775  2.359591e+09\n",
       "2  20296.048412  4109.805750  3.950391e+08\n",
       "3  26062.046122   940.242439  6.783462e+08\n",
       "4  59872.343332  1386.377851  3.582775e+09"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, ['P', 'PT', 'PO']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Отношение NShared к FOI_hits_N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nshared_foihits(df):\n",
    "    df.loc[:, \"frac_NShared_FOI_hits_N\"] = df.loc[:, \"NShared\"] / df.loc[:, \"FOI_hits_N\"] \n",
    "    df.loc[:, \"frac_NShared_FOI_hits_N\"] = df.loc[:, \"frac_NShared_FOI_hits_N\"].fillna(-1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(nshared_foihits, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = nshared_foihits(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NShared</th>\n",
       "      <th>FOI_hits_N</th>\n",
       "      <th>frac_NShared_FOI_hits_N</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   NShared  FOI_hits_N  frac_NShared_FOI_hits_N\n",
       "0        0           6                      0.0\n",
       "1        0           7                      0.0\n",
       "2        0           4                      0.0\n",
       "3        0           9                      0.0\n",
       "4        0           8                      0.0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[:, [\"NShared\", \"FOI_hits_N\", \"frac_NShared_FOI_hits_N\"]].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Усредняем дисперсии"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_std(df):\n",
    "    for station in range(4):\n",
    "        cols = [f'MatchedHit_D{axis}[{station}]' for axis in 'XYZ']\n",
    "        mask = np.isclose(df.loc[:, cols], -1).any(axis=1)\n",
    "        \n",
    "        name = f'MatchedHit_D[{station}]_mean'\n",
    "        df[name] = -1\n",
    "        df.loc[~mask, name] = df.loc[~mask, cols].mean(axis=1)\n",
    "        \n",
    "    for station in range(4):\n",
    "        cols = [f'Mextra_D{axis}2[{station}]' for axis in 'XY']\n",
    "        mask = np.isclose(df.loc[:, cols], -1).any(axis=1)\n",
    "        \n",
    "        name = f'Mextra_D2[{station}]_mean'\n",
    "        df[name] = -1\n",
    "        df.loc[~mask, name] = df.loc[~mask, cols].mean(axis=1)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(mean_std, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = mean_std(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Удаляем лишние колонки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import operator\n",
    "from functools import reduce\n",
    "\n",
    "def drop_columns(df):\n",
    "    mask = [df.columns.str.startswith('MatchedHit_X'),\n",
    "            df.columns.str.startswith('MatchedHit_Y'),\n",
    "            df.columns.str.startswith('Lextra_X'),\n",
    "            df.columns.str.startswith('Lextra_Y'),\n",
    "            df.columns.str.startswith('FOI_hits_N['),\n",
    "            df.columns == \"ndof\",\n",
    "            df.columns.str.startswith('MatchedHit_DX') & ~df.columns.str.endswith('_mean'),\n",
    "            df.columns.str.startswith('MatchedHit_DY') & ~df.columns.str.endswith('_mean'),\n",
    "            df.columns.str.startswith('MatchedHit_DZ') & ~df.columns.str.endswith('_mean'),\n",
    "            df.columns.str.startswith('Mextra_D') & ~df.columns.str.endswith('_mean')]\n",
    "    mask = reduce(operator.or_, mask)\n",
    "    df.drop(columns=df.columns[mask], inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, test = map(drop_columns, [train, test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = drop_columns(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предсказание модели"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I know this is incorrect. See it as a low-hanging fruit to beat the baseline. Catboost affirmatively [refused](https://github.com/catboost/catboost/pull/399) to support negative weights. At the same time, its evaluation is [extremly fast](https://catboost.ai/news/best-in-class-inference-and-a-ton-of-speedups), so we are using it as the timing benchmark for Track 2. Feel free to use a patched version with disabled negative weights check."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ncl[0]',\n",
       " 'ncl[1]',\n",
       " 'ncl[2]',\n",
       " 'ncl[3]',\n",
       " 'avg_cs[0]',\n",
       " 'avg_cs[1]',\n",
       " 'avg_cs[2]',\n",
       " 'avg_cs[3]',\n",
       " 'MatchedHit_TYPE[0]',\n",
       " 'MatchedHit_TYPE[1]',\n",
       " 'MatchedHit_TYPE[2]',\n",
       " 'MatchedHit_TYPE[3]',\n",
       " 'MatchedHit_Z[0]',\n",
       " 'MatchedHit_Z[1]',\n",
       " 'MatchedHit_Z[2]',\n",
       " 'MatchedHit_Z[3]',\n",
       " 'MatchedHit_T[0]',\n",
       " 'MatchedHit_T[1]',\n",
       " 'MatchedHit_T[2]',\n",
       " 'MatchedHit_T[3]',\n",
       " 'MatchedHit_DT[0]',\n",
       " 'MatchedHit_DT[1]',\n",
       " 'MatchedHit_DT[2]',\n",
       " 'MatchedHit_DT[3]',\n",
       " 'NShared',\n",
       " 'FOI_hits_N',\n",
       " 'PT',\n",
       " 'P',\n",
       " 'find_closest_hit_per_station_0',\n",
       " 'find_closest_hit_per_station_1',\n",
       " 'find_closest_hit_per_station_2',\n",
       " 'find_closest_hit_per_station_3',\n",
       " 'find_closest_hit_per_station_4',\n",
       " 'find_closest_hit_per_station_5',\n",
       " 'find_closest_hit_per_station_6',\n",
       " 'find_closest_hit_per_station_7',\n",
       " 'find_closest_hit_per_station_8',\n",
       " 'find_closest_hit_per_station_9',\n",
       " 'find_closest_hit_per_station_10',\n",
       " 'find_closest_hit_per_station_11',\n",
       " 'find_closest_hit_per_station_12',\n",
       " 'find_closest_hit_per_station_13',\n",
       " 'find_closest_hit_per_station_14',\n",
       " 'find_closest_hit_per_station_15',\n",
       " 'find_closest_hit_per_station_16',\n",
       " 'find_closest_hit_per_station_17',\n",
       " 'find_closest_hit_per_station_18',\n",
       " 'find_closest_hit_per_station_19',\n",
       " 'find_closest_hit_per_station_20',\n",
       " 'find_closest_hit_per_station_21',\n",
       " 'find_closest_hit_per_station_22',\n",
       " 'find_closest_hit_per_station_23',\n",
       " 'find_closest_hit_per_station_24',\n",
       " 'find_closest_hit_per_station_25',\n",
       " 'find_closest_hit_per_station_26',\n",
       " 'find_closest_hit_per_station_27',\n",
       " 'find_closest_hit_per_station_28',\n",
       " 'find_closest_hit_per_station_29',\n",
       " 'find_closest_hit_per_station_30',\n",
       " 'find_closest_hit_per_station_31',\n",
       " 'find_closest_hit_per_station_32',\n",
       " 'find_closest_hit_per_station_33',\n",
       " 'find_closest_hit_per_station_34',\n",
       " 'find_closest_hit_per_station_35',\n",
       " 'find_closest_hit_per_station_36',\n",
       " 'find_closest_hit_per_station_37',\n",
       " 'find_closest_hit_per_station_38',\n",
       " 'find_closest_hit_per_station_39',\n",
       " 'find_closest_hit_per_station_40',\n",
       " 'find_closest_hit_per_station_41',\n",
       " 'find_closest_hit_per_station_42',\n",
       " 'find_closest_hit_per_station_43',\n",
       " 'MatchedHit_R[0]',\n",
       " 'MatchedHit_R[1]',\n",
       " 'MatchedHit_R[2]',\n",
       " 'MatchedHit_R[3]',\n",
       " 'MatchedHit_A[0]',\n",
       " 'MatchedHit_A[1]',\n",
       " 'MatchedHit_A[2]',\n",
       " 'MatchedHit_A[3]',\n",
       " 'Lextra_R[0]',\n",
       " 'Lextra_R[1]',\n",
       " 'Lextra_R[2]',\n",
       " 'Lextra_R[3]',\n",
       " 'MatchedHit_R[1/0]_delta',\n",
       " 'MatchedHit_R[1/0]_delta_abs',\n",
       " 'MatchedHit_R[1/0]_frac',\n",
       " 'MatchedHit_R[2/1]_delta',\n",
       " 'MatchedHit_R[2/1]_delta_abs',\n",
       " 'MatchedHit_R[2/1]_frac',\n",
       " 'MatchedHit_R[3/2]_delta',\n",
       " 'MatchedHit_R[3/2]_delta_abs',\n",
       " 'MatchedHit_R[3/2]_frac',\n",
       " 'MatchedHit_R[0/3]_delta',\n",
       " 'MatchedHit_R[0/3]_delta_abs',\n",
       " 'MatchedHit_R[0/3]_frac',\n",
       " 'Lextra_R[1/0]_delta',\n",
       " 'Lextra_R[1/0]_delta_abs',\n",
       " 'Lextra_R[2/1]_delta',\n",
       " 'Lextra_R[2/1]_delta_abs',\n",
       " 'Lextra_R[3/2]_delta',\n",
       " 'Lextra_R[3/2]_delta_abs',\n",
       " 'Lextra_R[0/3]_delta',\n",
       " 'Lextra_R[0/3]_delta_abs',\n",
       " 'distance_LextraMatchedHit[0]',\n",
       " 'distance_LextraMatchedHit[1]',\n",
       " 'distance_LextraMatchedHit[2]',\n",
       " 'distance_LextraMatchedHit[3]',\n",
       " 'distance_LextraMatchedHit_SE',\n",
       " 'distance_LextraMatchedHit_AE',\n",
       " 'MatchedHit_offset_abs',\n",
       " 'MatchedHit_offset_rel',\n",
       " 'MatchedHit_offset_rel_div_abs',\n",
       " 'ratio_MextraMatchedHit[0]',\n",
       " 'ratio_MextraMatchedHit[1]',\n",
       " 'ratio_MextraMatchedHit[2]',\n",
       " 'ratio_MextraMatchedHit[3]',\n",
       " 'cl_size[0]',\n",
       " 'cl_size[1]',\n",
       " 'cl_size[2]',\n",
       " 'cl_size[3]',\n",
       " 'PO',\n",
       " 'frac_NShared_FOI_hits_N',\n",
       " 'MatchedHit_D[0]_mean',\n",
       " 'MatchedHit_D[1]_mean',\n",
       " 'MatchedHit_D[2]_mean',\n",
       " 'MatchedHit_D[3]_mean',\n",
       " 'Mextra_D2[0]_mean',\n",
       " 'Mextra_D2[1]_mean',\n",
       " 'Mextra_D2[2]_mean',\n",
       " 'Mextra_D2[3]_mean']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = list(filter(lambda s: s not in utils.TRAIN_COLUMNS, train.columns))\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_g = [catboost.CatBoostClassifier().load_model(f\"models/catboost_model_{i}.cbm\") for i in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = [model.predict_proba(test.loc[:, features].values)[:, 1] for model in models_g]\n",
    "predictions = np.asarray(predictions).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>catboost_model_0</th>\n",
       "      <th>catboost_model_1</th>\n",
       "      <th>catboost_model_2</th>\n",
       "      <th>catboost_model_3</th>\n",
       "      <th>catboost_model_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.954220</td>\n",
       "      <td>0.956185</td>\n",
       "      <td>0.944788</td>\n",
       "      <td>0.952753</td>\n",
       "      <td>0.948069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.612289</td>\n",
       "      <td>0.601686</td>\n",
       "      <td>0.626082</td>\n",
       "      <td>0.630309</td>\n",
       "      <td>0.629644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.878670</td>\n",
       "      <td>0.873413</td>\n",
       "      <td>0.892294</td>\n",
       "      <td>0.897895</td>\n",
       "      <td>0.900006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.780969</td>\n",
       "      <td>0.823781</td>\n",
       "      <td>0.803654</td>\n",
       "      <td>0.785359</td>\n",
       "      <td>0.818781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.836963</td>\n",
       "      <td>0.828953</td>\n",
       "      <td>0.839748</td>\n",
       "      <td>0.887421</td>\n",
       "      <td>0.849181</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   catboost_model_0  catboost_model_1  catboost_model_2  catboost_model_3  \\\n",
       "0          0.954220          0.956185          0.944788          0.952753   \n",
       "1          0.612289          0.601686          0.626082          0.630309   \n",
       "2          0.878670          0.873413          0.892294          0.897895   \n",
       "3          0.780969          0.823781          0.803654          0.785359   \n",
       "4          0.836963          0.828953          0.839748          0.887421   \n",
       "\n",
       "   catboost_model_4  \n",
       "0          0.948069  \n",
       "1          0.629644  \n",
       "2          0.900006  \n",
       "3          0.818781  \n",
       "4          0.849181  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred = pd.DataFrame(predictions,\n",
    "                       columns=[\"catboost_model_{}\".format(i) for i in range(predictions.shape[1])],\n",
    "                       index=test.index)\n",
    "df_pred.to_csv(\"preds/catboost_test_preds.csv\", sep=',', index=False)\n",
    "df_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.951203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.620002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.888456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.802509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.848453</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   prediction\n",
       "0    0.951203\n",
       "1    0.620002\n",
       "2    0.888456\n",
       "3    0.802509\n",
       "4    0.848453"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred = pd.DataFrame(df_pred.mean(axis=1))\n",
    "df_pred.columns = [\"prediction\"]\n",
    "df_pred.to_csv(\"catboost_submission_vbugaevskii_mean.csv\", index_label=utils.ID_COLUMN, sep=',')\n",
    "df_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}