dypolychord_paper_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# dyPolyChord paper results\n",
    "\n",
    "This notebook contains the code used to make the Gaussian mixture model results with `dyPolyChord`. For more information see the dynamic nested sampling paper ([Higson et al., 2017](https://arxiv.org/abs/1704.03459)) and the [dyPolyChord module](https://github.com/ejhigson/dyPolyChord).\n",
    "\n",
    "This notebook requires nested sampling run data which you can generate using:\n",
    "* ``make_gaussian_mix_runs.py`` for likelihood_name = 'gaussian_mix_4comp_4sep'.\n",
    "* ``make_fit_runs.py`` for likelihood_name = 'fit'.\n",
    "\n",
    "See the module docstrings for information about the random seeding used for the numerical results in the paper.\n",
    "\n",
    "The ``bsr`` package is also required; it can be downloaded at https://github.com/ejhigson/bsr."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Select likelihood to run\n",
    "\n",
    "Must be 'fit' or 'gaussian_mix_4comp_4sep'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "likelihood_name = 'fit'\n",
    "assert likelihood_name in ['gaussian_mix_4comp_4sep', 'fit']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "import numpy as np\n",
    "import copy\n",
    "import pandas as pd\n",
    "import scipy.interpolate\n",
    "import matplotlib.pyplot as plt\n",
    "import nestcheck.data_processing\n",
    "import nestcheck.diagnostics_tables\n",
    "import nestcheck.estimators as e\n",
    "import nestcheck.plots\n",
    "import nestcheck.pandas_functions\n",
    "import dyPolyChord.output_processing\n",
    "import dyPolyChord.python_likelihoods as likelihoods\n",
    "import bsr.plotting\n",
    "import bsr.data\n",
    "import bsr.likelihoods\n",
    "import bsr.basis_functions\n",
    "import matplotlib\n",
    "%matplotlib inline\n",
    "%autoreload 2\n",
    "\n",
    "# Set plot size, font and fontsize to match LaTeX template\n",
    "# NB A4 paper is 8.27 × 11.69 inches (=210 × 297 mm)\n",
    "# STCO\n",
    "# ----\n",
    "# Font (=caption font): \\OT1/ptm/m/n/10 (= times size 10)\n",
    "# Footnote font: \\OT1/cmr/m/n/10\n",
    "# Abstract font: \\OT1/cmr/m/n/10.95\n",
    "textwidth = 6.85066 * 0.99  # make 1% smaller to ensure everything fits\n",
    "textheight = 9.2144 * 0.99  # make 1% smaller to ensure everything fits\n",
    "colwidth = 3.30719\n",
    "# Thesis\n",
    "# ------\n",
    "# textwidth = 5.1\n",
    "# Matplotlib setup\n",
    "# ----------------\n",
    "fontsize = 8.5  # matplotlib default is 10\n",
    "matplotlib.rc('text', usetex=True)\n",
    "matplotlib.rc('font', family='serif', serif='Times New Roman', size=fontsize)\n",
    "\n",
    "# Get estimator functions\n",
    "# -----------------------\n",
    "\n",
    "latex_str_map = {'mathrm{log}': 'log',\n",
    "                 'std': 'St.Dev.\\\\',\n",
    "                 'St.Dev. ': 'St.Dev.\\\\ ',\n",
    "                 'mean': 'Mean',\n",
    "                 'true values': 'Analytic values',\n",
    "                 'None': ''}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get fit likelihood object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up\n",
    "fit_func = bsr.basis_functions.gg_1d\n",
    "nfunc = 4\n",
    "y_error_sigma = 0.05\n",
    "x_error_sigma= 0.05\n",
    "npoints = 120\n",
    "data = bsr.data.generate_data(\n",
    "    fit_func, nfunc, y_error_sigma, x_error_sigma=x_error_sigma, npoints=npoints)\n",
    "fit_likelihood_obj = bsr.likelihoods.FittingLikelihood(\n",
    "    data, fit_func, nfunc, adaptive=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get estimator function list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def likelihood_calls(run, logw=None, simulate=False):\n",
    "    return run['output']['nlike']\n",
    "\n",
    "def fit_mean(ns_run, logw=None, simulate=False, like=fit_likelihood_obj, x1=0.5):\n",
    "    if logw is None:\n",
    "        logw = nestcheck.ns_run_utils.get_logw(ns_run, simulate=simulate)\n",
    "    w_rel = np.exp(logw - logw.max())\n",
    "    return like.fit_mean(ns_run['theta'], x1)\n",
    "\n",
    "estimator_list = [likelihood_calls,\n",
    "                  e.count_samples,\n",
    "                  e.logz]\n",
    "\n",
    "estimator_names = ['likelihood calls'] + [e.get_latex_name(est) for est in estimator_list[1:]]\n",
    "\n",
    "if likelihood_name == 'gaussian_mix_4comp_4sep':\n",
    "    estimator_list += [e.param_mean,\n",
    "                       functools.partial(e.param_mean, param_ind=1),\n",
    "                       functools.partial(e.param_mean, param_ind=2),\n",
    "                       functools.partial(e.param_mean, param_ind=3),\n",
    "                       functools.partial(e.param_mean, param_ind=4),\n",
    "                       functools.partial(e.param_mean, param_ind=5),\n",
    "                       e.param_squared_mean,\n",
    "                       functools.partial(e.param_cred, probability=0.5),\n",
    "                       functools.partial(e.param_cred, probability=0.84),\n",
    "                       e.r_mean,\n",
    "                       functools.partial(e.r_cred, probability=0.5),\n",
    "                       functools.partial(e.r_cred, probability=0.84)]\n",
    "    estimator_names = ['likelihood calls'] + [e.get_latex_name(est) for est in estimator_list[3:]]\n",
    "elif likelihood_name == 'fit':\n",
    "    estimator_list += [functools.partial(e.param_mean, param_ind=ind)\n",
    "                       for ind in range(0, fit_likelihood_obj.ndim)]\n",
    "    estimator_names += [r'$\\overline{{{}}}$'.format(name[1:-1])\n",
    "                        for name in fit_likelihood_obj.get_param_latex_names()]\n",
    "    for x in np.linspace(0.0, 1.0, 11):\n",
    "        estimator_list += [functools.partial(fit_mean, x1=x)]\n",
    "        estimator_names += ['$y({:.1f})$'.format(x)]\n",
    "    for x in [0.25, 0.75]:\n",
    "        estimator_list += [functools.partial(fit_mean, x1=x)]\n",
    "        estimator_names += ['$y({:.2f})$'.format(x)]\n",
    "estimator_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analytic Results for Gaussian mixture model\n",
    "\n",
    "#### Convolutions of Gaussians formulae\n",
    "\n",
    "From [Petersen and Pedersen (2018)](http://www.cim.mcgill.ca/~dudek/417/Papers/matrixOperations.pdf), Section 8.1.8, the product of two Gaussians is\n",
    "\n",
    "\\begin{equation}\n",
    "\\mathcal{N}_x(\\mu_1,\\Sigma_1) \\times \\mathcal{N}_x(\\mu_2,\\Sigma_2) = c_c\\mathcal{N}_x(\\mu_c,\\Sigma_c),\n",
    "\\end{equation}\n",
    "\n",
    "\n",
    "where\n",
    "\n",
    "\\begin{align}\n",
    "c_c & = \\mathcal{N}_{\\mu_1}(\\mu_2,(\\Sigma_1+\\Sigma_2)), \\\\\n",
    "\\mu_c & = (\\Sigma_1^{-1}+\\Sigma_2^{-1})^{-1} (\\Sigma_1^{-1}\\mu_1 + \\Sigma_2^{-1}\\mu_2), \\\\\n",
    "\\Sigma_c & = (\\Sigma_1^{-1}+\\Sigma_2^{-1})^{-1}.\n",
    "\\end{align}\n",
    "\n",
    "#### Application to the mixture with a Gaussian prior\n",
    "\n",
    "The posterior for the mixture of 4 unit Gaussians with a Gaussian prior which was used in the paper can be found by convolving the prior with each component to produce a sum of 4 shifted Gaussians.\n",
    "\n",
    "In our case we have $\\Sigma_m = \\mathcal{I}$ for the components and $\\Sigma_\\pi = \\mathcal{I} \\frac{1}{\\sigma_\\pi^2}$, $\\mu_\\pi = \\vec{0}$ for the prior. Hence for each component the convolution with the prior is a spherically symmetric Gaussian with\n",
    "\n",
    "\\begin{equation}\n",
    "\\mu_m = \\mu_m \\frac{\\sigma_\\pi^2}{1 + \\sigma_\\pi^2}$.\n",
    "\\end{equation}\n",
    "\n",
    "Furthermore, for every component\n",
    "\n",
    "\\begin{equation}\n",
    "\\sigma_c = \\frac{\\sigma_\\pi}{\\sqrt{1 + \\sigma_\\pi^2}}\n",
    "\\end{equation}\n",
    "\n",
    "and\n",
    "\n",
    "\\begin{equation}\n",
    "c_c = \\mathcal{N}_{0}\\left(\\mu_1,1 + \\frac{1}{\\sigma_\\pi^2}\\right),\n",
    "\\end{equation}\n",
    "\n",
    "which gives spherically symmetric Gaussian with $\\sigma = \\sqrt{1 + \\sigma_\\pi^2}$. Furthermore we have $|\\mu|=\\mathrm{sep}=4$ for all components so $c_c$ is the same for all components and the Bayesian evidence is\n",
    "\n",
    "\\begin{equation}\n",
    "\\mathcal{Z} = c_c.\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def gaussian_mix_logz(sep=4, dim=10, prior_scale=10):\n",
    "    rad = np.zeros(dim)\n",
    "    rad[0] = sep\n",
    "    const_sigma = np.sqrt(1 + (prior_scale ** 2))\n",
    "    return likelihoods.Gaussian(sigma=const_sigma)(rad)[0]\n",
    "\n",
    "def gaussian_mix_means(sep=4, dim=10, prior_scale=10, weights=np.asarray([0.4, 0.3, 0.2, 0.1])):\n",
    "    positions = [(0, sep), (0, -sep), (sep, 0), (-sep, 0)][:len(weights)]\n",
    "    factor = prior_scale / np.sqrt(1 + (prior_scale ** 2))\n",
    "    thetas = np.zeros(dim)\n",
    "    comp_pos = np.asarray([[0,  4],\n",
    "                         [0,  -4],\n",
    "                         [4,  0],\n",
    "                         [-4, 0]])\n",
    "    for i in range(2):\n",
    "        thetas[i] = np.sum(comp_pos[:, i] * weights) * factor\n",
    "    return thetas\n",
    "\n",
    "\n",
    "true_values_dict = {e.get_latex_name(e.logz): gaussian_mix_logz()}\n",
    "param_means = gaussian_mix_means()\n",
    "for i, val in enumerate(param_means):\n",
    "    true_values_dict[e.get_latex_name(e.param_mean, param_ind=i)] = val\n",
    "true_values_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "start_run = 0  # zero indexed\n",
    "n_run = 20\n",
    "# Shared Settings\n",
    "pc_run_dict = {}\n",
    "method_names = []\n",
    "for dynamic_goal in [None, 0, 0.25, 1]:\n",
    "    if dynamic_goal is None:\n",
    "        key = 'standard'\n",
    "    else:\n",
    "        key = 'dynamic $G=' + str(dynamic_goal) + '$'\n",
    "    method_names.append(key)\n",
    "    if likelihood_name == 'gaussian_mix_4comp_4sep':\n",
    "        root = dyPolyChord.output_processing.settings_root(\n",
    "            likelihood_name,\n",
    "            'gaussian',  # prior name\n",
    "            10,  # dimensions\n",
    "            dynamic_goal=dynamic_goal,\n",
    "            nrepeats=50,\n",
    "            prior_scale=10,\n",
    "            ninit=200, init_step=100,\n",
    "            nlive_const=500).replace('.', '_')\n",
    "    elif likelihood_name == 'fit':\n",
    "        root = fit_likelihood_obj.get_file_root(2000, 400, dynamic_goal=dynamic_goal)\n",
    "    print(root)\n",
    "    pc_run_dict[key] = nestcheck.data_processing.batch_process_data(\n",
    "        [root + '_' + str(i).zfill(3) for i in range(start_run + 1, start_run + n_run + 1)],\n",
    "        process_func=nestcheck.data_processing.process_polychord_run,\n",
    "        func_kwargs={}, errors_to_handle=OSError,\n",
    "        save_name='cache/' + root + '_' + str(start_run + 1) + '_to_' + str(start_run + n_run),\n",
    "        load=False, save=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot nlive as a function of Log X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate logL(logX) and logX(logL) numerically by combining many runs together to get accurate\n",
    "# logX values for each point logL, then interpolate\n",
    "comb = nestcheck.ns_run_utils.combine_ns_runs(pc_run_dict['standard'][:100])\n",
    "logx = nestcheck.ns_run_utils.get_logx(comb['nlive_array'])\n",
    "logl_given_logx = scipy.interpolate.interp1d(logx, comb['logl'], bounds_error=False, fill_value=-np.inf)\n",
    "logx_given_logl = scipy.interpolate.interp1d(comb['logl'], logx, bounds_error=False, fill_value=-np.inf)\n",
    "plot_dict = {}\n",
    "for method in method_names:\n",
    "    plot_dict[method] = pc_run_dict[method][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make the plot\n",
    "if likelihood_name == 'gaussian_mix_4comp_4sep':\n",
    "    logx_min = -35\n",
    "    ymax = 3000\n",
    "elif likelihood_name == 'fit': \n",
    "    ymax = 14000\n",
    "    logx_min = -55\n",
    "fig = nestcheck.plots.plot_run_nlive(method_names, plot_dict, logx_min=logx_min, ymax=ymax,\n",
    "                                     logx_given_logl=logx_given_logl,\n",
    "                                     logl_given_logx=logl_given_logx,\n",
    "                                     figsize=(textwidth, 2.2))\n",
    "# For thesis, use left=0.1\n",
    "adjust_dict = {'left': 0.085, 'right': 0.99, 'bottom': 0.185, 'top': 0.975}\n",
    "fig.subplots_adjust(**adjust_dict)\n",
    "fig.savefig('plots/nlive_' + likelihood_name + '.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Triangle Plot Diagram\n",
    "\n",
    "This cell makes an accurate plot of the posterior numerically by combining evenly weighted samples from a lot of nested sampling runs. It requires the `getdist` package, which is available on PyPI and at [https://github.com/cmbant/getdist](https://github.com/cmbant/getdist)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "param_limits = {}\n",
    "if likelihood_name == 'gaussian_mix_4comp_4sep':\n",
    "    names = [r'\\theta_\\hat{' + str(i) + r'}' for i in range(1, 11)]\n",
    "    for name in names:\n",
    "        param_limits[name] = [-8, 8]\n",
    "    names_to_plot = names[:4]\n",
    "elif likelihood_name == 'fit':\n",
    "    names = fit_likelihood_obj.get_param_latex_names()\n",
    "    names = [name[1:-1] for name in names]\n",
    "    for name in names:\n",
    "        param_limits[name] = [0, 1]\n",
    "    names_to_plot = names[:4]\n",
    "try:\n",
    "    from getdist import plots, MCSamples\n",
    "    np.random.seed(0)\n",
    "    evens = []\n",
    "    for run in pc_run_dict['dynamic $G=1$']:\n",
    "        logw = nestcheck.ns_run_utils.get_logw(run)\n",
    "        w_rel = np.exp(logw - logw.max())\n",
    "        temp = run['theta'][np.where(w_rel > np.random.random(w_rel.shape))[0], :]\n",
    "        evens.append(temp)\n",
    "    samples = MCSamples(samples=np.vstack(evens), names=names, labels=names)\n",
    "    g = plots.getSubplotPlotter(width_inch=colwidth)\n",
    "    g.triangle_plot([samples], names_to_plot, shaded=True, param_limits=param_limits)\n",
    "    # Save as a png as due to the very large number of samples this takes up less memory than a pdf\n",
    "    g.fig.savefig('plots/triangle_plot.png', dpi=400)\n",
    "except ImportError:\n",
    "    print('Install getdist to make this plot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Colour contour diagram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if likel\n",
    "    run = pc_run_dict['dynamic $G=1$'][0]\n",
    "    print('Run uses {} samples'.format(['logl'].shape[0]))\n",
    "    fig = bsr.plotting.plot_runs(\n",
    "        fit_likelihood_obj,\n",
    "        run,\n",
    "        nfunc_list=[nfunc], plot_data=True,\n",
    "        figsize=(6.5, 1.9),\n",
    "        subplot_height=1.2,\n",
    "        subplot_width=1.8)\n",
    "    fig.savefig('plots/fit_fgivenx.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results Tables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Efficiency gain DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Generate efficiency gain df\n",
    "true_values = np.full(len(estimator_names), np.nan)\n",
    "if likelihood_name == 'gaussian_mix_4comp_4sep':\n",
    "    include_true_values=True\n",
    "    include_rmse=True\n",
    "    for i, name in enumerate(estimator_names):\n",
    "        try:\n",
    "            true_values[i] = true_values_dict[name]\n",
    "        except KeyError:\n",
    "            pass\n",
    "elif likelihood_name == 'fit':\n",
    "    include_true_values=False\n",
    "    include_rmse=False\n",
    "method_values = []\n",
    "for method in method_names:\n",
    "    method_values.append([nestcheck.ns_run_utils.run_estimators(run, estimator_list)\n",
    "                          for run in pc_run_dict[method]])\n",
    "eff_df = nestcheck.pandas_functions.efficiency_gain_df(\n",
    "    method_names, method_values, estimator_names,\n",
    "    true_values=true_values, include_true_values=include_true_values,\n",
    "    include_rmse=include_rmse)\n",
    "eff_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eff_df.xs('std efficiency gain', level='calculation type')[estimator_names[2:3] + estimator_names[-13:]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementation Errors DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# diagnostic tests df\n",
    "error_summary_dict = {}\n",
    "n_simulate = 10\n",
    "for method in method_names:\n",
    "    save_name = ('cache/' + likelihood_name + '_4func_' + method.replace('$', '').replace('=', '_')\n",
    "                 .replace(' ', '_') + '_' + str(n_simulate) + 'sim')\n",
    "    print(save_name)\n",
    "    df_temp = nestcheck.diagnostics_tables.run_list_error_values(\n",
    "        pc_run_dict[method], estimator_list[1:], estimator_names[1:], n_simulate=n_simulate, parallel=True,\n",
    "        load=False, save=True, thread_pvalue=False, bs_stat_dist=False, save_name=save_name)\n",
    "    # error_values_dict[method] = df_temp\n",
    "    error_summary_dict[method] = nestcheck.diagnostics_tables.error_values_summary(df_temp)\n",
    "# values_df = pd.concat(error_values_dict)\n",
    "diag_df = pd.concat(error_summary_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show implementation errors\n",
    "diag_df.xs('implementation std', level='calculation type')[estimator_names[2:]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get paper results tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns to use in paper\n",
    "if likelihood_name == 'gaussian_mix_4comp_4sep':\n",
    "    cols = [\n",
    "        'samples',\n",
    "        '$\\\\mathrm{log} \\\\mathcal{Z}$',\n",
    "        '$\\\\overline{\\\\theta_{\\\\hat{1}}}$',\n",
    "        '$\\\\overline{\\\\theta_{\\\\hat{2}}}$',\n",
    "        '$\\\\mathrm{median}(\\\\theta_{\\\\hat{1}})$',\n",
    "        '$\\\\mathrm{C.I.}_{84\\\\%}(\\\\theta_{\\\\hat{1}})$',\n",
    "        '$\\\\overline{|\\\\theta|}$']\n",
    "elif likelihood_name == 'fit':\n",
    "    cols = [\n",
    "        'samples',\n",
    "        '$\\\\mathrm{log} \\\\mathcal{Z}$',\n",
    "        '$y(0.1)$',\n",
    "        '$y(0.3)$',\n",
    "        '$y(0.5)$',\n",
    "        '$y(0.7)$',\n",
    "        '$y(0.9)$']    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paper Table 2\n",
    "paper_eff_df = nestcheck.pandas_functions.paper_format_efficiency_gain_df(eff_df[cols])\n",
    "try:\n",
    "    import texunc\n",
    "    texunc.print_latex_df(paper_eff_df, min_dp=1, min_dp_no_error=2, max_power=5, str_map=latex_str_map)\n",
    "except ImportError:\n",
    "    print('Install texunc to get the results tables in the format used in the paper LaTeX file.')\n",
    "paper_eff_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Paper Table 3\n",
    "imp_df = copy.deepcopy(diag_df[cols[1:]].xs('implementation std', level='calculation type', drop_level=False))\n",
    "row_names = (imp_df.index.get_level_values(1).astype(str) + ' ' +\n",
    "             imp_df.index.get_level_values(0).astype(str))\n",
    "row_names = row_names.str.replace('dynamic', '').str.replace('implementation', 'Implementation')\n",
    "imp_df.index = [row_names, imp_df.index.get_level_values(2)]\n",
    "try:\n",
    "    import texunc\n",
    "    texunc.print_latex_df(imp_df, str_map=latex_str_map)\n",
    "except ImportError:\n",
    "    print('Install texunc to get the results tables in the format used in the paper LaTeX file.')\n",
    "imp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Paper Table 7\n",
    "rmse_df = copy.deepcopy(eff_df.loc[:, ~np.isnan(eff_df.loc[('true values', '' , 'value'), :].values)])\n",
    "row_names = (rmse_df.index.get_level_values(0).astype(str) + ' ' +\n",
    "             rmse_df.index.get_level_values(1).astype(str))\n",
    "row_names = row_names.str.replace('dynamic', '').str.replace('rmse', 'RMSE')\n",
    "rmse_df.index = [row_names, rmse_df.index.get_level_values(2)]\n",
    "try:\n",
    "    import texunc\n",
    "    texunc.print_latex_df(rmse_df, str_map=latex_str_map,  min_dp_no_error=4, max_power=5)\n",
    "except ImportError:\n",
    "    print('Install texunc to get the results tables in the format used in the paper LaTeX file.')\n",
    "rmse_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "widgets": {
   "state": {
    "046860fa7a394ed5a47a0de9992d39d5": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "04cd7826807e42d38908dd5a631f0685": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "09eca06f81de439eb3ccdd24c1573d69": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "0e3f2509f0ed4f9cad624bbd1e621965": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "10eb8ade85394e8eadfa9aef29948f78": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "16696ce0eccc4de49c99b9d7933e62a9": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "184b8f4195ca4d0fa49537dc5de07847": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "2ce752916e2d41369027bd576705280f": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "328316273148463882ef43163fef9225": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "35a06868701d4f07a2944ecb5128da24": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "43cd464c9bb8445db7b4674ec7d25ebb": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "848519415cd0447ba23f45e79c5f3e21": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "8c173c64620844a9a73ca0de708eed5f": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "961a5ebb7b674cb79126169166a0e08f": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "9bdf45d43ee34bca9bdae8cc21f6acf8": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "a52bb12907a44d0f99c67b3c801743f1": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "a66fa192611748419254cec4f99bf448": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "aa7d065645e3483c81aa7ad4dbf2e38a": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "ae57ca2860f14fef87cc80063e5cc9bf": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "d4cefe0a911443caa0e2f744e52389cb": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "d79dad432c1f4cb1a5af8494c9c0a218": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "dfe72262b582497494b109f85e61c7b6": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "e003bc38fa974508970c57df8e2a7136": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "e619781993b340d880627aa0bc39270c": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "e66c4867bbd4431fa4c28c33d44bf589": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "ed95749dcc6b40d2811e7b7e34b9bcf4": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "eef43101d7c44130a6bb6f343befd690": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "f5e87fe4dd574409801b4618d4986c81": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "fa55465c96894bf4b630e139a3a8c03a": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    },
    "fcd4128b66774be28f908c3ae86d9b0f": {
     "views": [
      {
       "cell_index": 5
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}