ECAI_Exp3.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "6eXzKXL9J76b"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm.notebook import trange, tqdm\n",
        "\n",
        "ETA_Q = 5\n",
        "ETA_R = 5\n",
        "EPS = 1e-8\n",
        "STEP = 0.2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g4GAQ9f1J76e"
      },
      "source": [
        "# The power of contrastive examples\n",
        "\n",
        "We compare RL, contrastive examples and demonstrations."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RgcJDZxsJ76g"
      },
      "source": [
        "## Environment definition\n",
        "\n",
        "The environment looks like this:\n",
        "\n",
        "```\n",
        " ---- ---- ---- ---- ---- \n",
        "|  1 |  2 |  3 |  4 |  5 |\n",
        " ---- ---- ---- ---- ----\n",
        "|  6 |    |  7 |    |  8 |\n",
        " ----      ----      ----\n",
        "|  9 |    | 10 |    | 11 |\n",
        " ----      ----      ----\n",
        "| 12 |              | 13 |\n",
        " ----                ----\n",
        "| 14 |              | 15 |\n",
        " ----                ----  \n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MhM-GZkGJ76g",
        "outputId": "1f629d7e-5b33-48a7-cded-ffbd8ef3919a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "States: ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15']\n"
          ]
        }
      ],
      "source": [
        "S = list(map(lambda x : str(x), range(1, 16)))\n",
        "\n",
        "print('States:', S)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UlZgXAGoJ76h",
        "outputId": "61927666-0c17-4e31-9e1a-4db17a79721f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Actions: ('Up', 'Down', 'Left', 'Right', 'No-Op')\n"
          ]
        }
      ],
      "source": [
        "A = ('Up', 'Down', 'Left', 'Right', 'No-Op')\n",
        "\n",
        "print('Actions:', A)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zPS3eBhJ76i",
        "outputId": "b0c03090-214b-43d4-b4b3-e49e592120dc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Transition probabilities ok?\n",
            "\t- Action Up: True\n",
            "\t- Action Down: True\n",
            "\t- Action Left: True\n",
            "\t- Action Right: True\n",
            "\t- Action No-Op: True\n"
          ]
        }
      ],
      "source": [
        "#               1   2   3   4   5   6   7   8   9   10  11  12  13  14  15           \n",
        "Pu = np.array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 1\n",
        "               [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 2\n",
        "               [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 3\n",
        "               [0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 4\n",
        "               [0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 5\n",
        "               [.8, 0., 0., 0., 0., .2, 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 6\n",
        "               [0., 0., .8, 0., 0., 0., .2, 0., 0., 0., 0., 0., 0., 0., 0.],  # 7\n",
        "               [0., 0., 0., 0., .8, 0., 0., .2, 0., 0., 0., 0., 0., 0., 0.],  # 8\n",
        "               [0., 0., 0., 0., 0., .8, 0., 0., .2, 0., 0., 0., 0., 0., 0.],  # 9\n",
        "               [0., 0., 0., 0., 0., 0., .8, 0., 0., .2, 0., 0., 0., 0., 0.],  # 10\n",
        "               [0., 0., 0., 0., 0., 0., 0., .8, 0., 0., .2, 0., 0., 0., 0.],  # 11\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., .8, 0., 0., .2, 0., 0., 0.],  # 12\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., .8, 0., .2, 0., 0.],  # 13\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., .8, 0., .2, 0.],  # 14\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., .8, 0., .2]]) # 15\n",
        "\n",
        "#               1   2   3   4   5   6   7   8   9   10  11  12  13  14  15           \n",
        "Pd = np.array([[.2, 0., 0., 0., 0., .8, 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 1\n",
        "               [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 2\n",
        "               [0., 0., .2, 0., 0., 0., .8, 0., 0., 0., 0., 0., 0., 0., 0.],  # 3\n",
        "               [0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 4\n",
        "               [0., 0., 0., 0., .2, 0., 0., .8, 0., 0., 0., 0., 0., 0., 0.],  # 5\n",
        "               [0., 0., 0., 0., 0., .2, 0., 0., .8, 0., 0., 0., 0., 0., 0.],  # 6\n",
        "               [0., 0., 0., 0., 0., 0., .2, 0., 0., .8, 0., 0., 0., 0., 0.],  # 7\n",
        "               [0., 0., 0., 0., 0., 0., 0., .2, 0., 0., .8, 0., 0., 0., 0.],  # 8\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., .2, 0., 0., .8, 0., 0., 0.],  # 9\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],  # 10\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., .2, 0., .8, 0., 0.],  # 11\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., .2, 0., .8, 0.],  # 12\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., .2, 0., .8],  # 13\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],  # 14\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]]) # 15\n",
        "\n",
        "\n",
        "#               1   2   3   4   5   6   7   8   9   10  11  12  13  14  15           \n",
        "Pl = np.array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 1\n",
        "               [.8, .2, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 2\n",
        "               [0., .8, .2, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 3\n",
        "               [0., 0., .8, .2, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 4\n",
        "               [0., 0., 0., .8, .2, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 5\n",
        "               [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 6\n",
        "               [0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],  # 7\n",
        "               [0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],  # 8\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],  # 9\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],  # 10\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],  # 11\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],  # 12\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],  # 13\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],  # 14\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]]) # 15\n",
        "\n",
        "\n",
        "\n",
        "#               1   2   3   4   5   6   7   8   9   10  11  12  13  14  15           \n",
        "Pr = np.array([[.2, .8, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 1\n",
        "               [0., .2, .8, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 2\n",
        "               [0., 0., .2, .8, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 3\n",
        "               [0., 0., 0., .2, .8, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 4\n",
        "               [0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 5\n",
        "               [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],  # 6\n",
        "               [0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],  # 7\n",
        "               [0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],  # 8\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],  # 9\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],  # 10\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],  # 11\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],  # 12\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],  # 13\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],  # 14\n",
        "               [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]]) # 15\n",
        "\n",
        "\n",
        "Pn = np.eye(len(S))\n",
        "\n",
        "P = [Pu, Pd, Pl, Pr, Pn]\n",
        "\n",
        "print('Transition probabilities ok?')\n",
        "\n",
        "for aidx in range(len(A)):\n",
        "    print('\\t- Action %s:' % A[aidx], np.all(np.isclose(P[aidx].sum(axis=1), 1.)))\n",
        "    \n",
        "gamma = 0.99"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "utyA_tD3J76k"
      },
      "outputs": [],
      "source": [
        "# Rewards are a convex combination of 7 features/rewards\n",
        "CELL_GOAL = np.zeros((len(S), len(A)))\n",
        "CELL_GOAL[14, :] = 1\n",
        "\n",
        "CELL_10 = np.zeros((len(S), len(A)))\n",
        "CELL_10[9, :] = 1\n",
        "\n",
        "CELL_14 = np.zeros((len(S), len(A)))\n",
        "CELL_14[13, :] = 1\n",
        "\n",
        "\n",
        "RFEAT = (CELL_GOAL, CELL_10, CELL_14)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zcthHE-sJ76k"
      },
      "source": [
        "## MDP solver\n",
        "\n",
        "The next function is used to compute the optimal $Q$-function for MDPs. For computational efficiency, we pre-compute the optimal $Q$-function for all MDPs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "KOVzIT71J76l"
      },
      "outputs": [],
      "source": [
        "def vi(M):\n",
        "    ''' \n",
        "    Compute optimal Q-function for MDP M.\n",
        "\n",
        "    :param M: 5-tuple corresponding to MDP description: (S, A, P, R, gamma)\n",
        "    :return: np.array with shape (|S|, |A|), corresponding to Q* for M.\n",
        "    '''\n",
        "\n",
        "    S = M[0]\n",
        "    A = M[1]\n",
        "    P = M[2]\n",
        "    R = M[3]\n",
        "    gamma = M[4]\n",
        "    \n",
        "    Q = np.zeros((len(S), len(A)))\n",
        "    quit = False\n",
        "    niter = 0\n",
        "    \n",
        "    while not quit:\n",
        "        Qnew = np.zeros((len(S), len(A)))\n",
        "        V = np.max(Q, axis=1, keepdims=True)\n",
        "        for a in range(len(A)):\n",
        "            Qnew[:, a, None] = R[:, a, None] + gamma * P[a].dot(V)\n",
        "            \n",
        "        if np.linalg.norm(Q - Qnew) < EPS:\n",
        "            quit = True\n",
        "            \n",
        "        Q = Qnew\n",
        "        niter += 1\n",
        "        \n",
        "    # print('Finished after %i iterations.' % niter)\n",
        "    return Q\n",
        "\n",
        "# --\n",
        "\n",
        "def greedy(Q):\n",
        "    ''' \n",
        "    Compute greedy policy with respect to Q.\n",
        "    \n",
        "    :param Q: np.array with shape (|S|, |A|).\n",
        "    :return: np.array with shape (|S|, |A|), corresponding to pi_g(Q).\n",
        "    '''\n",
        "    \n",
        "    pol = (Q == np.max(Q, axis=1, keepdims=True)).astype(int)\n",
        "    pol = pol / np.sum(pol, axis=1, keepdims=True)\n",
        "\n",
        "    return pol\n",
        "\n",
        "# --\n",
        "\n",
        "def boltzmann(Q):\n",
        "    ''' \n",
        "    Computes the Boltzmann policy with respect to Q.\n",
        "\n",
        "    :param Q: np.array with shape (|S|, |A|).\n",
        "    :return: np.array with shape (|S|, |A|) corresponding to pi_B(Q).\n",
        "    '''\n",
        "    \n",
        "    Q = ETA_Q * (Q - np.max(Q, axis=1, keepdims=True))\n",
        "\n",
        "    pol = np.round(np.exp(Q), 3)\n",
        "    pol = pol / np.sum(pol, axis=1, keepdims=True)\n",
        "    \n",
        "    return pol\n",
        "\n",
        "# --"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "M462WAssJ76m"
      },
      "outputs": [],
      "source": [
        "w = np.random.randn(4)\n",
        "r = np.sum([RFEAT[m] * w[m] for m in range(len(RFEAT))], axis=0)\n",
        "M = (S, A, P, r, gamma)\n",
        "Q = vi(M)\n",
        "pol = boltzmann(Q)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zHt6k7pbJ76m"
      },
      "source": [
        "## RL updater\n",
        "\n",
        "We perform maximum likelihood estimation using SGD, where \n",
        "\n",
        "$$r_t\\sim\\mathrm{Normal}(w^\\top\\phi(s_t,a_t);\\eta)$$"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "gGr9dYM7J76m"
      },
      "outputs": [],
      "source": [
        "# Weight update\n",
        "def rl_update(wold, s, a, r):\n",
        "    ''' \n",
        "    Updates the weights wold given the triplet (s, a, r) and the weight w. \n",
        "    The update is a Stochastic Gradient Descent (SGD) update on the negative log likelihood, assuming the\n",
        "    reward is disturbed by 0-mean Gaussian noise.\n",
        "    \n",
        "    :param wold: np.array with shape (nfeat,)\n",
        "    :param s: int (state index)\n",
        "    :param a: int (action index)\n",
        "    :param r: float (reward)\n",
        "    :return: np.array with shape (nfeat,)\n",
        "    '''\n",
        "    \n",
        "    feat = np.array([RFEAT[m][s, a] for m in range(len(RFEAT))])\n",
        "    \n",
        "    grad = feat * (feat.dot(wold) - r)    \n",
        "    wnew = wold - STEP * grad\n",
        "\n",
        "    return wnew"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-bLtdUSwJ76n"
      },
      "source": [
        "## Demonstration updater\n",
        "\n",
        "We perform maximum likelihood estimation using SGD, where \n",
        "\n",
        "$$\\mathbb{P}[a_t\\mid s_t]=\\frac{\\exp\\{\\eta Q^*_w(s,a)\\}}{\\sum_{a'}\\exp\\{\\eta_Q Q^*_w(s,a')\\}},$$\n",
        "\n",
        "where $Q^*_w$ is the optimal $Q$-function for the MDP with reward $w^\\top\\phi$."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "O-ltVWL7J76n"
      },
      "outputs": [],
      "source": [
        "def policy_feat(pol, feat):\n",
        "    '''\n",
        "    Compute the policy-weighted feature matrix. \n",
        "    \"pol\" can be a single action index or a full \n",
        "    |S| x |A| policy matrix.\n",
        "    \n",
        "    :param pol: int (action index) or np.array with shape (|S|, |A|) (policy)\n",
        "    :param feat: tuple (each element is a (|S|, |A|)-shaped np.array)\n",
        "    :return: np.array with shape (|S|, |nfeat|)\n",
        "    '''\n",
        "    \n",
        "    if isinstance(pol, int):\n",
        "        return np.hstack([feat[m][:, pol, None] for m in range(len(feat))])\n",
        "    \n",
        "    else:\n",
        "        return np.hstack([np.sum(feat[m] * pol, axis=1, keepdims=True) for m in range(len(feat))])\n",
        "    \n",
        "# --\n",
        "\n",
        "def policy_trans(pol, P):\n",
        "    '''\n",
        "    Compute the policy-weighted transition probabilities.\n",
        "    \n",
        "    :param pol: np.array with shape (|S|, |A|) (policy)\n",
        "    :param P: tuple (each element is a (|S|, |S|)-shaped np.array)\n",
        "    :return: np.array with shape (|S|, |S|)\n",
        "    '''\n",
        "    \n",
        "    return np.sum([np.diag(pol[:, a]).dot(P[a]) for a in range(len(A))], axis=0)\n",
        "\n",
        "# --\n",
        "\n",
        "def policy_rew(pol, r):\n",
        "    '''\n",
        "    Compute the policy-weighted reward.\n",
        "    \n",
        "    :param pol: np.array with shape (|S|, |A|) (policy)\n",
        "    :param r: np.array with shape (|S|, |A|) (reward)\n",
        "    :return: np.array with shape (|S|,)\n",
        "    '''\n",
        "    \n",
        "    return np.sum(pol * r, axis=1)\n",
        "\n",
        "# --\n",
        "\n",
        "def discounted_freq(pol, P):\n",
        "    '''\n",
        "    Compute the discounted visitation frequencies given a policy pol.\n",
        "    \n",
        "    :param pol: np.array with shape (|S|, |A|) (policy)\n",
        "    :param P: tuple (each element is a (|S|, |S|)-shaped np.array)\n",
        "    :return: np.array with shape (|S|, |S|)\n",
        "    '''\n",
        "    \n",
        "    Ppol = policy_trans(pol, P)\n",
        "    return (1 - gamma) * np.linalg.inv(np.eye(len(S)) - gamma * Ppol)\n",
        "\n",
        "# --\n",
        "    \n",
        "# Weight update\n",
        "def demo_update(wold, s, a, pol=None):\n",
        "    ''' \n",
        "    Updates the weights wold given the pair (s, a) and the weight wold. \n",
        "    The update is a SGD update on the negative log likelihood, assuming the\n",
        "    actions are selected from a Boltzmann distribution (pol, which can be\n",
        "    precomputed).\n",
        "    \n",
        "    :param wold: np.array with shape (|nfeat|,)\n",
        "    :param s: int (state index)\n",
        "    :param a: int (action index)\n",
        "    :param pol: np.array with shape (|S|, |A|) (policy)\n",
        "    :return: np.array with shape (nfeat,)\n",
        "    '''\n",
        "\n",
        "    if pol is None:\n",
        "        rhat = np.sum([RFEAT[m] * w[m] for m in range(len(RFEAT))], axis=0)\n",
        "        M = (S, A, P, rhat, gamma)\n",
        "        Q = vi(M)\n",
        "        pol = boltzmann(Q)\n",
        "\n",
        "    feat = policy_feat(greedy(Q), RFEAT) # (nS, nF)\n",
        "    disc = discounted_freq(greedy(Q), P) # (nS, nS)\n",
        "    \n",
        "    grad = np.zeros(wold.shape)\n",
        "\n",
        "    for a_idx in range(len(A)):\n",
        "              #              (nS, nF)                     (nS, nS) (nS, nS) (nS, nF)\n",
        "        grad += ETA_Q * (policy_feat(a_idx, RFEAT) + gamma * P[a_idx].dot(disc).dot(feat))[s, :] * (pol[s, a_idx] - int(a == a_idx))\n",
        "        \n",
        "    # Gradients are too small, so we push the step-size up\n",
        "    \n",
        "    wnew = wold - 2 * STEP * grad\n",
        "        \n",
        "    return wnew"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vQFCuICrJ76n",
        "outputId": "9d002a13-8328-42e0-ec9b-c3ebc598ecb5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "policy_feat(.): (15, 3)\n",
            "policy_feat(.): (15, 3)\n",
            "policy_trans(.): (15, 15)\n",
            "policy_rew(.): (15,)\n",
            "discounted_freq(.): (15, 15)\n"
          ]
        }
      ],
      "source": [
        "print('policy_feat(.):', policy_feat(pol, RFEAT).shape)\n",
        "print('policy_feat(.):', policy_feat(0, RFEAT).shape)\n",
        "print('policy_trans(.):', policy_trans(pol, P).shape)\n",
        "print('policy_rew(.):', policy_rew(pol, r).shape)\n",
        "print('discounted_freq(.):', discounted_freq(pol, P).shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r7k45CqBJ76o"
      },
      "source": [
        "## Contrastive examples updater\n",
        "\n",
        "We consider the following contrastive examples:\n",
        "\n",
        "- In state $s$ action $a$ is better than action $a'$ because it will eventually lead you through state $s'$ and that is good.\n",
        "- In state $s$ action $a$ is worse than action $a'$ because it will eventually lead you through state $s'$ and that is bad.\n",
        "\n",
        "We consider that, given the target reward $r$,\n",
        "\n",
        "- A state $s$ is good if $r^*(s)>0$ for the optimal policy $\\pi^*_r$ given that reward.\n",
        "- A state $s$ is bad if $r^*(s)<0$ for the optimal policy $\\pi^*_r$ given that reward.\n",
        "- An action $a$ is _better_ than an action $a'$ in state $s$ and leading to state $s'$ if the following two conditions are cumulatively met:\n",
        "\n",
        " - $Q^*(s,a)>Q^*(s,a')$ ($a$ is better than $a'$)\n",
        "    * The transition probabilities by first taking action $a$ and then following $\\pi^*$ lead to larger transition probability from $s$ to $s'$ than those same probabilities taking action $a'$.\n",
        "    \n",
        "- An action $a$ is _worse_ than an action $a'$ in state $s$ and leading to state $s'$ if the following two conditions are cumulatively met:\n",
        "\n",
        " - $Q^*(s,a)<Q^*(s,a')$ ($a$ is worse than $a'$)\n",
        "    * The transition probabilities by taking action $a$ in $s$ and following $\\pi^*$ elsewhere lead to larger transition probability from $s$ to $s'$ than those same probabilities taking action $a'$."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "MTFJKY7ZJ76o"
      },
      "outputs": [],
      "source": [
        "def sigmoid(z, eta):\n",
        "    ''' \n",
        "    Compute sigmoid function.\n",
        "    \n",
        "    :param z: float\n",
        "    :param eta: float\n",
        "    :return: float\n",
        "    '''\n",
        "    \n",
        "    return 1 / (1 + np.exp(-eta * z))\n",
        "\n",
        "# --"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQ4Px7V8J76o"
      },
      "source": [
        "We perform maximum likelihood estimation using SGD, where \n",
        "\n",
        "$$\\mathbb{P}[(s,a,a',s',k)\\mid w]=\\sigma(kr_w(s'))\\sigma(k(Q(s,a)-Q(s,a')))\\mathbf{P}_{a}^\\infty(s'\\mid s)$$\n",
        "\n",
        "(see precise definition in the paper)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "Uq4h5tnFJ76p"
      },
      "outputs": [],
      "source": [
        "# Weight update\n",
        "def explain_update(wold, s0, s1, a0, a1, k, Q=None):\n",
        "    '''\n",
        "    Updates the weights wold given the tuple (s0, s1, a0, a1, k) \n",
        "    and the weight wold. The update is a SGD update on the negative \n",
        "    log likelihood, assuming the distribution above.\n",
        "\n",
        "    '''\n",
        "    \n",
        "    # Compute policy\n",
        "    r = np.sum([RFEAT[m] * wold[m] for m in range(len(RFEAT))], axis=0)\n",
        "\n",
        "    if Q is None:\n",
        "        M = (S, A, P, r, gamma)\n",
        "        Q = vi(M)\n",
        "        \n",
        "    pol = boltzmann(Q)\n",
        "\n",
        "    paux = pol.copy()\n",
        "    paux[s0] = np.zeros(len(A))\n",
        "    paux[s0, a0] = 1.0\n",
        "        \n",
        "    # Component 1 - reward\n",
        "    fpol = policy_feat(greedy(Q), RFEAT)      # (nS, nF)\n",
        "    rpol = policy_rew(greedy(Q), r) # (nS,)\n",
        "\n",
        "    grad_R = ETA_R * k * fpol[s1, :] * (sigmoid(k * rpol[s1], ETA_R) - 1) # (nF,)\n",
        "    #print('grad_R:')\n",
        "    #print(grad_R)\n",
        "    \n",
        "    # Component 2 - Q-values\n",
        "    fa0 = policy_feat(a0, RFEAT)   # (nS, nF)\n",
        "    fa1 = policy_feat(a1, RFEAT)   # (nS, nF)\n",
        "    freq = discounted_freq(greedy(Q), P) # (nS, nS)\n",
        "\n",
        "    diff_Q = Q[s0, a0] - Q[s0, a1]       # scalar\n",
        "    diff_P = P[a0][s0, :] - P[a1][s0, :] # (nS,)\n",
        "    diff_F = fa0[s0] - fa1[s0]           # (nF,)\n",
        "    \n",
        "    grad_Q = ETA_Q * k * (diff_F + gamma * diff_P.dot(freq).dot(fpol)) * (sigmoid(k * diff_Q, ETA_Q) - 1) # (nF,)\n",
        "    #print('grad_Q:')\n",
        "    #print(grad_Q)\n",
        "\n",
        "    # Component 3 - Pam\n",
        "    Pam = discounted_freq(paux, P) # (nS, nS)\n",
        "    pf = Pam.dot(fpol)             # (nS, nF)\n",
        "    \n",
        "    grad_P = np.zeros(len(RFEAT))\n",
        "    \n",
        "    for k in range(len(RFEAT)):\n",
        "        # for a single k\n",
        "        grad_aux = np.zeros((len(S), len(S)))\n",
        "\n",
        "        for a in range(len(A)):\n",
        "            for ap in range(len(A)):\n",
        "                fa  = policy_feat(a, RFEAT)  # (nS, nF)\n",
        "                fap = policy_feat(ap, RFEAT) # (nS, nF)\n",
        "                    #  (nS,)            (nS, nS) (nS, nS) (nS,)\n",
        "                dT  = fap[:, k] + gamma * P[ap].dot(freq).dot(fpol[:, k]) # (nS,)\n",
        "                tmp = ETA_Q * np.diag(paux[:, a] * dT * (pol[:, ap] - int(a == ap))).dot(P[a]) # (nS, nS)\n",
        "\n",
        "                grad_aux += tmp\n",
        "\n",
        "        dPk = Pam.dot(grad_aux).dot(Pam)      # (nS, nS)\n",
        "        if np.isclose(Pam[s0, s1], 0) and not np.isclose(dPk[s0, s1], 0):\n",
        "            print('Whoa! Zero!')\n",
        "            print('dPk:', dPk[s0, s1])\n",
        "\n",
        "        elif np.isclose(Pam[s0, s1], 0):\n",
        "            grad_P[k] = 0.                        # scalar\n",
        "\n",
        "        else:\n",
        "            grad_P[k] = dPk[s0, s1] / Pam[s0, s1] # scalar\n",
        "        \n",
        "    #print('grad_P:')\n",
        "    #print(grad_P)\n",
        "    \n",
        "    grad = grad_R + grad_Q + grad_P\n",
        "    \n",
        "    wnew = wold - STEP * grad\n",
        "    \n",
        "    return wnew"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EHi5ZyhkJ76p",
        "outputId": "d6a36af7-4437-494c-c6f5-f4ad65c07d95"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "P[a][s, :]: (15,)\n",
            "freq: (15, 15)\n",
            "fpol: (15, 3)\n",
            "P[a][s, :].dot(freq): (15,)\n",
            "P[a][s, :].dot(freq).dot(fpol): (3,)\n"
          ]
        }
      ],
      "source": [
        "freq = discounted_freq(pol, P)\n",
        "fpol = policy_feat(pol, RFEAT)\n",
        "print('P[a][s, :]:', P[0][0, :].shape)\n",
        "print('freq:', freq.shape)\n",
        "print('fpol:', fpol.shape)\n",
        "print('P[a][s, :].dot(freq):', P[0][0, :].dot(freq).shape)\n",
        "print('P[a][s, :].dot(freq).dot(fpol):', P[0][0, :].dot(freq).dot(fpol).shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2b_HimgRJ76p"
      },
      "source": [
        "## Experiments\n",
        "\n",
        "We now generate a random reward, and compare the performance of an RL updater, a demo updater and an contrastive examples updater in randomly selected sample rewards, demos and contrastive examples.\n",
        "\n",
        "We start by defining the samplers."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "EaY9kXR1J76p"
      },
      "outputs": [],
      "source": [
        "def sample_reward(M):\n",
        "    ''' Samples a reward for MDP M. '''\n",
        "\n",
        "    S = M[0]\n",
        "    A = M[1]\n",
        "    R = M[3]\n",
        "    \n",
        "    # Sample a random state\n",
        "    sample_s = np.random.choice(len(S))\n",
        "    \n",
        "    # Sample a random action\n",
        "    sample_a = np.random.choice(len(A))\n",
        "    \n",
        "    # Sample a noisy reward\n",
        "    sample_r = R[sample_s, sample_a] + np.random.randn() / ETA_R\n",
        "    \n",
        "    return (sample_s, sample_a, sample_r)\n",
        "\n",
        "# -- \n",
        "\n",
        "def voice_reward(M, s, a, r):\n",
        "    ''' Converts a reward sample into a natural language string. '''\n",
        "    \n",
        "    return 'The reward in state %s when performing action %s is %.2f.' % (M[0][s], M[1][a], r)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "NlSSm3f0J76q",
        "outputId": "3a8060ac-187d-4936-c054-68cfe32e75b4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The reward in state 1 when performing action Down is 0.18.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "voice_reward(M, *sample_reward(M))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "YJNptXPOJ76q"
      },
      "outputs": [],
      "source": [
        "def sample_demo(M):\n",
        "    ''' Samples a demonstration for MDP M. '''\n",
        "\n",
        "    S = M[0]\n",
        "    A = M[1]\n",
        "    \n",
        "    # Sample a random state\n",
        "    sample_s = np.random.choice(len(S))\n",
        "\n",
        "    # Sample a noisy action\n",
        "    Q = vi(M)\n",
        "    pol = boltzmann(Q)[sample_s, :]\n",
        "    sample_a = np.random.choice(len(A), p=pol)\n",
        "    \n",
        "    return (sample_s, sample_a)\n",
        "\n",
        "# --\n",
        "\n",
        "def voice_demo(M, s, a):\n",
        "    ''' Converts a demonstration sample into a natural language string. '''\n",
        "\n",
        "    return 'In state %s you should perform action %s.' % (M[0][s], M[1][a])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "6G6YXh88J76q",
        "outputId": "3787c8d4-4ef7-42c7-dfff-d33e7fed0b68"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'In state 6 you should perform action No-Op.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ],
      "source": [
        "Q = vi(M)\n",
        "pol = boltzmann(Q)\n",
        "print(pol.sum(axis=1))\n",
        "voice_demo(M, *sample_demo(M))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "s_RXa80KJ76q"
      },
      "outputs": [],
      "source": [
        "def sample_explain(M, echo=False):\n",
        "    ''' Samples an contrastive examples for MDP M. '''\n",
        "\n",
        "    S = M[0]\n",
        "    A = M[1]\n",
        "    P = M[2]\n",
        "    R = M[3]\n",
        "    gamma = M[4]\n",
        "    \n",
        "    Q = vi(M)\n",
        "    pol = boltzmann(Q)\n",
        "    rpol = policy_rew(pol, R)\n",
        "\n",
        "    sample_complete = False\n",
        "    \n",
        "    while not sample_complete:\n",
        "        \n",
        "        # This ensures that the search for next state doesn't go on forever\n",
        "        stop_sample = False\n",
        "        \n",
        "        # Sample a random initial state\n",
        "        sample_s0 = np.random.choice(len(S))\n",
        "        \n",
        "        if echo:\n",
        "            print('Sampled state %s' % S[sample_s0])\n",
        "\n",
        "        # Sample a random initial action\n",
        "        sample_a0 = np.random.choice(len(A))\n",
        "        \n",
        "        if echo:\n",
        "            print('Sampled action %s' % A[sample_a0])\n",
        "\n",
        "        paux = pol.copy()\n",
        "        paux[sample_s0] = np.zeros(len(A))\n",
        "        paux[sample_s0, sample_a0] = 1.0\n",
        "        \n",
        "        Pam = discounted_freq(paux, P)\n",
        "\n",
        "        # Sample a random second action (different from the first)\n",
        "        quit = False\n",
        "        while not quit:\n",
        "            sample_a1 = np.random.choice(len(A))\n",
        "            quit = sample_a1 != sample_a0\n",
        "\n",
        "        if echo:\n",
        "            print('Sampled another action: %s' % A[sample_a1])\n",
        "\n",
        "        # Sample a random via state with non-zero reward (to fit the \"good\" or \"bad\" description)\n",
        "\n",
        "        quit = False\n",
        "        while not quit:\n",
        "            paux = np.maximum(rpol * (Q[sample_s0, sample_a0] - Q[sample_s0, sample_a1]) * Pam[sample_s0, :], 0)\n",
        "\n",
        "            # Ensure that there is a next state to sample\n",
        "            if np.isclose(np.sum(paux), 0):\n",
        "                if echo:\n",
        "                    print('Zero support distribution. Restarting sampling.')\n",
        "                stop_sample = True\n",
        "                quit = True\n",
        "            else:\n",
        "                paux = paux / paux.sum()\n",
        "                if np.isclose(paux[sample_s0], 1.):\n",
        "                    if echo:\n",
        "                        print('Departing from sink. Restarting sampling.')\n",
        "                    stop_sample = True\n",
        "                    quit = True\n",
        "                else:\n",
        "                    if echo:\n",
        "                        print(np.round(paux, 3))\n",
        "                    sample_s1 = np.random.choice(len(S), p=paux)\n",
        "                    quit = (sample_s1 != sample_s0)\n",
        "\n",
        "        if not stop_sample:\n",
        "            if echo:\n",
        "                print('Sampled another state: %s' % S[sample_s1])\n",
        "\n",
        "            # Check which one is better\n",
        "            if Q[sample_s0, sample_a0] > Q[sample_s0, sample_a1]:\n",
        "                k = 1\n",
        "            else:\n",
        "                k = -1\n",
        "                \n",
        "            sample_complete = True\n",
        "            \n",
        "    if echo:\n",
        "        print('Q(%s, %s) =' % (S[sample_s0], A[sample_a0]), Q[sample_s0, sample_a0])\n",
        "        print('Q(%s, %s) =' % (S[sample_s0], A[sample_a1]), Q[sample_s0, sample_a1])\n",
        "        print('reward(%s) =' % S[sample_s1], np.sum(R * pol, axis=1)[sample_s1])\n",
        "        print('P_m[%s](%s, %s) =' % (A[sample_a0], S[sample_s0], S[sample_s1]), Pam[sample_s0, sample_s1])\n",
        "        \n",
        "    return (sample_s0, sample_s1, sample_a0, sample_a1, k)\n",
        "\n",
        "# --\n",
        "        \n",
        "def voice_explain(M, s0, s1, a0, a1, k):\n",
        "    ''' Converts an exmplanation sample into a natural language string. '''\n",
        "\n",
        "    REL1 = ('better', 'worse')\n",
        "    REL2 = ('good', 'bad')\n",
        "    \n",
        "    k = 1 - (k + 1) // 2\n",
        "    \n",
        "    return 'In state %s, action %s is %s than %s because it may eventually lead you through state %s and that is %s.' % (M[0][s0], M[1][a0], REL1[k], M[1][a1], M[0][s1], REL2[k])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        },
        "id": "yQLwYih7J76r",
        "scrolled": false,
        "outputId": "7e80168e-c386-4660-f0a5-e4c79a84db51"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sampled state 13\n",
            "Sampled action No-Op\n",
            "Sampled another action: Right\n",
            "Zero support distribution. Restarting sampling.\n",
            "Sampled state 13\n",
            "Sampled action Left\n",
            "Sampled another action: Up\n",
            "Zero support distribution. Restarting sampling.\n",
            "Sampled state 14\n",
            "Sampled action Up\n",
            "Sampled another action: Down\n",
            "[0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
            "Sampled another state: 10\n",
            "Q(14, Up) = 16.614283573623823\n",
            "Q(14, Down) = 15.92041665797559\n",
            "reward(10) = 0.19096048511266883\n",
            "P_m[Up](14, 10) = 0.6290687153266841\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'In state 14, action Up is better than Down because it may eventually lead you through state 10 and that is good.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "voice_explain(M, *sample_explain(M, echo=True))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5_t25yaJ76r"
      },
      "source": [
        "We now run a comparative study. Every 10 steps, for each of the three approaches, we:\n",
        "\n",
        "* Estimate the reward;\n",
        "* Compute the associated \"optimal\" policy;\n",
        "* Evaluate that policy in the correct MDP.\n",
        "\n",
        "The next function is used in the last step."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "1K4B1KnTJ76r"
      },
      "outputs": [],
      "source": [
        "def evaluate_reward(M, w):\n",
        "    ''' Computes the policy from weights w in the MDP M.'''\n",
        "\n",
        "    S = M[0]\n",
        "    A = M[1]\n",
        "    P = M[2]\n",
        "    R = M[3]\n",
        "    gamma = M[4]\n",
        "    \n",
        "    Raux = np.sum([w[i] * RFEAT[i] for i in range(len(RFEAT))], axis=0)\n",
        "    \n",
        "    Maux = (S, A, P, Raux, gamma)\n",
        "    pol  = greedy(vi(Maux))\n",
        "    \n",
        "    Ppol = policy_trans(pol, P)\n",
        "    Rpol = policy_rew(pol, R)\n",
        "    \n",
        "    Vaux = np.linalg.inv(np.eye(len(S)) - gamma * Ppol).dot(Rpol)\n",
        "\n",
        "    # Compute average value\n",
        "    return Vaux.sum() / len(S)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yzbntx4sJ76r"
      },
      "source": [
        "We can now run our comparative study."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "bd09808012e14ffabaeb2e3ebf461de3",
            "74b1ad1d52d141cc869adfdf3b54802c",
            "9a0c508ae5d54c538564a04f2cb667e6",
            "0158b3f8e8a043a0bf474cd8d8d17e21",
            "55869d9c62a04caa8cdb8138532d1053",
            "a6aaa5e2f6d846b980f993b7c964be9a",
            "3fc93ab0b58a44818f37649ed9ef82fe",
            "35b83e742de7475b999d75cb6fd61af7",
            "344b51abe9b3474ca52c11215844d12f",
            "29b2f53073654023a6ecdc45a28841d4",
            "ed9b3e1f986a4b55b7169c15fa9febfd"
          ]
        },
        "id": "KcBq_eJKJ76s",
        "scrolled": false,
        "outputId": "3e2c1fde-ff8a-495d-d3fa-e5ef076df3fa"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/30 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd09808012e14ffabaeb2e3ebf461de3"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Simulation constants\n",
        "NRUNS = 30\n",
        "NSTEPS = 15\n",
        "\n",
        "# For performance statistics\n",
        "perf_rl   = np.zeros((NRUNS, NSTEPS))\n",
        "perf_demo = np.zeros((NRUNS, NSTEPS))\n",
        "perf_expl = np.zeros((NRUNS, NSTEPS))\n",
        "perf_ref  = np.zeros((NRUNS, NSTEPS))\n",
        "\n",
        "# For reproducibility\n",
        "np.random.seed(40)\n",
        "\n",
        "samples_expl = []\n",
        "samples_demos = []\n",
        "samples_rl = []\n",
        "\n",
        "# Run sims\n",
        "for run in trange(NRUNS):\n",
        "    \n",
        "    # Initialize a random reward\n",
        "    wopt = np.random.randn(len(RFEAT))\n",
        "    ropt = np.sum([wopt[i] * RFEAT[i] for i in range(len(RFEAT))], axis=0)\n",
        "    \n",
        "    #print('Reward:')\n",
        "    #print(np.round(ropt,2))\n",
        "                  \n",
        "    # This is the \"target\" MDP\n",
        "    Mopt = (S, A, P, ropt, gamma)\n",
        "    \n",
        "    # Optimal policy\n",
        "    popt = greedy(vi(Mopt))\n",
        "    \n",
        "    #print('Optimal policy:')\n",
        "    #print(np.round(popt, 2))\n",
        "    \n",
        "    # Weights for 3 approaches\n",
        "    w_rl   = np.zeros(len(RFEAT))\n",
        "    w_demo = np.zeros(len(RFEAT))\n",
        "    w_expl = np.zeros(len(RFEAT))\n",
        "    \n",
        "    # Run test\n",
        "    for t in range(NSTEPS):\n",
        "        #print('\\n[Step %i]' % (t + 1))\n",
        "\n",
        "        # Sample a reward and update\n",
        "        #print('Sampling reward... ')\n",
        "        (s, a, r) = sample_reward(Mopt)\n",
        "        #print('Updating weights for reward... ')\n",
        "        w_rl = rl_update(w_rl, s, a, r)\n",
        "        #print('Weights:')\n",
        "        #print(w_rl)\n",
        "\n",
        "        \n",
        "        # Evaluate\n",
        "        #print('Evaluating reward...')\n",
        "        perf_rl[run, t] = evaluate_reward(Mopt, w_rl)\n",
        "        samples_rl.append(perf_rl[run, t])\n",
        "        #print(voice_reward(Mopt, s, a, r))\n",
        "        \n",
        "        # Sample a demo and update\n",
        "        #print('\\nSampling demo... ')\n",
        "        (s, a) = sample_demo(Mopt)\n",
        "        #print('Updating weights for demo... ')\n",
        "        w_demo = demo_update(w_demo, s, a)\n",
        "        #print('Weights:')\n",
        "        #print(w_demo)\n",
        "\n",
        "        \n",
        "        # Evaluate\n",
        "        #print('Evaluating reward...')\n",
        "        perf_demo[run, t] = evaluate_reward(Mopt, w_demo)\n",
        "        samples_demos.append(perf_demo[run, t])\n",
        "        #print(voice_demo(Mopt, s, a))\n",
        "        \n",
        "        # Sample an contrastive examples and update\n",
        "        #print('\\nSampling contrastive examples... ')\n",
        "        (s0, s1, a0, a1, k) = sample_explain(Mopt)\n",
        "        \n",
        "        #print('Updating weights for contrastive examples... ')\n",
        "        w_expl = explain_update(w_expl, s0, s1, a0, a1, k)\n",
        "        #print('Weights:')\n",
        "        #print(w_expl)\n",
        "        \n",
        "        # Evaluate\n",
        "        #print('Evaluating reward...')\n",
        "        perf_expl[run, t] = evaluate_reward(Mopt, w_expl)\n",
        "        samples_expl.append(perf_expl[run, t])\n",
        "        #print(voice_explain(M, s0, s1, a0, a1, k))\n",
        "        \n",
        "        perf_ref[run, t] = evaluate_reward(Mopt, wopt)\n",
        "        \n",
        "        #print('Done. Step complete.')\n",
        "        \n",
        "# Compute statistics\n",
        "rl_mean = np.mean(perf_rl, axis=0)\n",
        "rl_std = np.std(perf_rl, axis=0) / np.sqrt(NRUNS)\n",
        "\n",
        "\n",
        "\n",
        "demo_mean = np.mean(perf_demo, axis=0)\n",
        "demo_std = np.std(perf_demo, axis=0) / np.sqrt(NRUNS)\n",
        "\n",
        "expl_mean = np.mean(perf_expl, axis=0)\n",
        "expl_std = np.std(perf_expl, axis=0) / np.sqrt(NRUNS)\n",
        "\n",
        "ref_mean = np.mean(perf_ref, axis=0)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ref_mean"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h2Ru-lqnA9k9",
        "outputId": "c4442efc-856b-4d91-b43e-68e6f5e22cb2"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([61.05452237, 61.05452237, 61.05452237, 61.05452237, 61.05452237,\n",
              "       61.05452237, 61.05452237, 61.05452237, 61.05452237, 61.05452237,\n",
              "       61.05452237, 61.05452237, 61.05452237, 61.05452237, 61.05452237])"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rl_min = np.min(rl_mean, axis=0)\n",
        "rl_range = np.max(rl_mean, axis=0) - np.min(rl_mean, axis=0)\n",
        "rl_norm = (rl_mean - rl_min)/rl_range\n",
        "rl_normstd = np.std(rl_norm, axis=0) / np.sqrt(NRUNS)\n",
        "\n",
        "demo_min = np.min(demo_mean, axis=0)\n",
        "demo_range = np.max(demo_mean, axis=0) - np.min(demo_mean, axis=0)\n",
        "demo_norm = (demo_mean - demo_min)/demo_range\n",
        "demo_normstd = np.std(demo_norm, axis=0) / np.sqrt(NRUNS)\n",
        "\n",
        "expl_min = np.min(expl_mean, axis=0)\n",
        "expl_range = np.max(expl_mean, axis=0) - np.min(expl_mean, axis=0)\n",
        "expl_norm = (expl_mean - expl_min)/expl_range\n",
        "expl_normstd = np.std(expl_norm, axis=0) / np.sqrt(NRUNS)\n",
        "\n",
        "\n",
        "ref_min = np.min(ref_mean)\n",
        "ref_range = np.max(ref_mean) - np.min(ref_mean)\n",
        "ref_norm = (ref_mean - ref_min)/ref_range"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Wc3Y_dA-PVx",
        "outputId": "b260e5a5-993c-438c-d5b5-8235f8992b8f"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-23-a6cb8b7f165e>:19: RuntimeWarning: invalid value encountered in true_divide\n",
            "  ref_norm = (ref_mean - ref_min)/ref_range\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ref_min"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uExbHtK7-Tr0",
        "outputId": "34748c2f-ee24-4a96-d75d-c2b8a284f585"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "61.05452236561643"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rl_normstd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K8EGpEea-hNd",
        "outputId": "c32625bd-9672-487e-ebd9-b8ea99050c8b"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.05525263294021012"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470
        },
        "id": "d3cHTDXIJ76s",
        "outputId": "64b09565-53aa-456f-ef9c-8490e0201d70"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 400x500 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAHFCAYAAAAUkEVeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACICUlEQVR4nO2dd3xURdfHf3d7SzaNNGqACAhKFymPvgKKoKKCIooKgthAKYLAg2ADUXxsKKIigoVqwYIKIiqKFJGi8shD7yQESLKb7bv3zvvHZDfZFMi27CZ7vnzuh925c+eezd2dM3PmzDkCY4yBIAiCiEtk0RaAIAiCiB6kBAiCIOIYUgIEQRBxDCkBgiCIOIaUAEEQRBxDSoAgCCKOISVAEAQRx5ASIAiCiGNICRAEQcQxpAQIgiDimKgqgV9++QU33XQTsrOzIQgCvvjiC985t9uNKVOm4LLLLoNer0d2djbuvfdenD592q+NwsJCDBs2DImJiUhKSsKoUaNgsVhq+ZMQBEHUTaKqBKxWK9q3b4/58+dXOmez2bBz507MmDEDO3fuxOeff459+/Zh4MCBfvWGDRuG//73v1i/fj3WrFmDX375BQ888EBtfQSCIIg6jRArAeQEQcDq1atxyy23VFtn+/btuOKKK3Ds2DE0adIEe/fuxaWXXort27ejS5cuAIC1a9diwIABOHnyJLKzs2tJeoIgiLqJItoCBILJZIIgCEhKSgIAbNmyBUlJST4FAAB9+/aFTCbDtm3bcOutt1bZjtPphNPp9L2XJAmFhYVITU2FIAgR/QwEQRC1AWMMJSUlyM7OhkxWvdGnzigBh8OBKVOm4M4770RiYiIAID8/H+np6X71FAoFUlJSkJ+fX21bc+bMwTPPPBNReQmCIGKBEydOoFGjRtWerxNKwO12Y8iQIWCMYcGCBSG3N23aNEycONH33mQyoUmTJjhx4oRPwRAEQdRlzGYzGjdujISEhAvWi3kl4FUAx44dw48//ujXSWdmZqKgoMCvvsfjQWFhITIzM6ttU61WQ61WVypPTEwkJUAQRL3iYibumN4n4FUABw4cwA8//IDU1FS/8927d0dxcTF27NjhK/vxxx8hSRK6detW2+ISBEHUOaI6E7BYLDh48KDv/ZEjR7B7926kpKQgKysLt912G3bu3Ik1a9ZAFEWfnT8lJQUqlQpt2rTB9ddfj9GjR+Ptt9+G2+3G2LFjMXToUPIMIgiCqAFRdRH9+eefcc0111QqHz58OJ5++mnk5ORUed1PP/2E//u//wPAN4uNHTsWX3/9NWQyGQYPHox58+bBYDDUWA6z2Qyj0QiTyUTmIIIg6gU17ddiZp9ANCElQBDVI4oi3G53tMUgKiCXy6FQKKq1+de0X4v5hWGCIKKHxWLByZMnQWPF2ESn0yErKwsqlSroNkgJEARRJaIo4uTJk9DpdGjQoAFtpIwhGGNwuVw4e/Ysjhw5gtzc3AtuCLsQpAQIgqgSt9sNxhgaNGgArVYbbXGICmi1WiiVShw7dgwulwsajSaodmLaRZQgiOhDM4DYJdjRv18bYZCDIAiCqKOQEiAIgohjSAkQBBE35Ofn49prr4Ver/dFI453SAkQBFGvGDFiRLV5SV599VXk5eVh9+7d2L9/PwCgWbNmEAQBgiBAp9Phsssuw3vvvVeLEkcXUgIEQcQNhw4dQufOnZGbm+sXhv7ZZ59FXl4e9uzZg7vvvhujR4/Gd999F0VJaw9SAgRBBITVZYXVZfXbQOYSXbC6rHB6nFXWlZjkK3OLblhdVjg8jhrVDRfNmjXDZ599hg8//BCCIGDEiBG+cwkJCcjMzETz5s0xZcoUpKSkYP369WG7dyxDSoAgiIAwzDHAMMeAc7ZzvrKXfnsJhjkGjP12rF/d9P+kwzDHgOOm476y+dvnwzDHgFFfjfKr2+z1ZjDMMWDv2b2+siW7l4RN7u3bt+P666/HkCFDkJeXh9dff71SHUmS8Nlnn6GoqCikXbh1CVICBEHEBQ0aNIBarYZWq0VmZiaMRqPv3JQpU2AwGKBWq3HbbbchOTkZ999/fxSlrT1oxzBBEAFhmWYBAOiUOl/Z5J6TMf7K8VDI/LuUgkk86ZNWWbbjeEzXMRjdaTTkMrlf3aPjjlaqO6LDiHCKXi2TJ0/GiBEjkJeXh8mTJ+ORRx5By5Yta+Xe0YaUAEEQAaFX6SuVqeQqqOSVzSdV1VXKlVDKlTWuWxukpaWhZcuWaNmyJT755BNcdtll6NKlCy699NJauX80IXMQQRBEORo3bow77rgD06ZNi7YotQLNBAiCqHeYTCbs3r3br6xietoLMW7cOLRr1w5//PEHunTpEmbpYgtSAgRB1Dt+/vlndOzY0a9s1KhR1dSuzKWXXorrrrsOM2fOxLfffhtu8WIKyiwGyixGEFXhcDhw5MgR5OTkBB2mmIgsF3pGNe3XaE2AIAgijiElQBAEEceQEiAIgohjSAkQBEHEMaQECIIg4hhSAgRBEHEMKQGCIIg4hpQAQRBEHENKgCAIIo4hJUAQRNxAieYrQ0qAIIh6RSiJ5rVaLZo1a4YhQ4bgxx9/rEWpowcpAYIg4oaLJZrft28fPvzwQyQlJaFv376YPXt2FKWtHSiKKEEQgWG1Vn9OLgfKBzK7UF2ZDNBqL15XXznZTDA0a9YMx44dAwB8+OGHGD58OJYsWQKgLNE8ADRp0gRXXXUVsrKyMHPmTNx2221o1apVWGSIRWgmQBBEYBgM1R+DB/vXTU+vvm7//v51mzWrul6YqEmi+fKMGzcOjDF8+eWXYZMhFqGZAEEQcUHFRPMXIyUlBenp6Th69GjkhYsipAQIgggMi6X6c3L/5PEoKKi+rqyCISIGO1vGGARBiLYYEYWUAEEQgRGIjT5SdWuB8+fP4+zZs8jJyYm2KBGF1gQIgiCq4PXXX4dMJqvW3bS+QDMBgiDqHYEmmi8pKUF+fj7cbjeOHDmCjz/+GO+99x7mzJmDli1bRlja6EJKgCCIekegieZnzpyJmTNnQqVSITMzE1deeSU2bNiAa665JtKiRh1SAgRB1CuWLFni8/+vCfXd++di0JoAQRBEHENKgCAIIo4hJUAQBBHHkBIgCIKIY0gJEARBxDGkBAiCIOIYUgIEQRBxDCkBgiCIOIaUAEEQRBxDSoAgiLiBEs1XhpQAQRD1ikATzQPArl27cPvttyMjIwMajQa5ubkYPXq0r87Ro0d9yegrHlu3bq2NjxUxSAkQBBE3VJVofs2aNbjyyivhdDqxdOlS7N27Fx9//DGMRiNmzJjhd/0PP/yAvLw8v6Nz587R+ChhgwLIEQRRIxgDbLbo3FunA0JN8FVVovm33noL9913HwYMGIDVq1f76ubk5KBbt24oLi72ayM1NbVGqSnrEqQECIKoETZbWPO+B4TFEnrise3bt+Pee+9FYmIiXn/9dWi1Wqxbtw7nzp3DE088UeU18bBuEFVz0C+//IKbbroJ2dnZEAQBX3zxhd95xhhmzpyJrKwsaLVa9O3bFwcOHPCrU1hYiGHDhiExMRFJSUkYNWoULBfKgUoQRFxSMdG80Wj09SetW7euURs9evSAwWDwO+o6UZ0JWK1WtG/fHiNHjsSgQYMqnZ87dy7mzZuHDz74ADk5OZgxYwb69euHf/75BxqNBgAwbNgw5OXlYf369XC73bjvvvvwwAMPYNmyZbX9cQiiXqPTXTjHfKTvHQkYYwHVX7lyJdq0aRMZYaJEVJVA//790b9//yrPMcbw2muv4cknn8TNN98MgNvxMjIy8MUXX2Do0KHYu3cv1q5di+3bt6NLly4AgDfeeAMDBgzAf/7zH2RnZ9faZyGI+o4gxFwu+JC55JJLAAD/+9//0L1794vWb9y4cb1LNxmz3kFHjhxBfn4++vbt6yszGo3o1q0btmzZAgDYsmULkpKSfAoAAPr27QuZTIZt27ZV27bT6YTZbPY7CIKIP6677jqkpaVh7ty5VZ6vuDBcH4nZheH8/HwAQEZGhl95RkaG71x+fr7PzcuLQqFASkqKr05VzJkzB88880yYJSYIIlaoaaJ5vV6P9957D7fffjsGDhyIxx57DC1btsS5c+ewatUqHD9+HCtWrPDVP3/+fKW+JSkpyWeerovErBKIJNOmTcPEiRN9781mMxo3bhxFiQiCCCeBJJq/+eabsXnzZsyZMwd33XWXrz/o3bs3Zs2a5Ve3vGXCy/LlyzF06NDwCV/LxKwS8PrinjlzBllZWb7yM2fOoEOHDr46BQUFftd5PB4UFhZe0JdXrVZDrVaHX2iCIKJOoInmAaBLly747LPPqj3frFmzgBeR6woxuyaQk5ODzMxMbNiwwVdmNpuxbds23wJO9+7dUVxcjB07dvjq/Pjjj5AkCd26dat1mQmCIOoaUZ0JWCwWHDx40Pf+yJEj2L17N1JSUtCkSROMHz8es2bNQm5urs9FNDs72xcXpE2bNrj++usxevRovP3223C73Rg7diyGDh1KnkEEQRA1IKpK4I8//sA111zje++10w8fPhxLlizBE088AavVigceeADFxcXo1asX1q5d67cIs3TpUowdOxZ9+vSBTCbD4MGDMW/evFr/LARBEHURgdVXQ1cAmM1mGI1GmEwmJCYmRlscgogJHA4Hjhw5gpycnDrt/VKfudAzqmm/FrNrAgRBEETkISVAEAQRx5ASIAiCiGNICRAEQcQxpAQIgiDiGFICBEEQAfL000/7IhfU9fuQEiAIol5y4sQJjBw5EtnZ2VCpVGjatCnGjRuH8+fPB9ROVQmvJk2a5BfNoC5DSoAgiHrH4cOH0aVLFxw4cADLly/HwYMH8fbbb2PDhg3o3r07CgsLQ2rfYDBUGZW0LkJKgCCIgLBarbBarX4B1VwuF6xWK5xOZ5V1JUnylbndblitVjgcjhrVDYYxY8ZApVLh+++/x9VXX40mTZqgf//++OGHH3Dq1ClMnz4dAA8M99xzz+HOO++EXq9Hw4YNMX/+fF87zZo1AwDceuutEATB976imWbEiBG45ZZb8PzzzyMjIwNJSUl49tln4fF4MHnyZKSkpKBRo0ZYvHixn5xTpkzBJZdcAp1Oh+bNm2PGjBlBf+ZgISVAEERAeHPrnjt3zlf20ksvwWAwYOzYsX5109PTYTAYcPz4cV/Z/PnzYTAYKoV2btasGQwGA/bu3esrCzQaKMDzjq9btw6PPPIItFqt37nMzEwMGzYMK1eu9Cmxl156Ce3bt8euXbswdepUjBs3DuvXrwfAk9MDwOLFi5GXl+d7XxU//vgjTp8+jV9++QWvvPIKnnrqKdx4441ITk7Gtm3b8NBDD+HBBx/EyZMnfdckJCRgyZIl+Oeff/D6669j4cKFePXVVwP+zKFASoAgiHrFgQMHwBirNhdwmzZtUFRUhLNnzwIAevbsialTp+KSSy7Bo48+ittuu83XETdo0AAATxyTmZnpe18VKSkpmDdvHlq1aoWRI0eiVatWsNls+Pe//43c3FxMmzYNKpUKmzZt8l3z5JNPokePHmjWrBluuukmTJo0CatWrQrXn6JGxGw+AYIgYhNLabZ5Xbns75MnT8b48eOhUPh3Kd58H+VH5GPGjMHo0aMhl8v96h49erRS3REjRgQtZ03DolXMLdy9e3e89tprAd+vbdu2kMnKxtUZGRlo166d771cLkdqaqpfDpSVK1di3rx5OHToECwWCzweT63HL6OZAEEQAaHX66HX6yEIgq9MpVJBr9dXStbkrVu+c1QqldDr9ZUCnlVXN1BatmwJQRD8zErl2bt3L5KTky84qg+GirIKglBlmXfNY8uWLRg2bBgGDBiANWvWYNeuXZg+fTpcLldY5boYpAQIgqhXpKam4tprr8Vbb70Fu93udy4/Px9Lly7FHXfc4VNiW7du9auzdetWP1OSUqmEKIphl3Pz5s1o2rQppk+fji5duiA3NxfHjh0L+30uBikBgiDqHW+++SacTif69euHX375BSdOnMDatWtx7bXXomHDhpg9e7av7m+//Ya5c+di//79mD9/Pj755BOMGzfOd75Zs2bYsGED8vPzUVRUFDYZc3NzfYnsDx06hHnz5mH16tVha7+mkBIgCKLekZubiz/++APNmzfHkCFD0KJFCzzwwAO45pprsGXLFqSkpPjqPv744/jjjz/QsWNHzJo1C6+88gr69evnO//yyy9j/fr1aNy4caXk9aEwcOBATJgwAWPHjkWHDh2wefNmzJgxI2zt1xRKKgNKKkMQVREPSWWaNWuG8ePHY/z48dEWJSgoqQxBEAQREqQECIIg4hjaJ0AQRNzi3ZsQz9BMgCAIIo4hJUAQxAUh35HYJRzPhpQAQRBV4g3rUNs7WImaY7PZAAS3s9oLrQkQBFElCoUCOp0OZ8+ehVKp9AvnQEQXxhhsNhsKCgqQlJRUKQ5TIJASIAiiSgRBQFZWFo4cORKVcAbExfFGNw0FUgIEQVSLSqVCbm4umYRiEKVSGdIMwAspAYIgLohMJqu3O4YJWhgmCIKIa0gJEARBxDGkBAiCIOIYUgIEQRBxDCkBgiCIOIaUAEEQRBxDSoAgCCKOISVAEAQRx5ASIAiCiGNICRAEQcQxpAQIgiDiGFICBEEQcQwpAYIgiDiGlABBEEQcQ0qAIAgijiElQBAEEceQEiAIgohjSAkQBEHEMaQECIIg4hhSAgRBEHEMKQGCIIg4hpQAQRBEHENKgCAIIo4hJUAQBBHHkBIgCIKIY0gJEARBxDExrQREUcSMGTOQk5MDrVaLFi1a4LnnngNjzFeHMYaZM2ciKysLWq0Wffv2xYEDB6IoNUEQRN0hppXAiy++iAULFuDNN9/E3r178eKLL2Lu3Ll44403fHXmzp2LefPm4e2338a2bdug1+vRr18/OByOKEpOEARRNxBY+WF1jHHjjTciIyMDixYt8pUNHjwYWq0WH3/8MRhjyM7OxuOPP45JkyYBAEwmEzIyMrBkyRIMHTq0Rvcxm80wGo0wmUxITEyMyGchCIKoTWrar8X0TKBHjx7YsGED9u/fDwD4888/sWnTJvTv3x8AcOTIEeTn56Nv376+a4xGI7p164YtW7ZU267T6YTZbPY7CIIg4hFFtAW4EFOnToXZbEbr1q0hl8shiiJmz56NYcOGAQDy8/MBABkZGX7XZWRk+M5VxZw5c/DMM89ETnCCIIg6QkzPBFatWoWlS5di2bJl2LlzJz744AP85z//wQcffBBSu9OmTYPJZPIdJ06cCJPEBEEQdYuYnglMnjwZU6dO9dn2L7vsMhw7dgxz5szB8OHDkZmZCQA4c+YMsrKyfNedOXMGHTp0qLZdtVoNtVodUdkJgiDqAjE9E7DZbJDJ/EWUy+WQJAkAkJOTg8zMTGzYsMF33mw2Y9u2bejevXutykoQBFEXiemZwE033YTZs2ejSZMmaNu2LXbt2oVXXnkFI0eOBAAIgoDx48dj1qxZyM3NRU5ODmbMmIHs7Gzccsst0RWeIAiiDhDTSuCNN97AjBkz8Mgjj6CgoADZ2dl48MEHMXPmTF+dJ554AlarFQ888ACKi4vRq1cvrF27FhqNJoqSEwRB1A1iep9AbUH7BAiCqG/Ui30CBEEQRGQhJUAQBBHHkBIgCIKIY0gJEARBxDGkBAiCIOIYUgIEQRBxDCkBgiCIOIaUAEEQRBwTsBI4c+YM7rnnHmRnZ0OhUEAul/sdBEEQRN0h4LARI0aMwPHjxzFjxgxkZWVBEIRIyEUQBEHUAgErgU2bNuHXX3+9YKhmgiAIom4QsDmocePGoHBDBEEQ9YOAlcBrr72GqVOn4ujRoxEQhyAIgqhNAjYH3XHHHbDZbGjRogV0Oh2USqXf+cLCwrAJRxAEQUSWgJXAa6+9FgExCIIgiGgQkBJwu93YuHEjZsyYgZycnEjJRBAEQdQSAa0JKJVKfPbZZ5GShSAIgqhlAl4YvuWWW/DFF19EQBSCIAiitgl4TSA3NxfPPvssfvvtN3Tu3Bl6vd7v/GOPPRY24QiCIIjIEnCO4QutBQiCgMOHD4csVG1DOYYJgqhv1LRfC3gmcOTIkZAEIwiCIGIHiiJKEAQRxwQ8Exg5cuQFz7///vtBC0MQBEHULgErgaKiIr/3brcbe/bsQXFxMXr37h02wQiCIIjIE7ASWL16daUySZLw8MMPo0WLFmERiiAIgqgdwrImIJPJMHHiRLz66qvhaI4gCIKoJcK2MHzo0CF4PJ5wNUcQBEHUAgGbgyZOnOj3njGGvLw8fPPNNxg+fHjYBCPqPoz5H1WVlS+vyf81rRMotZ0gry6n5Ajn3yraf4eKnyXa8lSHRgPodJFpO2AlsGvXLr/3MpkMDRo0wMsvv3xRzyGibsAYIEmAKPL/y78uXwaUvS5/eMsrdvzlX19ICQhC9f9fTO6adlAVf+zRypJaVacT7YytF+oIwy1bbT2HQP/O1f0NovFsPB4gKyuGlMBPP/0UCTmIWqC6Dl0U+eFy8S+cx1O5bsXOuLoDqPxaJvMvv1B9giD8KS6ObPsBK4HevXvj888/R1JSkl+52WzGLbfcgh9//DFcshEhIopAYSHgdPKOvarRvLdjl8n4IZeX/a9Ulr0nCKJ+ErAS+Pnnn+FyuSqVOxwO/Prrr2ERiggPRUXAmTOASlXWySuV3L4ok9HomyCIAJTAX3/95Xv9zz//ID8/3/deFEWsXbsWDRs2DK90RNBYLMDZs4BeD6jV0ZaGIIhYpcZKoEOHDhAEAYIgVLkzWKvV4o033gircERweDxcAchkpAAIgrgwNVYCR44cAWMMzZs3x++//44GDRr4zqlUKqSnp0Mul0dESKLmMAacPw9YrUBycrSlIQgi1qmxEmjatCkAHiKCiF1KSoBz54CEBLL5EwRxcYLy+/joo4/Qs2dPZGdn49ixYwCAV199FV9++WVYhSMCw+XiZiClkh8EQRAXI2AlsGDBAkycOBEDBgxAcXExRFEEACQnJ+O1114Lt3xEDfGagex2wGCItjQEUYeoagdjqEf5DTgeD+B281Ga0wk4HPyw2/lhs3H7rcXCj5ISwGwGTCa+SaC4mNeJEAG7iL7xxhtYuHAhbrnlFrzwwgu+8i5dumDSpElhFY6oOWYz3xNA2TEJ4gIwxjtkb8fsdPKOOBLxIqraKl/V6/L1gcp23EIRMDYAEJktw0Gll+zYsWOlcrVaDavVGhahiMBwOLgZSK0GFAE/UYKop1TX4Xu3xTPGd0MqFOFfQLvQlvqK5y+GxRJe2SoQcJeRk5OD3bt3+xaKvaxduxZt2rQJm2BEzZAkvhDsdJI3EBHHBNLhq1Q8EA95TgAIMoromDFj4HA4wBjD77//juXLl2POnDl47733IiEjcQG8JkOjMdqSEEQtEUiHr1ZTh38RAlYC999/P7RaLZ588knYbDbcddddyM7Oxuuvv46hQ4dGQkaiGux2PgvQavl3niDqDYyVBbzyLq6KIrd9ulxlZd4OX6mkDj9IAlICHo8Hy5YtQ79+/TBs2DDYbDZYLBakp6dHSj6iGkSRrwN4POQNRNRRvB29N2xteS8ab0fv9bIByiIbyuXU4YeRgJSAQqHAQw89hL179wIAdDoddJEKck1ckKIi7hFUIZgrQcQW3pF8+cNrwnG7y8rKe8Z4O3qVisLY1gIBm4OuuOIK7Nq1q9LCMFF7WK3cDKTX0++DiBKMlXXg5ZNPlPeLd7v9z1XV0avVZSFuiagQsBJ45JFH8Pjjj+PkyZPo3Lkz9Hq93/nLL788bMIRlfEGhwMoOBwRIbymmYqduyRVnXnIm3XIm3HI26mXj19OHX3MIjAW2C4JWRUPUhAEMMYgCIJvB3Fdwmw2w2g0wmQyITGGd1sxBhQUcCWQnEzmUCIC2Gxli03lO3ivT3v5zt1rqqHkFBGl+JQFac2NSG/b4OKVy1HTfi2ozWJEdLBYeGgICg5HRAS7nWchEsWyzEM0eq/3BKwEaC0gOrhcfBagUFBwOCICOBxcAXg8fJRBxA2k5usA5YPDVViCIYjQ8YafdTpJAcQhpATqAOWDw5EZiAgrbjefYtpsFH0wTqFwYyHiDZqn0+kglPbQLpcLbrcbCoUC6nIuPN66Wq3Wt8Dudrvhcrkgl8uh0Wgq1ZXJtDh7Vga1GmDMDZvNBZnMv67dbgNjDGq1xpfdzePxwOVyQhBk0Gq1Qda1gzEJKpUaitLIdKIowul0BFhXgFZbtp/E4XBAkkQolSooS21bgdSVJAkOh7307142NXI6nfB4PFAolFCpVBCEC9cVxbK6AMAYg91uK31G/s/T43EHVFcu93/2Nht/nhpN2bMPpK7b7Ybb7YIg8Gfvdeeo+nnyut7viXfgUKmuxwPP6dNwFZ6HkJQEbbkRht1h53VVav/vidvFn5FGW6muSqnyf/YuJ4TSv08wdR1OByRJgkKhhFKhLI3QLMHhdPDnqdX5PpvTxesqFUr/70m5ur5n73JCFEUoFAqolOWep/d7EkBdrUZb9uzdLng8HsjlcqhV5Z6n3QbGqq4rk1VdV6PW+J69w+lGRHN5MYKZTCYGgJlMpoCvBcAAsIKCAl/ZrFmzGAB2//33+9XV6XQMADty5Iiv7NVXX2UA2F133eVXNy0tjQFgGzbsYXv2MHbqFGNz577LALB+/W5mp04x39GoUVMGgH3zze++sjfe+JgBYP/6V1+/updccikDwD755Cdf2aJFqxkA1qVLD7+67dt3YQDYBx+sYadOMXbkCGNvvvk9A8BatGjP1q1j7MsvGfvkE8Zat7669DOvYq++ytiLLzI2atQmBoAlJbVkjz3G2IMPMjZiBGPZ2QMYANa+/WJ23XWM9e7NWOfOuxgAplRmsy5dGOvYkbHLLmMsIeE2BoClpr7JmjZlrGFDxlJT95f+3Y3MYGBMo2FMqWQMGF5aPrdcYPeTpWUKJgiMyWT8EIRHGAAmCE8xhYJfr1AU+Z6nSuViGg1vWy6fxAAwuXySr0ytdpWrW8TUasbUasbk8qcYACaTPcJUKuY7AAUDwBSKk6X3Ykwmm1sqw3Aml5fJBhhL295f7nO8WVp2W4XA9dml5bvKlS0uLRtQoW7L0vJN5cpWlZZdzQRBYoIgMZlMYkD70s+xjsnlEpPLJSaTfV1atwtTKiTfIQg9+GeTf87UKompVRJTKn4s/WyXMo1a8pULQl/+t5R9xBRyiSnkEpPJtpW229QnA5ft5tLyd8vJu6e0LK3CZ7urtPzVcmWHS8t0TCaTfJ9DEEbxzyY85/sMCvkZ3/P0yqWQS0wQHiv9HNOYTMb/NoJQ4qsLlJST99+lZeMqyOatW1CubFZp2f0V6upKy4+UK3uV3X2bLeC+qab9WlAzgeLiYnz66ac4dOgQJk+ejJSUFOzcuRMZGRlo2LBhME1Wy6lTpzBlyhR89913sNlsaNmyJRYvXowuXboA4Fr5qaeewsKFC1FcXIyePXtiwYIFyM3NDasc0cJsBmrzozidwPHjwJEjZfsR5s4FnnwSOHUKvhHJoUNAv36Vr68qhmBxMTBvXuXyP/+sXOZ2A3/8Ubn8/Hl+VCSQKLvlw7iXL/N4Ktd1uSqXed3la1LX61Jfkaru5d13FW0YE0r/LyuTJAGAd4ZQNlNweyrbJT2iAI/oX5cxwOGsXFespl2vDOGhrC3+OfzLJSZA8lSWoewzlJUzJpSTzf98eGWufQLeJ/DXX3+hb9++MBqNOHr0KPbt24fmzZvjySefxPHjx/Hhhx+GTbiioiJ07NgR11xzDR5++GE0aNAABw4cQIsWLdCiRQsAwIsvvog5c+bggw8+QE5ODmbMmIG///4b//zzj5/J5EKEsk8gUuagc+esOHkS0Ou10Ov9TQKhmoPsdoYzZzQ4flyOI0eAw4c9OHzYiePHZTh9WluuE7ADkACo4bUcCoIIjcYBlYq3q1LxTWtKpR1KpQStVg2NRlFaJkKhcECtFqDX63x1ZTIHVCoRWq0KGo2ydC+RCMYcUCgE6HQ6XxBISXJAEHhdtVpZGvpdgijaoVAABoMeCoW3rhOS5IFKpYRCoSozH5RO3TUave+zec1BcjmvCwCSxOBwcBOPWq0r3f8CeDxl5iBlOZOAw2GDIAAaTdmzd7vLTDyqctN8u91a2q4Wcrms9Bl52+V1vaYNh8PfHCQI/Nl7PJW/J1U9+/LfE7VaUyqvt64EtcUGWbEJMBjgAfN9TzTqsmfvcNohMQkqJW+XMcAjeuB2OwHwut52HU5uClQq1FDI+ffEI4pwuR0QIECt1kMQGITSdhlKzYZyRanJrqyu1xQjCIDL7YAoilCpuDlIACAxCQ5nqdlGw817DNwc5PFIUCpVUMi9JkYJTpe91BSjh/dr7XQ6IUoeyGX8efLPzOBw8mev05aZDd1uXlcpL3v25etqyz17j8cFt8cNhVwBtVrla8NW+uy9dYXS74m3bvnvic377NVlfUThiWI0vjQFOV0js08gYCXQt29fdOrUCXPnzkVCQgL+/PNPNG/eHJs3b8Zdd92Fo0ePBiTohZg6dSp+++03/Prrr1WeZ4whOzsbjz/+uC+rmclkQkZGBpYsWVJtVFOn0wmn0+l7bzab0bhx45jZLCaKwOnTPMtcKLGBCgqAnTuBo0eBw4f5/0eOAHl5lUfE5dHrgZwcfjRr5v+6QQNanK6zMMaDThUU8KiDlIGoThBzm8W2b9+Od955p1J5w4YNkZ+fH2hzF+Srr75Cv379cPvtt2Pjxo1o2LAhHnnkEYwePRoA37iWn5+Pvn37+q4xGo3o1q0btmzZUq0SmDNnDp555pmwyhpOiop4etFQFIDJBPTuzduqCoOh+o4+LY06+npJcTEPOqXTkQIgfAT8TVCr1TCbzZXK9+/fjwYNAtNUF+Pw4cO+xPb//ve/sX37djz22GNQqVQYPny4T+lkZGT4XZeRkXFBhTRt2jRMnDjR9947E4gFwhUc7sMPuQJITQV69Srr4Js1A5o3B1JSqKOPK8zmshykKtXF6xNxQ8BKYODAgXj22WexatUqADxu0PHjxzFlyhQMHjw4rMJJkoQuXbrg+eefBwB07NgRe/bswdtvv43hw4cH3a5arfaz1ccK4QoOZ7eXLdDOnAncdlvoshF1GIuFm4C8izIEUY6Ax5ovv/yyL5GM3W7H1VdfjZYtWyIhIQGzZ88Oq3BZWVm49NJL/cratGmD48ePAwAyMzMBAGfOnPGrc+bMGd+5uoTVyn+voW7aXLWKzyYaNgRuvjk8shF1FJuNh4OQyXg8IIKoQMAzAaPRiPXr12PTpk3466+/YLFY0KlTJz+7fLjo2bMn9u3b51e2f/9+X/yinJwcZGZmYsOGDejQoQMAbtrZtm0bHn744bDLE0kY43Z8pTI0M43HA3iXbB56iOIMxTXegHCMUbwRolqCXh3q1asXevXqFU5ZKjFhwgT06NEDzz//PIYMGYLff/8d7777Lt59910A3BQ1fvx4zJo1C7m5uT4X0ezsbNxyyy0RlS3c2O3hmQV88w1w7Bi3+d95Z3hkI+ogFBCOqCEBK4F5Ve36Ae+QNRoNWrZsiauuusrnsxwKXbt2xerVqzFt2jQ8++yzyMnJwWuvvYZhw4b56jzxxBOwWq144IEHUFxcjF69emHt2rU13iMQK5SU8P9D+bMxBrz5Jn89ciRPQE/EIeUDwhmN0ZaGiHEC3ieQk5ODs2fPwmazITk5GQDf1KXT6WAwGFBQUIDmzZvjp59+ihmPm4sR7aQybjf34ZfLQzPb/vQTcPfd3APw99954hkiznC7+QzAaqWIg/WESO8TCHhh+Pnnn0fXrl1x4MABnD9/HufPn8f+/fvRrVs3vP766zh+/DgyMzMxYcKEQJuOW6xWPmgLdfIyfz7//+67SQHEJR4P9wKyWEgBEDUm4JlAixYt8Nlnn/kWYr3s2rULgwcPxuHDh7F582YMHjwYeXl54ZQ1YkRzJiBJPFaPy8U3cAXLjh3AwIF8IXjzZiA7O3wyEnUAUeQmoOJirgAoI1i9IeZ2DOfl5cFTRRQsj8fj26CVnZ2NEq+Rm7ggdnvZzD0U3nqL/z9oECmAuEOSeHS94mK+CEwKoMYIlhLITx3zj6Mil8OTW+aaLj9+GEJpeO/KDQjwtGpXVvfEUchKiiE4HRDstrLDYYfgcsF612hfXf3y96D66w9+rny90v/PfLujVnZ2B3yHa665Bg8++CDee+89dOzYEQCfBTz88MPo3bs3AODvv/9GTk5OeCWtp5jNfNYeyoLwgQPA2rW8nTrmGUuECmM841BhIVcAYXDIiAcSX30G6i0/QXF4P4QKxhApMQn5P+31vU96fgrU2zdV2Q5TqpC39ZjvvfGVp6D55ftq72sdOsqnpFW7f4f2h6+rrSs47GCGyHt2BawEFi1ahHvuuQedO3f2JW/weDzo06cPFi1aBAAwGAx4+eWXwytpPcTl4l5BoXrxeGcB119fu2GniSjjVQDnznFbIikAP2Tnz0K5ZydUf++A/PQJFD+/wHdOcWQ/lIf4HiQxJQ1QlG2okRL8p+VSUgrE9Kwq78EqbMSRjMkQM7LBNFpIGi2YVgdW7n+IHkDGw3bYbrgdrg5X+M5JFeoyTe249wW8JuDlf//7H/bv3w8AaNWqFVq1ahVWwWqTaK0JFBXxGP0pKcG3ceoU0KMHXxP8+mugU6fwyUfEMF4TUGEhdwejXYFQHNgL9fZNvo5fcfqE3/n8tbshNeBxxlS/b4Jgt8LdrhOk1PDGPAs3Mbcm4KV169Zo3bp1sJfHPZLETbihhnJ5912uAHr0IAUQN3gVwPnz8RkSmjHITx2H6u8dsF/THygdMes//wj6VYvLqgkCPM0vgatdZ7jbdQQr537nuiKyG13rEkF9e06ePImvvvoKx48fh6tC+qRXXnklLILVd2w2foSyl6ewEFi6lL8eMyY8chExjihy809RUVysAcjO5kO1ZxfkJ45CcfIIFCeOQnHgH8iLeJo5cdGXcHW4AgDg7NoL8vxTcLXrCNdlneG+tEOt2NTrOgErgQ0bNmDgwIFo3rw5/ve//6Fdu3Y4evQoGGPoREPRGmM28/WhUBw5PviAexe1bQtcfXX4ZCNilPJuoPVEAQiWEshPHuWd+4kjkJ88Cuud9/u8czS/rkfS7CcqXccUSrhbX8Y3x5Xi6D0Ajt4Dak32+kLASmDatGmYNGkSnnnmGSQkJOCzzz5Deno6hg0bhuuvvz4SMtY7nM7QF4RtNqB0HR5jxtC+oHqPN864yVS3FABjEExFgEoNpuNB7FS//4rEBXMhP3HEN6Ivj6tDN58S8ORcAlfbDvA0agaxUTN4GufA06wF3Je0BdR1KzRMrBKwEti7dy+WL1/OL1YoYLfbYTAY8Oyzz+Lmm2+uc9E7o4HFwgcwocT1Wr6cWwSaNQNuuCFsohGxiNvNdwKXlMTsRjBZ/iloftsA+Zk8yAv4IfP+b7Wg6JnXYb9xCABAkCSo/vrDd62YkgaxcQ48jZrB07gZ3K3L/O5dHbvh3Iff1frniScCVgJ6vd63DpCVlYVDhw6hbdu2AIBz586FV7p6iCjy2XwoISLcbuDtt/nrhx6Kv3XBuMIbDC4KCkCwWaE4tA/ys3m+zt3bscvP5KHkoUmwD+AZi5RHDyLp+SnVtiU/V+B77WpzOQpfeBeexnx0T3b76BJw93HllVdi06ZNaNOmDQYMGIDHH38cf//9Nz7//HNceeWVkZCxXmGzcTt+KLF9vviCJ6Jv0AC4/fawiUbEGi6XfyygSCsAxvzsiqq//kDqmKrzdAOA/NRx32tPo2awX90PUnoWxPRMiOlZENOzIWZkQUrPAtPqym5jTIbj2psi8xmIgAlYCbzyyiuwWCwAgGeeeQYWiwUrV65Ebm4ueQZdBMb4grBCEbwNX5LKNoeNHk3JouotTidXAF4XslpY9DE+9zik5FSUPDwFUCggZmTDk5ENKSO7tFPP5B17eiak9Cx4mrbwXSs2aoqiV5ZEXEYi/ASkBERRxMmTJ3H55ZcD4Kaht712CeKihGNB+IcfgP37+XrCPfeETzYihnA4uAJwOGotGqj2y+XQf7kcTCaD45oBcLfrCE9OLgq+3RHxexPRJSAlIJfLcd1112Hv3r1ISkqKkEj1F4uFrwkEu7mzfNKYe+8NPehcpGCsLB5XTV9X1UbFc1W9ru58TYkVryqf7A47UHAWcLsAQyJgibyA6gN7kPnCvwEAZ0dOwfmmnYBy8R9r408UVNiCIAnk89SmXNXhdkf2CQRsDmrXrh0OHz5MAeICxOPh3n2hmG9+/52HjFargVGjwifbhWCML0R7PPx/SfI3HVf12vve+7ri+4qvvZR/X94Dsrwp3Pu6/PVVnb/Q5yn/f0xht0EwnwF0biAxsfTzRVZQwWxCyjOjIHM54PxXX2DMWKQIZfes7b9TVd+LUAh1kFCRC31nI4bMA53u4tWCJWAlMGvWLEyaNAnPPfccOnfuDH2FBNbRyMxVFwjHgrB3FnD77UBGRnjkKo8kVe7wBYHPXBQKLrtazTvamnTugSgEL7EyMq91rFbAkgcYPaVTPCny92QMeGIccPIY0Lgx1AteQ3oyaufeRM1RiIA+cto4YCUwYADfkTdw4EAI5X6xjDEIggBRFMMnXT2BMT4LUCqD7+T++Qf48UfeAT/0UOgyiaJ/h88Yb1up5IfRyDt87/tQFrOJi2CxAHl5XOvW5iDq7beBdesAlQp45x1KRxenBKwEfvrpp0jIUa9xOPjvvMKkKSC8HkE33ggEYoljjHf03s7eq6Plct65a7U8iqlK5d/hE7VESQlXAEBouweDITOTfwFmzgTat6/dexMxQ8A/96spSE3AlJTwQV6wnevx48BXX/HXgQSK83i4S6q3Y09M5GsS3s5eqaw70QfqJSYTkJ/Pp2ChjBCC5dZbgSuuoFR0cU5Qu09+/fVX3H333ejRowdOnToFAPjoo4+waVPV2XfiGbebd8ShLAi/8w4fwV99NdCu3cXre/F6GObk8CM7m4/6ExK4PKQAokhxMZ8ByOW1qwA8Hh5+1kvDhmTni3MCVgKfffYZ+vXrB61Wi507d8LpdAIATCYTnn/++bALWNex2XhnHKwSOHcOWLGCvw40XLTLxZWAShWT4Wbil6IivuVbqURE3T6q4qWXgGuvBbZvr937EjFLwF3DrFmz8Pbbb2PhwoW+9JIA0LNnT+zcuTOswtV1GOMDPpUq+MHWokVciXTsyBPH1BSXq8zmT8QI3kBwp0/zVffafjjff89dzPLzy9YhiLgnYCv1vn37cNVVV1UqNxqNKC4uDodM9Qa7nXv+BbveV1LCcwYAgYeLdjj4fUPNXEaEAUniD/PcOf6lMBj4yKA2OXYMGDeOvx41Chg4sHbvT8QsAc8EMjMzcfDgwUrlmzZtQvPmzcMiVH2hpHTXZbC296VL+dphixZAv341v87rEVTbziZEFdhsPBH0iRNcGSQn174CcDiABx7gi1OdOwNPPlm79ydimoCVwOjRozFu3Dhs27YNgiDg9OnTWLp0KSZNmkS5BMoR6oKw0wksXMhfjxkTmE3f5YqOtSEmsFq5P64U5Q1PXtPP8eN8NGA08gXgaCzCzpwJ7NnDvQIWLKh9JUTENAGbg6ZOnQpJktCnTx/YbDZcddVVUKvVmDRpEh599NFIyFgnsVp5R56SEtz1n3/OTbeZmdyTLxC8O5ODjVFUZ3G7+R/N4eAdbnIy/782Nz5UNP3o9dG1yX31FZ9SCgIwfz73BiKIcgT86xAEAdOnT8fkyZNx8OBBWCwWXHrppTAYDJGQr04iSXxBONjfviiWbQ574IHABm6M8fvH5eMoLuYdr9HI/z9xgk/FkpL4HyTScbdtNuD8eW7DU6u5Eoq2++U11/DUc23aAFWs5RFEwErg448/xqBBg6DT6XDppZdGQqY6j3dB2GgM7vq1a4HDh3nfNWxYYNc6HNwMFHemILud+7/r9XwRxmDgGtHhAM6c4Z1zYiI/tNrw+sy63dzts7CQa2CjMXY2YSQk8I0mMRkxj4gFAv4lTJgwAenp6bjrrrvw7bffUqygKjCb+QAwmH6GMT5rB4ARIwIf0Tud/HcfV6EfGOMdsMfjP/0SBN7heyPfFRUBR49yO73JxOuHgiTxdo4f5/Z/tZpr7mgrAMa4O6i34w/2y0jEBQF/M/Ly8rBixQoIgoAhQ4YgKysLY8aMwebNmyMhX53D5QotccymTcCff3LLxciRgV3rDfMcjQgEUaWkhJuCLhR8TaXiHXRiIn9IJ05wt8lz57jmDJSqvH5ixR/3vfeA++4DHn2UZgDERQl4vKhQKHDjjTfixhtvhM1mw+rVq7Fs2TJcc801aNSoEQ4dOhQJOesMVivvY4K1yXtnAXfdBaSmBnatd2dyXJmCPB5u6qlpIKSKpqL8fH9TkU53YTt+LJt+AL4TeNYs/rpTp+ivSRAxT0hGA51Oh379+qGoqAjHjh3D3r17wyVXncS7IBzs+uNffwG//sr7lAceCPx6h4N7E8XVzN9k4po30DDIXlORVsu1dlERP8p7FZXv3GPN66cqzp3jccY9HuDmm/lsgIg9vGn1vFP38q+ByuVud0TFCUoJeGcAS5cuxYYNG9C4cWPceeed+PTTT8MtX53CZivLCx4M3lnAzTcDjRsHdq0o1n4ssqjjdPIRuVYb2ohXpeKHKPIHePw4b9PrVSSKsef1UxFR5BtK8vOBli2BuXNjT8ZYQxT5AMCbQSmcVExpVj79XvmsTOVfew/vrFYu5+fl8ojGmApYCQwdOhRr1qyBTqfDkCFDMGPGDHTv3j0SstU5TCb+zIIZiZ84AXzzDX/9yCOBX+/1Coq0F2TM4F0MDmUzRkXkcr6q7jUV5eVx5SBJsWn6Kc/LL/MFJa0WePfdOPURvghuNz9crrLY7t4MSpHYVCOXV93ZX+x1LROwEpDL5Vi1ahX69esHeYUfxJ49e9AukFjH9Qink29UDdYe/8UXvO/p0YO7dAdz/7S0OBr8Wa3cfBOJ2BgVTUVAbO+yPXWqbGPJSy8BrVpFV55YwJsc23swVtbpp6Xx0ZJ3BhhX9tPKBKwEli5d6ve+pKQEy5cvx3vvvYcdO3bErcuoxcK/a8H2SV9+yf8fNCjwa91u/v2OmwVhSeLmGW8+zEgSy52/l4YNgVWrgF9+CXx7eX1BkspMO253mVnFa75Tq8s6/bgZKdWMoBeGf/nlFyxatAifffYZsrOzMWjQIMz3GrXjDFHkC8LBdsL79gF79/LvbP/+gV/vcHCTYaytU0YMs5kv0iYlRVuS2OGKK/gRL5S354si79hVKv5D8EZp9eZMJS5IQEogPz8fS5YswaJFi2A2mzFkyBA4nU588cUXcb172GYri9cTDF98wf+/5prg+jW3m3sFxcUAx+3mswCNJu6n8XjzTZ4gpr6af7zhcEWRHx5P2QKuTMY7eaORj768nX5c7ZIMDzX+Fd10001o1aoV/vrrL7z22ms4ffo03njjjUjKVidgjA9MFYrgOmHGykxBt9wS+PUuF//ux40pqKiIa9zazsgVa6xeDcyZA9x0E3cNrat4zTh2O7epmkxl+zBMJj7NZYx/yZOTeY7Uxo3LcqZmZfGRk05HCiBIavxX++677/DYY4/h4YcfRm5ubiRlqlM4naHtEN69m29c1Wr5oC5QvMlj6oLpOmRsNt45xLvny/79wOTJ/PXo0XyhM5bxjuQrjui9XjEKBfek0Wj4oVTyMm+5QkGzvghSYyWwadMmLFq0CJ07d0abNm1wzz33YOjQoZGUrU5gsfDvdLCmR68pqF+/wAe3cZU8xusSKklxovGqwWrlHb/dDvTqBUycGF15JKmscy//2rvZCfDvyL2LV96O3nsu2Kk0ETI1VgJXXnklrrzySrz22mtYuXIl3n//fUycOBGSJGH9+vVo3LgxEuKiNyrD4+Ez1mBnAaLIw70DfINYoDidcZQ8pqSE/7EvFB+ovsMYnwEcPMgXgebPj+y+heo6eK9d3uvb7t3YpFDwTl6lKisrf87rN0/EFAJjwUeY2rdvHxYtWoSPPvoIxcXFuPbaa/GVt1erQ5jNZhiNRphMJiQG0MlYrTwoZVJScN/tTZuAO+7g1+/aFfgAt7iYm0mzsgK/d53C4+G76dzu+DYFLVkCTJ/OO9RPPwW6dg3/PdxurnArdvByOR+9exdfK3by3t2tRMxQ034tpKfWqlUrzJ07FydPnsTy5ctDaapOUj5SbzB4F4QHDAhcAXhn23Ex+fLGB4qrmBgV8IaHBrgiiIQCYIwrgAYNgGbNyhZfc3L4+4YN+bnkZD4j0+vLbPikAOosIc0E6gvBzgQsFr6oG4xrqMsFdOzIR/MrV3LzbiDY7fz/Zs1iN5JBWHA4eCwfhSKOYmJUg8fDRw6DBkXGrGKx8A69USPyr68H1MpMgAien3/mCiAjAwgm9JLXK6heKwDGuLug2x2/CqD8GE2hAAYPjowCcLu5kklLIwUQZ5ASiBJeU9BNNwXekXvX5eq9dSSS8YHqCq+/Djz5ZHCJb2qK1wyUmhrfay5xCu2uiAI2G7BuHX8dzAaxuMgj7A3fLJPF7yagX34B/vMf3kn36gVcf31k7mO18i9TSgp578QhNBOIAuvXc5t+s2ZAhw6BX+908t3y9XotrqSEH/E6Ms3LA8aO5QrgrrsipwA8Hn40aEBmoDilPncjMYt3g9jAgYEPvESxbM9NvcXl4qEQtNp6rumqwe0GHn6Yz4TatgWefTZy9zKbuWdDPJvc4pw4/IVFl+Ji4Kef+OtgTEF2exwkjykqKrN5xSPPP89zBScm8gQxkfo7eM1AqalkBopj6pQSeOGFFyAIAsaPH+8rczgcGDNmDFJTU2EwGDB48GCcOXMmekJehG+/5QO9Nm2CC/7ocnFTUL39zdpsXAnEqxno2295xw8Ar73GbYaRwOPhX6YGDeI7DAdRd5TA9u3b8c477+Dyyy/3K58wYQK+/vprfPLJJ9i4cSNOnz6NQcFkZqklvKagYMJEuN3cbFtvTUHeZDHxHB9IoeAzgIcf5gGlIoXZzBeCyQwU99QJtwuLxYJhw4Zh4cKFmDVrlq/cZDJh0aJFWLZsGXr37g0AWLx4Mdq0aYOtW7fiyiuvrLI9p9MJZzmXO7PZHNkPUMqZM8Dmzfx1MEqg3iePsVh45xTP8YGuu457DmRkRO4eNhu3J5IZiEAdmQmMGTMGN9xwA/r27etXvmPHDrjdbr/y1q1bo0mTJtiyZUu17c2ZMwdGo9F3NG7cOGKyl+frr7mzR6dOQJMmgV/vdtfj/tHjAc6eLQs+Fm8UFZW9juSOXVHk7mVkBiJKiXklsGLFCuzcuRNz5sypdC4/Px8qlQpJFdJxZWRkID8/v9o2p02bBpPJ5DtOnDgRbrGrxGsKCmZB2Oms58ljiovjN1nMihXAVVcBGzdG/l5eb6B6O5ogAiWmzUEnTpzAuHHjsH79emjC6A6jVquhrmWbyrFjPFKoTMZ3CQeKw8F/t/Vy8OZw8FwBOl38mSf27OEB4RwO4M8/gauvjty9bDZuSyQzUETxSB54JE9Y21TKlJDLIjNDjmklsGPHDhQUFKBTp06+MlEU8csvv+DNN9/EunXr4HK5UFxc7DcbOHPmDDIzM6MgcfV4w0T07Amkpwd2LWN8rbReDt68yWLiMUy0yQQ8+CBXAH378s1hkUIU+X0aNarHi0rRgzEGu8eOEmcJSpwlcEvusLbfQNcAafrIZJCLaSXQp08f/P33335l9913H1q3bo0pU6agcePGUCqV2LBhAwYPHgyA5zg4fvw4ugcTlS2ChJJHuF4nj7FauSko3rxUGONZwY4e5TlzX3stshvjyAwUEVyiCza3DcX2YtjcNgCARqlBgjJ832ez0wwJUtjaq0hMK4GEhAS0a9fOr0yv1yM1NdVXPmrUKEycOBEpKSlITEzEo48+iu7du1frGRQN/vc/fqhUQP/+gV/vcPAZfL0LoSOKfGewN/NUPPHOO8DatfxL8c47wcUjryl2O79Pamp87sAOMxKTYHfbYXFZYHaa4RSdUMlVSFAnRMxkE0nq/C/v1VdfhUwmw+DBg+F0OtGvXz+89dZb0RbLD++C8DXX8I1egSBJfNBYLyOGms3cLbTCwn69Z8cOvisYAJ55BmjfPnL3EkWuBBo1qufbzCOPS3TB6rLC5DDB6rYCAHRKHfSquv3jrHNK4Oeff/Z7r9FoMH/+fMyfPz86Al0ExspMQcHmEdZo6qEpyOXiG8PiMT7QpZfyvAAeD3DPPZG9V0kJV7JkBgoKiUmwuW0+W79LckEtV8OoMUIm1I/vbZ1TAnWNnTt5Yiydju8DChSHg+8bqneu8974QCkp0Zak9tFqgVdf5YvhkfTSsdu5mS0tLf4UbYg4PU5u63cUw+7hafy0Ci0M6vrnvEDfjAjjnQX06xf4aF4UeR9R71znbTbuERRP3kCbNwMjR/onh4lk6GZJ4kogLY3MQDVElERYXBacNp/G0eKjOFVyCh7Jg0R1IpI0SVAr6qdXFc0EIogo8l3CQPCmoHoXMdQbHwiop5seKpCXBzz3XNlo4P33eVygSGM28wWoerTewhiLiA8+A4PdbUexoxgOjwMyQQatUosEeXx4rJESiCCbNwMFBfx3GMweIIcDyMqqZzP5khLuHx/oCnldw+UCFi3iZh+rlU/p7r0XGDo08vd2OOq0Gcjb0XskD0RJhFt0w+FxwCk6IUoiRCaClc+9HCKs9J9GoalXtv6aQkoggngHfzfcEPig1+Phv+N6ZQpyu7lLqFodmUWOo0eBzz8H7r8/uguhv/7K8wIfPMjfd+4MzJ4NXHZZ5O8tSdzclpUV094EEpP8OnuP5IHD44DD4+AdfWlnDwCCIEAhU0AuyKFSqKCQKSAgvGspQhzvoCYlECGcTh4aHqihKcjl4iNGmw3QauHwaKBNUEGjqUePyGTidupI+MSLIu/8LRZg3Liyckmq/dHwu+9yBZCaykNC3H577cngjcIaQ2Ygh8cBt+iGR/LAJbrg8Dj8On8vcpkcCpkCCpkCGoWmTvrc10XqUQ8TW2zcyPu8jAyg2n1rjPGpuzeEstvNh/8WC1wmGdIbyyCcVPEFVLU6ciPo2sBu52sBen1kPGJWrgT27uVTLq9bpMsF9OkD/N//cVfMSy4J/30Bfh+Xq2yh+5lngJwc4PHHa9fs5XDw70daWkx8TxhjKLQX4pztnC+MglzgHb1cJodWqYVckMf1KDwWICUQIcrnEa70exRFPuI3m/noH+Crv6U7wtxuQCkHtHo7r2cy8ZGkSsXtQ3o9f61W1w2brzc+kChGJm6NxQLMnctfT5tWNgresAE4fJgf77/PtfG99/Jt2+FalN64kZt+uncvk6F588jmBa4KrxkoMzMmbIiiJOKs7SzO2c5Bp9QhQR0fi6x1EVICEcBqBdat46/9YgU5nfyk2Vy2eKfXV9ISdocAvY5BnagGUNppShIfbZrNvEOVy3lHZjBw269azd/H4qjKYuGKLFIuoW+8wXMR5OQAI0aUlffrByxbBnz4IfD998DWrfxIS+MLtCNHBp+85dQp4Omny2x+ViufgUQrBpL33pEMP1FDnB4nCqwFMDlNSFQnQiGjbiaWoacTAb7/nvfxzZoB7S+TAJuD/0hLSviKr0ZzwUTBHlFAYkKFgFEyGb/O6y8qimW7biWJKxS1mne0Gk2ZUog2oshljFR8oBMngIUL+esZM/w/s0zG3bKuvho4fZorhGXLeIq3N98Ebr01cCXgdAJvvw3Mm1dmfrnvPm76iZYCcDr5d6lBg6ibgawuK/It+XB4HEjSJMWdp01dhJRABPDlEb7eCeF0AZ+mCwIfsV9kNOx0AWoVg057ERc4uZy35/UA8SYOLyjg75XKMqWQlBS9zsFk4jOBSI1Qn3+ed4I9elx4S3Z2NjBpEl80Xr8e2L4daN3avx2DAbjzTt6ZVsV//ws88AD3QgK4eWnWLKBNm7B9nIBhjM9ComwGYozB7DTjjOUMJEhI1kZ/RkLUDFICYaYo34GNP6sBCLil6ynAxXjnUsNO2OEUkJTIAt9MqlD4+5R6Fyvz8vj/GRm1v37gdJbFB4qEmUoUuYJTKoGnnqrZPZRKYMAAfngpLOSzCZcLeOUV4Prr+dpB9+7+bWZnl632z5zJ3b6ibX4rKeHfryiagSQm4bztPM7azkItV8OgjKOd4PUAmquFA0nio7G8PHy73Ay3R0Cbli5c0l5bpc2/OhgDREmAQR+GjTAqFb93YiLviAsKuJy1SVFR2bbnSCCXA3PmANu2ARVCjgeETscXdTt14qvyX3/N3Tr/7//4oq93Y1JyMvDBB8Avv/DFnmgrALebP9MoegO5RTfyS/JxxnoGOqUOWmXs7k0gqoZmAqEgioDZApSU+r/LZPjiJ560/pbrnQF3Ek4XoFExaDXh2w0JhYIrgnPnyuzGtTEjsFq5EqgNO3mwi7teNBre6d9+O0/3+NFHfNPZwYP86NGjbObQuXPo8oaLkhK+FyFKccYdHgfyS/JhcVtgVBvJr7+OQjOBULDb+Qjb4wESEpDvMGLLTu7Nc3M/R8DNOV3cKyjs66cKBe+Mz57lyiCMW+6rpHx8oEgESbPbuW1/377wt92uHfDiizz86+zZwKBBkQ30Fix2O1deKSlRmZGUOEtw0nQSNo8NyZpkUgB1GJoJhINSO/zX6zVgTEDny11onC0G3IwoCdDrImSyUSq5Iigo4J1GWlrkOg+zmR+R2rX6zjvAp5/yxd1ff42MKSQhgbublnc5jRW8EUIbNqz1fMGMMRTZi1BgLYBMJkOSJqlW70+EH1ICYeTLtdx985YgZgEuN6BSMGjUERylK5V8EfHMGa4AUlPDrwjcbj4L0GgiY3bKzwe8CYSeeCLqLpFRwWLhSqqW4yOJkohztnM4azsLnVIHjaI+hbeNX0gJhImjJ+TY9V8VZDKGm64NwhTkFKDTssi79ns3mHkVQbjNCUVFkYsPBPAFXJuNL+IGE5+7ruN2c3NeLS8Gu0QXzljOwOQwIUGdAKU8Bk1kRFCQEggTX6zjo6JeXV1okBq4ScctCkgw1JL3jkrFO5IzZ/j7cCkCu50rgUjFB9qzB1i1ir9++unoe+dEg5ISvrhfi4vBNrcNZyxnYHVbkaSlDWD1DXqaYYAx4MtSJXDL9YHPAjweQCGPsCmoImo1N9nk5/OOO1S88YE8nsjYqRnjHT9j3D0zlrx0agubjT+zWtwTYHKYcNJ0Ek7RiWRNMimAGvDVvq8waOUgvLr1Vb/y4V8Mx6CVg3DCdMJX9v2h7zFo5SC8sOkFv7oPfP0ABq0chIOFByMuL80EwsDegwrsP6yESslw/TXBeQVp1Ky21/jKQlDk53P7fSgLuSUlQHFx5OzUGzYAW7Zwmf/978jcI5aRJB6molGjWgkHIjEJhbZCFFgLoFKoaANYFTDGsPHYRnx/6HtM7TUViWr+3c+35GPbqW1omNDQr/6O0ztQ5Cjy5SwGgAJrAbad2oZkjb9i352/G6dKTsHqsiJNlxbRz0FKIAx4F4R793TCmBD4aN7pAlKSWHSsGxoNH13n5XHzSjChjz0evhisVEbOTv2vf/HYQB4P94qJN0pKuIKthcVgj+TBWetZnLefh16pr7e5dYPBI3l8AfEEQcDMn2biUNEhdGvUDTe34mtUfZv3RcOEhshOyPa79pV+r8AtupFlyPKV/avJv/Duje8iw+C/1+WFvi/A7rajaVLTCH8iUgIhwxjw5fdcCQSzN8Cb8ySqSaC8N8/L4/8HqghMJr45LJJmCrUaeOihyLUfy7hc/P/U1JA9riQm+R2iJPq990geWF1W2gBWgROmE5j6w1QcNx/HLyN+8eVAGNpuKI4WH0XzpOa+us2Tm6N5cvNKbVzXonJsq6ZJTavs6Hvn9Pa9LnYUh+ETVA8pgRD5c58GJ04roNdJuPZfwZiCALWKB42LKlqt/4ygpiNOp5OvBUQqPpDVys0fsbhhq7awWC64GFy+IxdZhU5d9E/hWPG8L4VjabpGbyrHZE1y3CZ7YYzhQOEBuEQX2qXzcCRpujRsPbUVDo8D+8/vR6u0VgCAR7o+Ek1RwwIpgRD59lceFqHf1c6gRvNOl4DUZBYb7u46nS8GEgTh4iEfvIvBLlfkZgFz5vANYS++eIEUbfWY0nSjSEnxK2aMwe6xo8RZAovL4lMEIhPBwPxy8MplcsgEGWSCDAIEKOVK33ta6K3MB39+gOk/TkfvZr3x0aCPAABapRbzrp+HVmmt0DKlZZQlDC+kBELA4wHW/cYXzIIxBTEGSEyATlvLgd0uhF7vrwguFPraGx8oUsliDhzgCWFEkR/xhncxuHFj30xIYhKsLiuKHcWwuCyQmASNQgO1Ql3W0dfDEbzJYcL+8/vR2NgYmYZMAECRvQiLdy+GUq7Eo1c86qv7/aHvsadgD3o16YUrGl4BgOc5eGfHOxAgYEL3Cb66Px75EWv2r8Hdl9+NTlmdAAA9G/eESq6CWqEGY8z397zhkhtq6+PWKqQEQuDXzXKcL1YgySjhqiudAV/vcgNqZS27htYEvZ6bIPLyePjkqswQ3mQxMlnkTDXPPsvv068f0LNnZO4Ry5jNfH0mIQFu0Q2r24oiexFsbhvkMjn0Kn29y9rl9Djxz9l/kG/JR//c/r7ysd+NxY9HfsScPnNwb/t7AQAmpwkvb3kZeqXeTwmsO7gOK/67Aiq5qkwJuK14ecvLkAmySkpg5X9Xwqgx+pRAy5SW2PPwHuhV0QnMV9vUr29QLfPJF/zPd0MfB1RB9INOp4AEQxC5A2oDg4F7pHgVQcWEJd5MaZGKD7RxI/Djjzz43ZNPRuYesYzLBchkcBj1KLGfh8lhgsPjgFqhhlFjrBdmnENFh7ArbxdyU3LRPrM9AOB0yWncuPxGqOVqHHj0gG9hunVqa+w9uxcSK5s1J6gScG/7e6GS+7vM9mjcAyqFymfPBwCNQoN729/rZyYDgK4Nu0Kj0KBfy36+MkEQ4kYBAIDAWKRDSsY+ZrMZRqMRJpMJiTVcEHU6gYx0BpNZwCfvFKJHF1fA9y00ydA4S4QxMYYfgdnMR/rZ2WVeRC4XcPw4fx2JbFYeD88Stm8fcP/9wDPPhP8eMYwkibAX5KHEqEFxogoe5oFWoa2zsfqdHie+O/gdDhUewsTuE33mlZk/zcSiXYvwYOcHMfPqmQD4Inevxb3QxNgEC25YgBQtXwuRmFQvFF8wFDuKkaZLQ7o+PaDratqv0UwgSL77DjCZBWSkutGtY+AKwO0GlLW9SzgYEhO5C+jp02WKoKiI26orLFaGjWXLuAJISgImTLho9fqCR/LA5rLBfD4Pdrjg1mZCr9TW+Tg9giBg/NrxcEtuDG03FA0T+T6PDpkdcGXDK9HE2MRXVy6TY8uoLZXaiFcFUBuQEgiSrCzglhs9SNeWBOXZ43QJ0GhqIWBcODAa+W7g06e5r3okF4MBvjMY4MnbI2VuiiGcHiesbivMDjMcLivUTie0TXKgT6ibeXo9kge78neha3ZXAIBKrsJNrW6CVqEFQ9mgZ1CbQRjUZlC0xCRKIXMQgjMHAYAl34Jj2/KQ3Djw7FmFxQIaZkpITqpDf/7iYm4acrsj2zkzBvz0E98lHJMLJqHDGIPD40CJqwQlzhK4JTc0cg00VieQmAApK7P2c0KHAYvLgntW34OdeTvx1dCvfLZ+InjIHFQPEUX++455U1BFjEYeKTTSoQsEAejd++L1apny4yXviDaYMrfohslpgs1tA2MMGoWGL0S6XBDkcogpyXVSAQCAXqlHmjYNGoUG+ZZ8tAcpgViHlEAUcLn4DuFaDxgXKoIQmYVgLytX8gXhIDeeOTwOnxdNJKhq0lxlh4/KZeWK4GE8/oxOqSsLy8AYZDY7pAZpUY4hEjg2tw0KmQIquQqCIGDutXNhdpprJe4NETqkBKKA0y2gQSqrq4O9yLBtGzBxIl9s3rw5oAT1HskDs8OMIkcRPJKnkstgOKjoWuj1cBEEwS/kQsX6VW3cqmpDl2C1QdKoISUFEcAvivx95m888u0juL7F9Zh+1XQAQLI2GcnaurmeEY+QEqhlGOOHVlPHTEGRRJJ4rgAA6N+/xgpAYhIsTgsK7YVwiA5oFdq66d8tioDHA5aRzvdF1CFOmk/icNFhfLHvC4y7chwMKgo5XdeoW9+4eoA3YFydWw+IJJ99Bvz1F/c4mjz5otUZY7C77ShyFKHEVQKVTAWj2lhnwyUIFiuYMREsoW50oOVDKfTP7Y+5feeif25/UgB1FDJI1DJOlwC9jtW1AV/ksNmAF0qzKj32GI+WeQGcHicKrAU4VXIKNrcNiepE6FS6OqsA4HQCMhmk5KQ6kS7zh8M/YNCqQbC6rL6yYZcP823qIuoepARqGVHiSoAoZcECntmscWNg1Khqq3kkDwrthThpPokiRxG0Ci0S1Al1exORdzE4NaVOLAbb3XZM/WEqfj/1Oxb8sSDa4hBhgsajtYjLDagUdWCXcG2Rlwe89RZ/PX16WbrLcvjZ/T0OaJV11O5fBYLVBkmvAzNGPltYONAqtZjXfx7WHlyLsVeMjbY4RJggJVCLOJ0CdNo6sku4NlCrgcGDgcOHgRtvrHTa5rL52/01ddfuXwmPBxBFsJSMmF0MZozho78+QovkFujZhEdx7dG4B3o07hFlyYhwEpvfvnqKWxSQYIih3AHRJiUFmDuX70Au17k7PU6YnCaYHCYAPFpkfUtz6FsMNsTurObDvz7Evzf8G5n6TGwYvgFJmqRoi0REAFICtYQoAnIZmYIAcB/Z8iP60tAQHskDs9OMYnsxXJILBqUBCnk9/Io6nYBCASklOaYXg2+/9HZ8/NfHuP3S25GorhsmKyJw6uEvLDZxugRo1HVwl3AkWLUKWLMGeOopoGVLX7Ysb8IUjUJTf0edjEFmd0BMb1DlGkhtITEJTo/TLzz1kt1LIEoiRnXiC/Q6pQ7fDfuu3iWuIfyhp1tLOF1ARgMWywO/2qGkhOcNPnsWrHt32Jtko8hRBIvLAoVMUb/s/hXxeCBYrJB0OrAI7AxmjMEtuf12TC/9aylOmE9gZMeRvgBkq/67CpPXT0afnD54/+b3fXUX7liIo6ajyDBk4MZL+BoNKYD6Dz3hWkAqXQbQRm/gFzvMmwecPQupWTOcueNGWEpOgYHBoDLUO7u/D8YgWG18IdiYyF1Cg4k/7tck81OWq/euxuT1k/Gvpv/C4psX+8rf+uMtHC0+it45vX1KIFGdCI/kQYG1wK/NwZcORomrhEw/QeJVwm7R7ZcBLVQ8kidsbVUFKYFawOkCNGraJcwOHQIWLoQAIG/iA7DCAb1SX387fwBwOvleAL0OLCWDLwSHONNZsnsJFu9ejHdufAet01oD4KYbu8eOs9azfnUHthoIs8OMZE1ZLJ9/NfkXto/ejjRdml/did0nhiRXvCExCS7RBbfohltyQ4AApUwJlVwV1kRABhigUURuBElKoBZwugSkJrNQB391FolJsLlskM/8N7RuN6w9rwD69kFifVz09VJq+oFCATEjnZt/wvQF+O3EbzhYeBBL/1qK53o/BwDo2aQnfhv5W6WY81N6Tql0vV6lrzd7LWoTj+SBW3TDJbogMhEyyKCSq6BX8r+nt/NXypR1yqRZj3+FsQFjAIMAnTb+XEO9nb/JaYL0049o/PMmMIUctulPQB4hBSBYLIDEwDQaQBWFhDSMQbBaAY8IlpzEo4IGuQAsSiI+3fspVu1ZhUU3L/Itlj/U+SH0atILg9sM9tU1qAwUuyeMMMb4KL/UvAPw9RGlXIkUbQo0Cg1UchVUclWdn8mSEogw8bhL2OvtU2wvhs1jg1wmR6OVXwMAbPcMhdgiJzI3drkBCWAJBghWGwSbDUypBNNqaidJi8MBmcPJF34zk0M2/cgEGRbuWIi95/bik38+wehOowEAnbM7o3N253BJTYArXG+n75E8ECBAJVdBo9AgRZMCtULt6/Tr0ii/JpASiDBOp4AEA6uvWRL9ECURVpcVJocJVo8VCpnCF9+n+M3/QLd0Fey33Ryx+ws2G1hqCqT0BoDTCcFmh2A2QzCXAABXBpHw0XW7uelHpeKmH2NiwKYfj+TBD4d/wHcHv8Mr170CuUwOQRDwaLdHccJ0Are2vjX8ctcRvGYYj+SBR/L4Je0JFwqBj/IT1YnQKXW+Dj8evKPq/yeMMh5RgEFfv01BoiTC4rLA5DDB5rFBKeM/Jr/gbioVbPfdHTkhHA5AqYTkjcOjVoOp1WDGRAh2BwSrFYK5BILdBKaQc3NRqOEaJImbfiQGlpLMI4EGqWTcohsT102EyWnCwEsGok/zPgCAm1tFTmnGGl7vGm+n7/WwUcgUUMgUMKgM0Cq1EbG5ezOj1emAhEFCSiCCeDyAoh6bgjySx7fJyyE6qozrr9ryO1xdOkY2YTxjkDmcfANWxU5YJgPT68D0OiA5qXR2UMJdNpkEplHzawLtVOx2bvoxGMBSknn7NWyDMYbNJzZj++ntGH/leAA8ONvozqNhd9t9Hj/1GYlJvtG9W+Idvte7RiFXIFmTDI1Sw+3wMiWUcmVcdtC1QUwrgTlz5uDzzz/H//73P2i1WvTo0QMvvvgiWrVq5avjcDjw+OOPY8WKFXA6nejXrx/eeustZGRkRFFyjneX8EUDxolimfnAYgEefpgnWDEYAL2+7LXBALRqBXTtyutKEnD0KC9PSOALkLVgr3SLbt/I3yE6oJarq0zqotizF8n3PQJPy+YoXPUBmC4y4ZIFu8M36i+PS3TBJbogF+R8Z6xSCWZUwqKRgTlU0LkAZQmfIbggwqEUIFOpoVOUyWl12+CS3DAodVDKlIDbDWvxWeRJZqhSGqBRwyzfs9udvxslrhK0a9DOl14xryQPW09uhVFjRO+c3gCA0yWnccend4CB4aZWN6FFcgsAwIQrJ0Tk7xNtREn0G+EDPP2m15PGqDFCLVf73itkinpnd49lYloJbNy4EWPGjEHXrl3h8Xjw73//G9dddx3++ecf6PXcxW3ChAn45ptv8Mknn8BoNGLs2LEYNGgQfvvttyhLz9cpU5Kq2CXMGPDf/wLr1gHffw9cdRUPpQwAJhPw44/VN3rnnWVKwGIB/vWvsnMyWZniSEgA+vUDpk4N2+cRJZEHdrOb4BSdUCuq7vy9nzFx9ksQGIOnVW7EFAAkCYLLhV3KQnz001t4vs/zPjvusr+XYfqP0zEgdwAW3rTQd8lVH1yNfEs+1t29Du2atIZgs+Pzv1dg3Lan0TvtCizr+QqfIchkuG7dMBwuOY6v+yxCV20LQAJ+cPyDhzZNQfdG3fHpkE997U76fhL2ntuL5YOX46qmVwEA9pzdg7HfjUWHjA4+JdAwsSEGthqIRHUi1PL6G0dEYhJKnCVgjEGtUEOr0CJFkwKVQuUb3ceDzT3WieknsHbtWr/3S5YsQXp6Onbs2IGrrroKJpMJixYtwrJly9C7N/+BLV68GG3atMHWrVtx5ZVXRkNsAHyQLgjlcoW4XMDWrWUd/+nTZZVdrjIlkJgIvPwy7+AtFsBq9X992WVl19ntvNO3WrlikSTAbOZHXh7QqVNYPgtjzGf2sbqt0Cg0Fw3voFmzFqqdf0LSalAy+bGwyFEVgs0Gm1qOYesexHn7eXTM7Ig7L7uz5g0oFGCJCWUJ3tUqMJnAF5MFgf9dAUiWErBkLVhqCtT56UjSJFVyycxJygFjDNpyM4k0bRp6NenlG+17eeuGt4L7wHUEu9sOu8eOBFUC0nRp0Cg0dd6Vsr4iMMbqjMH64MGDyM3Nxd9//4127drhxx9/RJ8+fVBUVISkpCRfvaZNm2L8+PGYMKHq6bXT6YTT6fS9N5vNaNy4MUwmExITa75l3pJvwbFteUhuXDkxut0OQABymkiQCYyP9g8fLqug1QJXXw1cdx3Qty+Qmlrj+1ZCkniaxvLKoqQESEsDWpfal/PyeOrGABdDnR4nih3FKHYUQy6TQ6/UX3SqLtjsSLt+EORnClAy/hFYH64+Y1iwFLvMSJLrIZRYIDVqiIUHVuDP/D8xqcckNE1qCgA+bxKZIPOLp+PwOADAbyHQa7KQCTKoBAVfO7BY4CouhFyugJDWADAm1o6raR3GI3lQ4iyBSq5CqjYVRo2ROv8oYTabYTQaL9qvxfRMoDySJGH8+PHo2bMn2rVrBwDIz8+HSqXyUwAAkJGRgfz8/GrbmjNnDp555pmIyCk/fQKajeug/30rnK+/W9pnCEC3brxjvvZa3vH36hW+lIJeM5Chms1Chw8Dt98O9OgBvPZajdwXRUmE2WlGkb0ILskVUEx//buLIT9TAE/DbFjvGxbAB7k4jDE8/+cbeG//Cnxx5eto37ATmEGP+zvdX6mu16ukIlVtwZfL5H6fjxn0YAY9FN5wz/Hg4xsCjDFY3VZ4JA+SNclI1aVCrai/pq76RJ1RAmPGjMGePXuwadOmkNuaNm0aJk4si5PinQkEBWNQ/rMbmo3fQ7NxHZQH/vGdsh3cBaR35G9mzuQJVKIxkjx0CDh3Dvj8c37/V165oCKwuWwotBfC4rZAIw8srLP8xCnoF30EACiZOiHs4ZIFQUC+/RwcohNfnv4Jl7frE9nFcEoDd1GcHiesbiv0Sj0yDZlIUCXQwm4dok4ogbFjx2LNmjX45Zdf0KhRI195ZmYmXC4XiouL/WYDZ86cQWZmZrXtqdVqqMOwaUj+zZe45ImJUBaWRWNkMhkc7bvB2rMfkpo1LKscgJkp7Fx7Lc/l+/DDwKefckXw8suVFJJLdMHkMKHYUQxBECr7+tcEUYS7w2VgMhmc114Tsugu0Y2lh1bj5qbXIUWdBAB44rKHcEtqT/S+pF/kFpyJiyIxCWanGTLIkKHPQLI2mRZ66yAx/cQYY3j00UexevVq/Pzzz8jJ8Q830LlzZyiVSmzYsAGDB/M4Kvv27cPx48fRvXv3yMuXmgZlYQEkrQ7OHtfAcdV1cPTqi2J5KpISGRQZMbRJ7IYbgDffBMaO5Uld5HLfzMQvmbvII3sGGwVRbNYEhR++w2P4hGE0+MBvU7Du1EactOZhRsdxAIDGqjQ0yeoJMTkppjNz1We8C79GtRGpulTolLpoi0QESUwrgTFjxmDZsmX48ssvkZCQ4LPzG41GaLVaGI1GjBo1ChMnTkRKSgoSExPx6KOPonv37rXiGSR1vRLHZiyEsn9fQF1m9hBNAvS6GFIAXgYO5N4uY8cCy5cDMhnszz2FQmcxSlwlUMvV4cnoJQhgCZUXy2uC10/Ba064p+Ug7Dq/By0Tm5U1b7WBNUiLamaueMUtulHiKoFGrkGjxEbBzRaJmCKmvYOqsysuXrwYI0aMAFC2WWz58uV+m8UuZA6qSE1X0StSlXeQ2w04nAKaNRZjN5Xk55+DjRsHsdUlOPb+KxD12pDj+mtXfAbFoSOwPPogWGJwCmDb2V2YtXse7ssdgkHN+gPgSsEhOqH1LuY6HBAkBrFxQ7LX1yKMMVhcFoiSiGQtX/gt73FFxB71wjuoJvpJo9Fg/vz5mD9/fi1IdHGcLgFabQ12CUcJxhgsN1wLG56HqX0raBKToVeEJqxQVIyEV96EzGSGp0UO7EMHX/yiKthSsBN/nPsLVrcNtza9HoIgQBCEMgVQPjxErP6B6yEOjwM2lw16lR5pCWkwqAy08FuPiGklUBdxuYEGqbGZS9jhcaDIXgSz0wzF//VAglLn+zEr/9gFd+cOQdnYDW+8A5nJDPclLQOKErrPdBhqmRLNErhn1uhL7oTJZcZDre+uspMRbHZImsrhIYjI4HUTVsgUyDRkIkmbRAu/9RAy5oURbwigWAsYJ0oiCu2FOGU+BbPTDIPKAL2qbNOX7qMVSB12PxJefNW3Q7amKPYdgG45D51QMn1SjTejLdq/Av/37e14etervjK9UoenOk5AhrZB5QskCYLHA5aSEnr0T+Ki2Nw8GZBRY0QTYxOk6dNIAdRT6KmGEadLgFrFYmotwOqyotBWCKvHCq1CW2VaQVZqWtEvXgomk8My+bGazQgYQ8Ls/0CQJDj69YHryq7VVi1xW+CRRCSreXiGqzK7QS7IIRNkcIouqC9iXxZsNp6sJYGyZ4ULxhhEJkJiEkSJ/y8xCW7JDa1Ci8aJjX35IIj6CymBMOJ0ARkNWMxEFjA7zDhjPQNBEKoP9AbAfscgQBRhfOYFGBZ9CMhlsEwce1FFoP7+R6i3/QGmVqPkifHV1vvw4Gd4btfruLflYJ+bZ25iDnbd/B0aaGsQLkMUAVECS06isA01oLrOXWSib52NgUEGGeQyrohlgownSVeooJKpkKhJpIXfOIGUQJhgrDRgXIx4LVqcFhTYCqCUKaFRXlwo+123Q2AMic++CMO7SwCZHJbxD1evCBiDYT6PzGkddQ/ERtnlTvFOyGs+yNQ2gMVjxfZzf4Ix5lNGNVIAAASLFSwxgadrJHwwxuDwOOAUnTXq3L1hmr3nvDMxuUwOuSCnxd44hZRAmHA4AbUqNtYDbC4bCqwFECDUSAH4rhs2BBBFJM7+DwxvL+IzgsceqrqyIKDo/fnQv/cBrKNH+Iq/PPY9Xt7zLh5sPQzDWvCUiH2ze+GTaxagZ0bXwDsatxsQZGBJSbQxrBS36IbdY4dH8kCr0CJNlwalTEmdOxEUpATChNMlICWJBZpaNuw4PA4UWAsgQaoU6rgm2O69E5AkJM55BewiphcpLRUlUyf6leXZzuCA+QhWHv7apwRkggy9Mq8IWBagdBaQmhL34SEkJsHhccDhcUApUyJBleDLh0tROolQICUQJiQW/V3CTo8TBZYCuCQXEtXBu1HaRgyDu8PlcHe4rMrziv/th6f1JdhasAvv7PsYD7a6G1eWBsob2uJmKGQK3NH8pqDv78Pp9M8bHIc4PU7YPXYAPPppliELepW+ykioBBEMpATCgMsNqBQMalX0TEFu0Y2z1rOwe+whKQBfe+UUgGC1Qb3hZzgGDoDyj11IHXY/HH2uxmf3JmPtyZ8hF+Q+JZCkSsT9rQJI6nIBZDY7xIz0uAsPIUoi7B47XB4XVHIVkjXJSFAnQKfUkacOEXZICYQBp1OALoq7hEVJxFnbWVjclgt6AQWFy43k0Y9CtWM3vv1zDf7vj3MAuCno/jZDIZcrcN8ld4Tvfl7sdjCNJugQFHUN7yKvw+OAAAE6pQ7pxnTolDry0iEiCimBMOAWBSQYomMKkpiEc7ZzMDvNSFQnhn8RUKWEs0c3qHbsxoCPt/F7JhhQMv4RtDIm44Wu08J7P6AsPERWZr1P5uKRPLC77XBLbmjkGjTQN4BBZYBWoaUFXaJWICUQIowJUMhZVLyCGGM4bzuPIkcRElSR29RjHfsAjptPoO0H3wIALGMfBEtJjsi9AG5+kvS6ejsLKD/qlwty6FV6GDVG6JQ62pVL1Dr0jQsDGnXt7xJmjKHIXoTz9vMwKA1h9RA5az+PF/6aj6uzrsTAJtcBAFKnPYvzLdpDnVcA27Dbw3avSpSGh5Ay0muUBrMiDo8DEgv/rKy8H35176s6J0Dw/V8ejUKDDH2Gb5GXRv1EtCAlECKCACToaz9gXLGjGOfs56BX6qGQh/cxLjv8JZYd/hIb87ehf6NroJQpAUGA+47b4A7rnSojWK2QDIaAN4Z5k5yo5Woub5iRyfksS1Yabksmq9l7b+fv7eQFCJDL5NAqtOTaScQEpARCRK2UwpYvvqaYHWacs53jHV6QGcDKwxiDXXRAp+Af5IFWd+HPwn/wcOt7ItKhVovHA0isxuEhvGYVu8cOjVyDTH0mhTsgiAAhJRAialXtrgeUDwehVoRugzpacgL/3jEXCkGOD69+DQCgVWjw/r/+E3LbgSJYbWDGRDD9hVMVMsZg99jhcDt8vvOJ6sSwKESCiDdICYSAVgvIksRai2lmd9uDCgdxITxMxK/52yATZDhcchzNE5qEpd2AcfHwEFKSsdrwEL7O3+OAVqFFdkI2EtQJ1PkTRAiQEggBuRzQaWtnFuDwOHDGcibocBBeREnEP8UHcFlKawBAy8RmePmKGejaoANySpO7RAPBZgNLTUFVtjXGGGxuG5yiE1qFFg0TGiJBnUCeNAQRBuhXVAdwia6whIM45yjEHT89gqOWk9h0w+fI0qUDAIaEI8RDKDgcVYaHkJgEm9sGl+iCXqn3+dBT508Q4YN+TTFOOMNBpKqToVfooBQU2Gc65FMCUaV83uBSP9uKnX+GPgMGVXjdYAmC4JASiGFEScQ52zmUuEqCCgdhdduw9NAXGHnJEChkCgiCgNevfAaJKgNS1ZHb7BUIgt0BpuZ5gyUmweqywiN5oFfqkWnIhF6pp86fICIIKYEYxRsOwuQ0BRUOwim60G/d3ThUcgwquQIjcocAQFTt/pWQJAguF1wZDWARbRDdIgwqA5K1yTCoDBQsjSBqAfqVxSDhCAehlqvwWNuRaKTPQkNdZgSkDB2P2QSTQoRZzaBX6tHE2ASNjY2RqE4kBUAQtQTNBGKMUMNBlE/fOCTnRtzc5LqLJnGvTURJhNPjhMtlh8rtgr5RLhJTsqFX6il0AkFEARpuxRjecBA6hS7gcBCb8n/HsI2PwVaahARATCgAxhgcbgdMDhOsLiuUMiWymAENG1+K7KxcGFQGUgAEESVICcQIoiSi0FaIs9azUMlUUCkC67xtHjvGbHkSP+Vtxpv/LImMkAHiET2wuCwwOU1gjCFVl4pGiY3QUJ0KY0Iq1OlZlDeYIKIMmYNiALvbjvO287C4LDyJSIAKAAB0Ci3e6/USlhz4BOPajoqAlDVDYhIcbgdckgsKmQJ6pZ7Hx1dquX8/Y4ClCMjMjLuMYQQRi5ASiCKiJMLkMKHQUQjGGIyawN1AS9wWJCj5DuKuDdqja4P2kRD1gjDG4BJdcHgcAACtQotkbTK0Sm3lXLg2G98VbDTWupwEQVSGzEFRwu62I9+S7wsGl6BOCEgBMMYw77/vo/d3Q3HKmh9BSaunvLlHkiSkaFPQOLExGhkbIVmbXFkBSBJPHp+WVu8zhhFEXYFmArWMxCQUO4pRaC+ExKSg3SFtHjtWHVmDk9Y8rD31M0ZdMjQC0lZGYhKcHiecohMKQQGtUotEdSI0Cs3FA7mVlACJiUBC/cwYRhB1EVICtYjdbUehvRAlrhJoFdqQQkHrlTqsuGY+Npz+DcNzbwujlFXjDeUgSiI0Cg3S9TwJulqurtkMxl2ajiY1tUa5AgiCqB1ICdQCEpO47d9eCI/kCXr075bc2G86grbJlwAAGumzIq4AvIlbnKITeoUeyfpk6FS6wOUvKeEKQHfhXAEEQdQuNCSLMA6PA/mWfJyxnoFckMOoMQalAOweB0b+OgkDfxiJHef+joCklfH69gsQkGXIQnZiNgzqIMI52O08OFxKCrmEEkSMQTOBCCExCSXOEpy3nYdH8iBBlRBSIDSZIINLcvNZhaskjJJWxuVxweaxQSVTId2QjkR1YvDhmxnjSiAryxcllCCI2IGUQARwepwotBfC5DRBI9cgURNaCGiA7/xd/K+Xsd90GB1S24ZBysp4RA+sbivkMjnSdGlIVIchX6/VCuj15BJKEDEKKYEwwhiD2WlGoa0QLskV8ug/z1aAzQV/YHCzAQD4hrCAFYAkXXQhVpRE2Nw2AECSJgmJ6kRolZUzfAWMKPIF4cxMQEFfNYKIReiXGSacHieK7EUodhZDLVfDqAlt5FvoLMYtP9yP49ZTUAgK3Nz0uuorMwZ4PIAoQvCI/H/G4Et8qVKB6Sp36uU9fgwqA5I1fINX2OL4eF1CDcGnwyQIIrKQEggRxhhKHGact52HU3KGPPr3kqwy4tqG/8KG05vQKbUdL5QkwCMCogeCKAIeEQLAO3u5HEypANNqwLRaPvJWKACPB7IzBRCsNjC9ziez3W2HS3L5PH70qjBH8XS5+AyEXEIJIqYRGGO1kyk9hjGbzTAajTCZTEhMrLn9XjSbcH7fLhSpGVRyVXhMKAA3o4gimNsNk6MYyTLeQTNBABQKMIWcj+7Var7z1lumUFTZ4QrmEsjyzwAyGewqGRweBzQKDVK0KdCrIpS5q7AQaNAAyMgIf9sEQVyUmvZrNBMIAbvHjmJ7MQwJDYPrSL1mHI8HgseD387swPdnt+DpNmMgqJSAUoUkQyNIarV/R69QBORqyRITYBddcJ08CqVbjfTU7NA8fi6G3c6DwyXHRgpLgiCqh5RAiAgQaqYAGAPcvLP32u8hCD4zjk2rxP1/PYsipwktmnXEsNy7w7KY6hbdsLgsUOhUSGneFsYiG1SSGoiUAvC6hGZnA6ro5zIgCOLCkBKIBJLER/fu0lE+Y2WmHKUSSDCAqVSwwQ2tLhFQKqEWBNzXcRQOFh3E4MuHhkUBWFwWeEQPUrQpZQHd9CVAXh5gsURmwdbbLrmEEkSdgJRAyDDA6Szr8FFqu1cqwFQqIMkISakElEowJTflMAD/2fwfvL/7fXw19CvkpuYCACZ0nxCW3LoSk2B2mKFWqJFlzPLP3OUN3hYJRVC6loHUVEAegXUGgiDCDimBUBAEMJUKgsTKvHK8nX3pgm2VlwH437n/wew0Y9V/V2H6VdMBICwKwC26UeIsgVFjRLo+veogdQkJ3BR1+nR4FYHZDCQlkUsoQdQhSAmEgk4HT8MsiAlpFxz5Oj1OrPjvCgxuMxgGFe8gp/aaisGXDkb/lv3DJo7dbYfD40ADfQOk6dIuvFZhMHC7fbhmBE4nV3oUH4gg6hSkBEJBEMDUqouaPkZ8OQK/HPsFRfYijL9yPAAgNzXXZwYKFe9OZbkgR8PEhjCqa5ihzGDgMX3y8vjGrlDi/FutQHo6zxpGEESdgXbxRACJSSi//WJo26HI0GcgKyEr7PcSJRFFjiJoFVo0MjZCkiYpsE1fXkUgk3FFEAzelJHkEkoQdQ5SAmFm49GN6L+0P9YfXu8ru6nVTfht5G+4o+0dYb2Xw+OAyWlCiiYFDRMbQqcMMla/1zQUjCKQJMDh4IvBlDKSIOocpATCzOYTm7GnYA/e2v6Wr0wmyMK3m7gUi8sCp8eJLEMWMhMyL57a8WLo9cEpAouFm5EC2GlNEETsQGsCIfJ3wd/IScpBY2NjAMBDXR4CBODBzg9G5H7eLGUahQbZxmzfQnNY8CqCvDzu6XOxjt3j4TMBig9EEHUW+uWGwH82/wd3fnYn5m6e6ytL1iZjWq9pSNGmhP1+LtGFYnsxEtWJaJTYKLwKwItez9cIFAquCC5ESQlfB9Drwy8HQRC1AimBEOjbvC/kghwKmQISkyJ6L5vbBpvbhnRDOrITskNKUn9RvDOCCykCh4OvASQnk0soQdRhyBwUAh0yO+D7u79H6watI3YPxhhMThOUMiUaJjREojoxvCGfq0On44rg9OnKpiHGuEdQZiYPFEcQRJ2l3swE5s+fj2bNmkGj0aBbt274/fffa+W+GYbIhUr2SB4UOYqgU+jQKLERjJoa+v+HC68iUCoBk6ms3OsSmpRUe7IQBBER6oUSWLlyJSZOnIinnnoKO3fuRPv27dGvXz8UFBREW7SgcXgcMDvNSNWmopGxUdi9i2qMTsfXCFQqrghEke8OTkujlJEEUQ+oF0rglVdewejRo3Hffffh0ksvxdtvvw2dTof3338/2qIFRYmzBC6Pi7t/GjIjF/e/pnhnBCoVcP48jxAayu5igiBihjo/lHO5XNixYwemTZvmK5PJZOjbty+2bNlS5TVOpxNOp9P33lRq6jBfzBumAhaXBZYSCxTu8PwZ3aIbVrcVarka6YZ0KD1KlAS7izcSGAw8V4BCwfcHEAQRs3j7s4slj6zzSuDcuXMQRREZFdIYZmRk4H//+1+V18yZMwfPPPNMpfLGjRtHREaCIIhoUVJSAuMF8nvUeSUQDNOmTcPEiRN97yVJQmFhIVJTUwNaeDWbzWjcuDFOnDgRUG5iaq9utBeJNqk9aq+22mSMoaSkBNnZ2ResV+eVQFpaGuRyOc6cOeNXfubMGWRmZlZ5jVqthlrt72efFIKnS2JiYtgeOLUXe+1Fok1qj9qrjTYvNAPwUucXhlUqFTp37owNGzb4yiRJwoYNG9C9e/coSkYQBBH71PmZAABMnDgRw4cPR5cuXXDFFVfgtddeg9VqxX333Rdt0QiCIGKaeqEE7rjjDpw9exYzZ85Efn4+OnTogLVr11ZaLA43arUaTz31VCXTErVXP9qLRJvUHrUXC22WR2AX8x8iCIIg6i11fk2AIAiCCB5SAgRBEHEMKQGCIIg4hpQAQRBEHENKIEjCGbr6l19+wU033YTs7GwIgoAvvvgiJNnmzJmDrl27IiEhAenp6bjllluwb9++oNtbsGABLr/8ct9mle7du+O7774LScbyvPDCCxAEAePHjw/q+qeffhqCIPgdrVuHluPh1KlTuPvuu5GamgqtVovLLrsMf/zxR1BtNWvWrJJ8giBgzJgxQbUniiJmzJiBnJwcaLVatGjRAs8999xFY8RciJKSEowfPx5NmzaFVqtFjx49sH379hpff7HvMGMMM2fORFZWFrRaLfr27YsDBw4E3d7nn3+O6667zrfLf/fu3UHL53a7MWXKFFx22WXQ6/XIzs7Gvffei9OnTwct39NPP43WrVtDr9cjOTkZffv2xbZt24JurzwPPfQQBEHAa6+9dsHPXFNICQRBuENXW61WtG/fHvPnzw+LfBs3bsSYMWOwdetWrF+/Hm63G9dddx2sVmtQ7TVq1AgvvPACduzYgT/++AO9e/fGzTffjP/+978hy7p9+3a88847uPzyy0Nqp23btsjLy/MdmzZtCrqtoqIi9OzZE0qlEt999x3++ecfvPzyy0hOTg6qve3bt/vJtn79egDA7bffHlR7L774IhYsWIA333wTe/fuxYsvvoi5c+fijTfeCKo9ALj//vuxfv16fPTRR/j7779x3XXXoW/fvjh16lSNrr/Yd3ju3LmYN28e3n77bWzbtg16vR79+vWDw+EIqj2r1YpevXrhxRdfDFk+m82GnTt3YsaMGdi5cyc+//xz7Nu3DwMHDgyqPQC45JJL8Oabb+Lvv//Gpk2b0KxZM1x33XU4e/ZsUO15Wb16NbZu3XrRUBABwYiAueKKK9iYMWN870VRZNnZ2WzOnDkhtw2ArV69OuR2ylNQUMAAsI0bN4atzeTkZPbee++F1EZJSQnLzc1l69evZ1dffTUbN25cUO089dRTrH379iHJUp4pU6awXr16ha29iowbN461aNGCSZIU1PU33HADGzlypF/ZoEGD2LBhw4Jqz2azMblcztasWeNX3qlTJzZ9+vSA26v4HZYkiWVmZrKXXnrJV1ZcXMzUajVbvnx5wO2V58iRIwwA27VrV9DyVcXvv//OALBjx46FpT2TycQAsB9++CHo9k6ePMkaNmzI9uzZw5o2bcpeffXVi7ZVE2gmECDe0NV9+/b1lV0sdHW08YbKTklJCbktURSxYsUKWK3WkMNyjBkzBjfccIPf3zJYDhw4gOzsbDRv3hzDhg3D8ePHg27rq6++QpcuXXD77bcjPT0dHTt2xMKFC0OWEeDfn48//hgjR44MOktcjx49sGHDBuzfvx8A8Oeff2LTpk3o379/UO15PB6IoghNhVShWq02pBmVlyNHjiA/P9/vORuNRnTr1i2mfzOCIIQUU8yLy+XCu+++C6PRiPbt2wfVhiRJuOeeezB58mS0bds2ZJnKUy92DNcmwYSujiaSJGH8+PHo2bMn2rVrF3Q7f//9N7p37w6HwwGDwYDVq1fj0ksvDbq9FStWYOfOnQHZnaujW7duWLJkCVq1aoW8vDw888wz+Ne//oU9e/YgIYjkN4cPH8aCBQswceJE/Pvf/8b27dvx2GOPQaVSYfjw4SHJ+sUXX6C4uBgjRowIuo2pU6fCbDajdevWkMvlEEURs2fPxrBhw4JqLyEhAd27d8dzzz2HNm3aICMjA8uXL8eWLVvQsmXLoOX0kp+fDwBV/ma852IJh8OBKVOm4M477wwpCNyaNWswdOhQ2Gw2ZGVlYf369UhLSwuqrRdffBEKhQKPPfZY0PJUBymBes6YMWOwZ8+ekEd0rVq1wu7du2EymfDpp59i+PDh2LhxY1CK4MSJExg3bhzWr19fafQZDOVHwJdffjm6deuGpk2bYtWqVRg1alTA7UmShC5duuD5558HAHTs2BF79uzB22+/HbISWLRoEfr37x+STXfVqlVYunQpli1bhrZt22L37t0YP348srOzg5bvo48+wsiRI9GwYUPI5XJ06tQJd955J3bs2BG0nHURt9uNIUOGgDGGBQsWhNTWNddcg927d+PcuXNYuHAhhgwZgm3btiE9PT2gdnbs2IHXX38dO3fujEiOcTIHBUgwoaujxdixY7FmzRr89NNPaNSoUUhtqVQqtGzZEp07d8acOXPQvn17vP7660G1tWPHDhQUFKBTp05QKBRQKBTYuHEj5s2bB4VCAVEUQ5I1KSkJl1xyCQ4ePBjU9VlZWZWUW5s2bUIyMQHAsWPH8MMPP+D+++8PqZ3Jkydj6tSpGDp0KC677DLcc889mDBhAubMmRN0my1atMDGjRthsVhw4sQJ/P7773C73WjevHlIsgLw/S5i/TfjVQDHjh3D+vXrQw4Frdfr0bJlS1x55ZVYtGgRFAoFFi1aFHA7v/76KwoKCtCkSRPf7+XYsWN4/PHH0axZs5BkBEgJBExdCF3NGMPYsWOxevVq/Pjjj8jJyQn7PSRJ8kvRGQh9+vTB33//jd27d/uOLl26YNiwYdi9ezfkcnlIslksFhw6dAhZWVlBXd+zZ89KLrX79+9H06ZNQ5Jr8eLFSE9Pxw033BBSOzabDTKZ/09XLpdDkqSQ2gV4x5WVlYWioiKsW7cON998c8ht5uTkIDMz0+83YzabsW3btpj5zXgVwIEDB/DDDz8gNTU17PcI9jdzzz334K+//vL7vWRnZ2Py5MlYt25dyHKROSgIwh262mKx+I1ajxw5gt27dyMlJQVNmjQJuL0xY8Zg2bJl+PLLL5GQkOCzuxqNRmi12oDbmzZtGvr3748mTZqgpKQEy5Ytw88//xz0FzAhIaHS+oRer0dqampQ6xaTJk3CTTfdhKZNm+L06dN46qmnIJfLceeddwYl34QJE9CjRw88//zzGDJkCH7//Xe8++67ePfdd4NqD+AdwOLFizF8+HAoFKH97G666SbMnj0bTZo0Qdu2bbFr1y688sorGDlyZNBtrlu3DowxtGrVCgcPHsTkyZPRunXrGn+nL/YdHj9+PGbNmoXc3Fzk5ORgxowZyM7Oxi233BJUe4WFhTh+/LjPl9+rtDMzM6ucXVyovaysLNx2223YuXMn1qxZA1EUfb+ZlJQUqFSqgNpLTU3F7NmzMXDgQGRlZeHcuXOYP38+Tp06Va1b8MU+b0WlpFQqkZmZiVatWlXZXkCExccoDnnjjTdYkyZNmEqlYldccQXbunVr0G399NNPDEClY/jw4UG1V1VbANjixYuDam/kyJGsadOmTKVSsQYNGrA+ffqw77//Pqi2qiMUF9E77riDZWVlMZVKxRo2bMjuuOMOdvDgwZDk+frrr1m7du2YWq1mrVu3Zu+++25I7a1bt44BYPv27QupHcYYM5vNbNy4caxJkyZMo9Gw5s2bs+nTpzOn0xl0mytXrmTNmzdnKpWKZWZmsjFjxrDi4uIaX3+x77AkSWzGjBksIyODqdVq1qdPnwv+LS7W3uLFi6s8/9RTTwXcntfNtKrjp59+Crg9u93Obr31Vpadnc1UKhXLyspiAwcOZL///nvQn7ci4XQRpVDSBEEQcQytCRAEQcQxpAQIgiDiGFICBEEQcQwpAYIgiDiGlABBEEQcQ0qAIAgijiElQBAEEceQEiAIgohjSAkQRDmaNWt2wbR9R48erVE6w0gSTRnCkf6UiC1ICRAxRVW5eMsfTz/9dFTla9y4MfLy8kLKzVAfZCDqDxRAjogp8vLyfK9XrlyJmTNn+kX0NBgM0RDLh1wuj3r441iQgag/0EyAiCm8USAzMzNhNBohCIJf2YoVK9CmTRtoNBq0bt0ab731lt/1U6ZMwSWXXAKdTofmzZtjxowZcLvdfnW+/vprdO3aFRqNBmlpabj11lv9zttsNowcORIJCQlo0qSJX/TQiqaYn3/+GYIgYMOGDejSpQt0Oh169OhRKRT1rFmzkJ6ejoSEBNx///2YOnUqOnToUO3foaioCMOGDUODBg2g1WqRm5uLxYsXVykDwFNi5ubmQqPR4JprrsEHH3wAQRBQXFwMAFiyZAmSkpKwbt06tGnTBgaDAddff72f0t2+fTuuvfZapKWlwWg04uqrr8bOnTurldHlcmHs2LHIysqCRqNB06ZNQ8ppQEQHUgJEnWHp0qWYOXMmZs+ejb179+L555/HjBkz8MEHH/jqJCQkYMmSJfjnn3/w+uuvY+HChXj11Vd957/55hvceuutGDBgAHbt2oUNGzbgiiuu8LvPyy+/jC5dumDXrl145JFH8PDDD1fq1Csyffp0vPzyy/jjjz+gUCj8wjovXboUs2fPxosvvogdO3agSZMmF81aNWPGDPzzzz/47rvvsHfvXixYsKDa1IRHjhzBbbfdhltuuQV//vknHnzwQUyfPr1SPZvNhv/85z/46KOP8Msvv+D48eOYNGmS73xJSQmGDx+OTZs2YevWrcjNzcWAAQNQUlJS5X3nzZuHr776CqtWrcK+ffuwdOnSsCQ5IWqZsMQiJYgIsHjxYmY0Gn3vW7RowZYtW+ZX57nnnmPdu3evto2XXnqJde7c2fe+e/fubNiwYdXWb9q0Kbv77rt97yVJYunp6WzBggWMMeYLO7xr1y7GWFkI4B9++MF3zTfffMMAMLvdzhhjrFu3bmzMmDF+9+nZsydr3759tXLcdNNN7L777qvyXEUZpkyZwtq1a+dXZ/r06QwAKyoqYoyVhV4uH2J7/vz5LCMjo1oZRFFkCQkJ7Ouvv/aVAWCrV69mjDH26KOPst69ezNJkqptg4h9aCZA1AmsVisOHTqEUaNGwWAw+I5Zs2bh0KFDvnorV65Ez549kZmZCYPBgCeffNIvLeTu3bvRp0+fC97r8ssv9732mqMKCgpqfI03o5n3mn379lWabVR8X5GHH34YK1asQIcOHfDEE09g8+bN1dbdt28funbtetH2dTodWrRo4Sdn+c915swZjB49Grm5uTAajUhMTITFYqk2reaIESOwe/dutGrVCo899hi+//77C34mIjYhJUDUCSwWCwBg4cKFfmn29uzZg61btwIAtmzZgmHDhmHAgAFYs2YNdu3ahenTp8PlcvnaqUlmNaVS6fdeEISLpm4sf403GXgo6R779++PY8eOYcKECTh9+jT69OnjZ7oJhqo+FyuXTmT48OHYvXs3Xn/9dWzevBm7d+9Gamqq39+vPJ06dcKRI0fw3HPPwW63Y8iQIbjttttCkpGofUgJEHWCjIwMZGdn4/Dhw2jZsqXf4c2hvHnzZjRt2hTTp09Hly5dkJubi2PHjvm1c/nll/vluq0NWrVqhe3bt/uVVXxfFQ0aNMDw4cPx8ccf47XXXqs2vWWrVq3wxx9/BNx+RX777Tc89thjGDBgANq2bQu1Wo1z585d8JrExETccccdWLhwIVauXInPPvsMhYWFAd+biB7kIkrUGZ555hk89thjMBqNuP766+F0OvHHH3+gqKgIEydORG5uLo4fP44VK1aga9eu+Oabb7B69Wq/Np566in06dMHLVq0wNChQ+HxePDtt99iypQpEZP70UcfxejRo9GlSxf06NEDK1euxF9//YXmzZtXe83MmTPRuXNntG3bFk6nE2vWrEGbNm2qrPvggw/ilVdewZQpUzBq1Cjs3r0bS5YsAVA2K6kJubm5+Oijj9ClSxeYzWZMnjz5gjOnV155BVlZWejYsSNkMhk++eQTZGZmIikpqcb3JKIPzQSIOsP999+P9957D4sXL8Zll12Gq6++GkuWLPHNBAYOHIgJEyZg7Nix6NChAzZv3owZM2b4tfF///d/+OSTT/DVV1+hQ4cO6N27N37//feIyj1s2DBMmzYNkyZN8plQRowYAY1GU+01KpUK06ZNw+WXX46rrroKcrkcK1asqLJuTk4OPv30U3z++ee4/PLLsWDBAp93kFqtrrGcixYtQlFRETp16oR77rkHjz32GNLT06utn5CQgLlz56JLly7o2rUrjh49im+//RYyGXUrdQnKMUwQUeDaa69FZmYmPvroo4i0P3v2bLz99ts4ceJERNon6g9kDiKICGOz2fD222+jX79+kMvlWL58OX744QesX78+bPd466230LVrV6SmpuK3337DSy+9hLFjx4atfaL+QkqAICKMIAj49ttvMXv2bDgcDrRq1QqfffYZ+vbtG7Z7HDhwALNmzUJhYSGaNGmCxx9/HNOmTQtb+0T9hcxBBEEQcQyt4BAEQcQxpAQIgiDiGFICBEEQcQwpAYIgiDiGlABBEEQcQ0qAIAgijiElQBAEEceQEiAIgohj/h9sNVP6lYtocAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Plot\n",
        "plt.figure(figsize=(4, 5))\n",
        "plt.fill_between(range(NSTEPS), rl_mean + rl_std, rl_mean - rl_std, color='g', alpha=0.1)\n",
        "plt.fill_between(range(NSTEPS), demo_mean + demo_std, demo_mean - demo_std, color='r', alpha=0.1)\n",
        "plt.fill_between(range(NSTEPS), expl_mean + expl_std, expl_mean - expl_std, color='b', alpha=0.1)\n",
        "plt.plot(range(NSTEPS), rl_mean, 'g:', label='LfR')\n",
        "plt.plot(range(NSTEPS), demo_mean, 'r--', label='LfD')\n",
        "plt.plot(range(NSTEPS), expl_mean, 'b-', label='LfCE')\n",
        "plt.plot(range(NSTEPS), ref_mean, 'k:', label='Optimal')\n",
        "#plt.xlabel('N. samples')\n",
        "#plt.ylabel(r'$||V_\\pi||_2$')\n",
        "plt.xlabel('Teaching signals')\n",
        "plt.ylabel('Average return')\n",
        "plt.ylim(0, 120)\n",
        "plt.xticks(range(0, 15))\n",
        "plt.legend(loc='best')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "wopt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cpeZBFJm7nx8",
        "outputId": "7272d9dd-911d-48c6-fee7-49c3206046d3"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-0.80559438, -1.25024501,  0.78905044])"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SW_VyigwJ76s",
        "outputId": "08d8bf2d-39a8-4e4e-f8fe-bdc12bfa7a91"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Optimal:\n",
            "[-0.80559438 -1.25024501  0.78905044]\n",
            "Contrastive Examples:\n",
            "[-0.00285149 -0.00392236  0.89322664]\n",
            "Demos:\n",
            "[-0.00241106 -0.25402383  0.09612059]\n",
            "RL:\n",
            "[-0.18596769 -0.22368096  0.26503212]\n"
          ]
        }
      ],
      "source": [
        "print('Optimal:')\n",
        "print(wopt)\n",
        "print('Contrastive Examples:')\n",
        "print(w_expl)\n",
        "print('Demos:')\n",
        "print(w_demo)\n",
        "print('RL:')\n",
        "print(w_rl)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "wopt = [-0.80559438, -1.25024501,  0.78905044]\n",
        "w_expl = [-0.00285149, -0.00392236,  0.89322664]\n",
        "w_demos = [ 1.09113395e-17, -4.06255996e-01,  9.26802108e-02]\n",
        "w_rl = [-0.18596769, -0.22368096,  0.26503212]"
      ],
      "metadata": {
        "id": "tsDEYkQgyjiw"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "wopt_mean = np.mean(wopt)"
      ],
      "metadata": {
        "id": "ngSlcZRs_5K4"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "wexpl_mean = np.mean(w_expl)"
      ],
      "metadata": {
        "id": "AaTL51BB_--V"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "wdemos_mean = np.mean(w_demos)"
      ],
      "metadata": {
        "id": "sR21S7VhADkP"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "wrl_mean = np.mean(w_rl)"
      ],
      "metadata": {
        "id": "BDJKdbsjAINP"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(wopt_mean, wexpl_mean, wdemos_mean, wrl_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WxTGyGjlAM1n",
        "outputId": "2c51eabb-94b9-4d4a-9c60-5d70e9f64652"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-0.42226298333333334 0.29548426333333333 -0.10452526173333333 -0.04820551000000001\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "expl_opt = wopt[0] - w_demos[0]\n",
        "expl_opt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nGHNlFU-AWw3",
        "outputId": "de46d843-6689-4648-e064-90d884c8a16f"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-0.80559438"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import scipy.stats\n",
        "from scipy.stats import f_oneway\n",
        "from scipy.stats import normaltest, ttest_rel, shapiro, mannwhitneyu, wilcoxon, ranksums, ttest_ind, kruskal, sem"
      ],
      "metadata": {
        "id": "G2_-AcfGzNvw"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ttest_rel(perf_rl, perf_demo)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZLgI-BpRidIr",
        "outputId": "52971870-b37f-4501-8238-357ec05dde38"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TtestResult(statistic=array([-1.27793327, -1.13314536, -0.78074495,  0.02751001, -0.54098063,\n",
              "       -1.2853725 , -0.95475253, -1.37754281, -2.28126511, -3.38298801,\n",
              "       -3.05575287, -3.03228008, -3.21306073, -2.97975579, -3.02189091]), pvalue=array([0.21140793, 0.26643481, 0.44127942, 0.9782414 , 0.59265256,\n",
              "       0.20883277, 0.34759425, 0.1788837 , 0.03006045, 0.0020702 ,\n",
              "       0.00478267, 0.00507279, 0.00320948, 0.00578374, 0.00520647]), df=array([29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29]))"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('normal test, reward: ', normaltest(rl_mean), 'demostration', normaltest(demo_mean), 'explanation', normaltest(expl_mean))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bdlFY-RAx34g",
        "outputId": "3fcf02f4-7f39-4718-c9be-003499d31631"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "normal test, reward:  NormaltestResult(statistic=1.6712937179221419, pvalue=0.4335939165894206) demostration NormaltestResult(statistic=6.079618656299149, pvalue=0.047844011130685135) explanation NormaltestResult(statistic=29.66295672849111, pvalue=3.620518641517437e-07)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/scipy/stats/_stats_py.py:1736: UserWarning: kurtosistest only valid for n>=20 ... continuing anyway, n=15\n",
            "  warnings.warn(\"kurtosistest only valid for n>=20 ... continuing \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mannwhitneyu(rl_mean,demo_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G-x-ciC3x8CR",
        "outputId": "4d2ae297-7ee1-4801-83ca-bc6a9f3ec366"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MannwhitneyuResult(statistic=44.0, pvalue=0.004776024960228471)"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mannwhitneyu(wopt,w_expl)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e0Xk-Rnay0NQ",
        "outputId": "5d8c8556-07ae-45a0-8948-c8c34d2dba73"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MannwhitneyuResult(statistic=2.0, pvalue=0.4)"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mannwhitneyu(w_rl,w_expl)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vou5LzBhy4jQ",
        "outputId": "4f92de3e-797a-4785-c749-5052ffe904a0"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MannwhitneyuResult(statistic=2.0, pvalue=0.4)"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mannwhitneyu(w_demo,w_expl)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i_PRDW6Ry7o7",
        "outputId": "eb00f349-08c7-475b-d634-6143b3c6737b"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MannwhitneyuResult(statistic=4.0, pvalue=1.0)"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mannwhitneyu(rl_mean,expl_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BPprhAUayBNa",
        "outputId": "a4798afb-4da4-4db3-cc26-b419a440f325"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MannwhitneyuResult(statistic=4.0, pvalue=7.3214532961732006e-06)"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mannwhitneyu(demo_mean,expl_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gpCTHAU0yDN2",
        "outputId": "ace13592-611b-45e6-b955-8f4f6fd2429b"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MannwhitneyuResult(statistic=16.0, pvalue=6.748052131560162e-05)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ttest_rel(rl_mean,demo_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_4VzqSMgyNnH",
        "outputId": "5f0d5473-fafd-4cb8-b0b9-fdc4ac3237da"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TtestResult(statistic=-8.347806126715604, pvalue=8.315740273608295e-07, df=14)"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ttest_rel(rl_mean,expl_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lOlJkPpNyQgg",
        "outputId": "a2349a22-3394-4cc2-dc32-015c7d15181b"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TtestResult(statistic=-21.702958031833415, pvalue=3.542197589593881e-12, df=14)"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ttest_rel(expl_mean,demo_mean)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ukSn--mbySu2",
        "outputId": "ee8c7855-b5ed-48a5-ceb1-0e3550972660"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TtestResult(statistic=5.151712539350739, pvalue=0.00014703165267441895, df=14)"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "latex_envs": {
      "LaTeX_envs_menu_present": true,
      "autoclose": false,
      "autocomplete": true,
      "bibliofile": "biblio.bib",
      "cite_by": "apalike",
      "current_citInitial": 1,
      "eqLabelWithNumbers": true,
      "eqNumInitial": 1,
      "hotkeys": {
        "equation": "Ctrl-E",
        "itemize": "Ctrl-I"
      },
      "labels_anchors": false,
      "latex_user_defs": false,
      "report_style_numbering": false,
      "user_envs_cfg": false
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bd09808012e14ffabaeb2e3ebf461de3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_74b1ad1d52d141cc869adfdf3b54802c",
              "IPY_MODEL_9a0c508ae5d54c538564a04f2cb667e6",
              "IPY_MODEL_0158b3f8e8a043a0bf474cd8d8d17e21"
            ],
            "layout": "IPY_MODEL_55869d9c62a04caa8cdb8138532d1053"
          }
        },
        "74b1ad1d52d141cc869adfdf3b54802c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a6aaa5e2f6d846b980f993b7c964be9a",
            "placeholder": "​",
            "style": "IPY_MODEL_3fc93ab0b58a44818f37649ed9ef82fe",
            "value": "100%"
          }
        },
        "9a0c508ae5d54c538564a04f2cb667e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35b83e742de7475b999d75cb6fd61af7",
            "max": 30,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_344b51abe9b3474ca52c11215844d12f",
            "value": 30
          }
        },
        "0158b3f8e8a043a0bf474cd8d8d17e21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_29b2f53073654023a6ecdc45a28841d4",
            "placeholder": "​",
            "style": "IPY_MODEL_ed9b3e1f986a4b55b7169c15fa9febfd",
            "value": " 30/30 [06:13&lt;00:00, 13.38s/it]"
          }
        },
        "55869d9c62a04caa8cdb8138532d1053": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6aaa5e2f6d846b980f993b7c964be9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fc93ab0b58a44818f37649ed9ef82fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35b83e742de7475b999d75cb6fd61af7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "344b51abe9b3474ca52c11215844d12f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "29b2f53073654023a6ecdc45a28841d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed9b3e1f986a4b55b7169c15fa9febfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}