modularize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/data_setup.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/data_setup.py\n",
    "\"\"\"\n",
    "Contains functionality for creating PyTorch DataLoaders for \n",
    "LIBS benchmark classification dataset.\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from load_libs_data import load_contest_train_dataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import Normalizer, MinMaxScaler\n",
    "from siamese_net import prepare_triplets\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "NUM_WORKERS = os.cpu_count()\n",
    "\n",
    "def create_dataloaders(\n",
    "    train_dir: str, \n",
    "    #test_dir: str, \n",
    "    batch_size: int, \n",
    "    device: torch.device,\n",
    "    num_workers: int=NUM_WORKERS, \n",
    "    split_rate: float=0.6,\n",
    "    random_st: int=102,\n",
    "    spectra_count: int=100\n",
    "    ):\n",
    "    \"\"\"Creates training and validation DataLoaders.\n",
    "\n",
    "    Takes in a training directory directory path and split the data\n",
    "    to train/validation. After, it turns them into PyTorch Datasets and \n",
    "    then into PyTorch DataLoaders.\n",
    "\n",
    "    Args:\n",
    "    train_dir: Path to training directory.\n",
    "    batch_size: Number of samples per batch in each of the DataLoaders.\n",
    "    num_workers: An integer for number of workers per DataLoader.\n",
    "\n",
    "    Returns:\n",
    "    A tuple of (train_dataloader, test_dataloader, class_labels).\n",
    "    Example usage:\n",
    "        train_dataloader, test_dataloader, class_labels, wavelengths = \\\n",
    "        = create_dataloaders(train_dir=path/to/train_dir,\n",
    "                                test_dir=path/to/test_dir,\n",
    "                                transform=some_transform,\n",
    "                                batch_size=32,\n",
    "                                num_workers=4)\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "\n",
    "    X, y, samples = load_contest_train_dataset(train_dir, spectra_count)\n",
    "    wavelengths = X.columns\n",
    "\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=split_rate, random_state=random_st, stratify=samples, shuffle = True)\n",
    "    del X, y, samples\n",
    "\n",
    "    if True:\n",
    "      scaler =  Normalizer(norm = 'max')\n",
    "      X_train = scaler.fit_transform(X_train)\n",
    "      X_val = scaler.fit_transform(X_val)\n",
    "\n",
    "    # Convert data to torch tensors\n",
    "    X_train = torch.from_numpy(X_train).unsqueeze(1).float() # Add extra dimension for channels\n",
    "    X_val = torch.from_numpy(X_val).unsqueeze(1).float() # Add extra dimension for channels\n",
    "    y_train = torch.from_numpy(np.array(y_train)).long()\n",
    "    y_val = torch.from_numpy(np.array(y_val)).long()\n",
    "\n",
    "    #device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    # If available, move data to the GPU\n",
    "    X_train.to(device)\n",
    "    X_val.to(device) \n",
    "    y_train.to(device)\n",
    "    y_val.to(device)\n",
    "\n",
    "    # Prepare triplets for the training, validation\n",
    "    train_triplets = prepare_triplets(X_train, y_train)\n",
    "    val_triplets = prepare_triplets(X_val, y_val)\n",
    "\n",
    "\n",
    "    # Create PyTorch DataLoader objects for the training and validation sets\n",
    "    train_dataloader = DataLoader(train_triplets, batch_size=batch_size, shuffle=True)\n",
    "    val_dataloader = DataLoader(val_triplets, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "\n",
    "    return train_dataloader, val_dataloader, y_train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/engine.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/engine.py\n",
    "\"\"\"\n",
    "Contains functions for training and validating a PyTorch model.\n",
    "\"\"\"\n",
    "import torch\n",
    "import wandb\n",
    "from tqdm.auto import tqdm\n",
    "from typing import Dict, List, Tuple\n",
    "\n",
    "def train_step(model: torch.nn.Module, \n",
    "               dataloader: torch.utils.data.DataLoader, \n",
    "               loss_fn: torch.nn.Module, \n",
    "               optimizer: torch.optim.Optimizer,\n",
    "               device: torch.device):\n",
    "    \"\"\"Trains a PyTorch model for a single epoch.\n",
    "\n",
    "    Turns a target PyTorch model to training mode and then\n",
    "    runs through all of the required training steps (forward\n",
    "    pass, loss calculation, optimizer step).\n",
    "\n",
    "    Args:\n",
    "        model: A PyTorch model to be trained.\n",
    "        dataloader: A DataLoader instance for the model to be trained on.\n",
    "        loss_fn: A PyTorch loss function to minimize.\n",
    "        optimizer: A PyTorch optimizer to help minimize the loss function.\n",
    "        device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
    "\n",
    "    Returns:\n",
    "        training loss\n",
    "    \"\"\"\n",
    "    model.train()\n",
    "    # Initialize running loss and number of batches for the training dataset\n",
    "    running_loss_train = 0.0\n",
    "    num_batches_train = 0\n",
    "\n",
    "    # Iterate over the training data loader\n",
    "    for i, data in enumerate(dataloader, 0):\n",
    "        # Get the inputs and labels for the current batch\n",
    "        inputs = [input.to(device) for input in data]\n",
    "        \n",
    "        input_a = inputs[0]\n",
    "        input_p = inputs[1]\n",
    "        input_n = inputs[2]\n",
    "        \n",
    "        # Compute the embeddings for the current batch\n",
    "        outputs = model(input_a, input_p, input_n)\n",
    "\n",
    "\n",
    "        # Split the embeddings into anchor, positive, and negative examples\n",
    "        anchor = outputs[0]\n",
    "        positive = outputs[1]\n",
    "        negative = outputs[2]\n",
    "        \n",
    "        # Compute the triplet loss for the current batch\n",
    "        loss_train = loss_fn(anchor, positive, negative)\n",
    "\n",
    "        # Backpropagate the loss and update the network parameters\n",
    "        loss_train.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # Update the running loss and number of batches for the training dataset\n",
    "        running_loss_train += loss_train.item()\n",
    "        num_batches_train += 1\n",
    "\n",
    "    # Compute the average loss for the training dataset\n",
    "    epoch_loss_train = running_loss_train / num_batches_train\n",
    "    return epoch_loss_train\n",
    "\n",
    "def val_step(model: torch.nn.Module, \n",
    "               dataloader: torch.utils.data.DataLoader, \n",
    "               loss_fn: torch.nn.Module, \n",
    "               device: torch.device):\n",
    "    \"\"\"validates a PyTorch model for a single epoch.\n",
    "\n",
    "    Turns a target PyTorch model to \"eval\" mode and then performs\n",
    "    a forward pass on a testing dataset.\n",
    "\n",
    "    Args:\n",
    "    model: A PyTorch model to be tested.\n",
    "    dataloader: A DataLoader instance for the model to be tested on.\n",
    "    loss_fn: A PyTorch loss function to calculate loss on the test data.\n",
    "    device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
    "\n",
    "    Returns:\n",
    "    test loss\n",
    "\n",
    "    \"\"\"\n",
    "    model.eval() \n",
    "\n",
    "\n",
    "    # Initialize running loss and number of batches for the validation dataset\n",
    "    running_loss_val = 0.0\n",
    "    num_batches_val = 0\n",
    "\n",
    "    # Turn on inference context manager\n",
    "    with torch.no_grad():\n",
    "        for i, data in enumerate(dataloader, 0):\n",
    "            # Get the inputs and labels for the current batch\n",
    "\n",
    "            inputs_val = [input.to(device) for input in data]\n",
    "\n",
    "            input_a = inputs_val[0]\n",
    "            input_p = inputs_val[1]\n",
    "            input_n = inputs_val[2]\n",
    "            # Compute the embeddings for the current batch\n",
    "            outputs = model(input_a, input_p, input_n)\n",
    "\n",
    "            # Split the embeddings into anchor, positive, and negative examples\n",
    "            anchor = outputs[0]\n",
    "            positive = outputs[1]\n",
    "            negative = outputs[2]\n",
    "\n",
    "            # Compute the triplet loss for the current batch\n",
    "            loss_val = loss_fn(anchor, positive, negative)\n",
    "\n",
    "            # Update the running loss and number of batches for the validation dataset\n",
    "            running_loss_val += loss_val.item()\n",
    "            num_batches_val += 1\n",
    "\n",
    "    # Compute the average loss for the validation dataset\n",
    "    epoch_loss_val = running_loss_val / num_batches_val\n",
    "    return epoch_loss_val\n",
    "\n",
    "def train(model: torch.nn.Module, \n",
    "          train_dataloader: torch.utils.data.DataLoader, \n",
    "          val_dataloader: torch.utils.data.DataLoader, \n",
    "          optimizer: torch.optim.Optimizer,\n",
    "          loss_fn: torch.nn.Module,\n",
    "          epochs: int,\n",
    "          device: torch.device) -> Dict[str, List]:\n",
    "  \"\"\"Trains and validates a PyTorch model.\n",
    "\n",
    "  Passes a target PyTorch models through train_step() and val_step()\n",
    "  functions for a number of epochs, training and validating the model\n",
    "  in the same epoch loop.\n",
    "\n",
    "  Calculates, prints and stores evaluation metrics throughout.\n",
    "\n",
    "  Args:\n",
    "    model: A PyTorch model to be trained and tested.\n",
    "    train_dataloader: A DataLoader instance for the model to be trained on.\n",
    "    val_dataloader: A DataLoader instance for the model to be tested on.\n",
    "    optimizer: A PyTorch optimizer to help minimize the loss function.\n",
    "    loss_fn: A PyTorch loss function to calculate loss on both datasets.\n",
    "    epochs: An integer indicating how many epochs to train for.\n",
    "    device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
    "\n",
    "  Returns:\n",
    "    A dictionary of training and validating loss as well as training and\n",
    "    validating accuracy metrics. Each metric has a value in a list for \n",
    "    each epoch.\n",
    "    In the form: {train_loss: [...],\n",
    "                  val_loss: [...]} \n",
    "    For example if training for epochs=2: \n",
    "                 {train_loss: [2.0616, 1.0537],\n",
    "                  val_loss: [1.2641, 1.5706]} \n",
    "  \"\"\"\n",
    "  # Create empty results dictionary\n",
    "  results = {\"train_loss\": [], \"val_loss\": []}\n",
    "\n",
    "  wandb.init(project='siamese_net_contest', entity='jakubv')\n",
    "  wandb.watch(model)  \n",
    "  # Loop through training and validating steps for a number of epochs\n",
    "  for epoch in tqdm(range(epochs)):\n",
    "      train_loss = train_step(model=model,\n",
    "                            dataloader=train_dataloader,\n",
    "                            loss_fn=loss_fn,\n",
    "                            optimizer=optimizer,\n",
    "                            device=device)\n",
    "      val_loss = val_step(model=model,\n",
    "                            dataloader=val_dataloader,\n",
    "                            loss_fn=loss_fn,\n",
    "                            device=device)\n",
    "\n",
    "      # Print out what's happening\n",
    "      print(\n",
    "          f\"Epoch: {epoch+1} | \"\n",
    "          f\"train_loss: {train_loss:.4f} | \"\n",
    "          f\"val_loss: {val_loss:.4f} \"\n",
    "      )\n",
    "      wandb.log({\n",
    "        \"epoch\": epoch+1,\n",
    "        \"training_loss\": train_loss, \n",
    "        \"validation_loss\": val_loss\n",
    "        })  \n",
    "      # Update results dictionary\n",
    "      results[\"train_loss\"].append(train_loss)\n",
    "      results[\"val_loss\"].append(val_loss)\n",
    "  wandb.finish()  \n",
    "  # Return the filled results at the end of the epochs\n",
    "  return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/utils.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/utils.py\n",
    "\"\"\"\n",
    "Contains various utility functions for PyTorch model training and saving.\n",
    "\"\"\"\n",
    "import torch\n",
    "from pathlib import Path\n",
    "\n",
    "def save_model(model: torch.nn.Module,\n",
    "               target_dir: str,\n",
    "               model_name: str):\n",
    "  \"\"\"Saves a PyTorch model to a target directory.\n",
    "\n",
    "  Args:\n",
    "    model: A target PyTorch model to save.\n",
    "    target_dir: A directory for saving the model to.\n",
    "    model_name: A filename for the saved model. Should include\n",
    "      either \".pth\" or \".pt\" as the file extension.\n",
    "\n",
    "  Example usage:\n",
    "    save_model(model=model_0,\n",
    "               target_dir=\"models\",\n",
    "               model_name=\"05_going_modular_tingvgg_model.pth\")\n",
    "  \"\"\"\n",
    "  # Create target directory\n",
    "  target_dir_path = Path(target_dir)\n",
    "  target_dir_path.mkdir(parents=True,\n",
    "                        exist_ok=True)\n",
    "\n",
    "  # Create model save path\n",
    "  assert model_name.endswith(\".pth\") or model_name.endswith(\".pt\"), \"model_name should end with '.pt' or '.pth'\"\n",
    "  model_save_path = target_dir_path / model_name\n",
    "\n",
    "  # Save the model state_dict()\n",
    "  print(f\"[INFO] Saving model to: {model_save_path}\")\n",
    "  torch.save(obj=model.state_dict(),\n",
    "             f=model_save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/train.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/train.py\n",
    "\"\"\"\n",
    "Trains a PyTorch image classification model using device-agnostic code.\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import torch\n",
    "import data_setup, engine, siamese_net, utils\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "\n",
    "\n",
    "# Setup hyperparameters\n",
    "NUM_EPOCHS = 50\n",
    "BATCH_SIZE = 128\n",
    "LEARNING_RATE = 0.0001\n",
    "INPUT_SIZE = 40000\n",
    "CHANNELS=50\n",
    "KERNEL_SIZES=[50, 10]\n",
    "STRIDES=[2, 2]\n",
    "PADDINGS=[1, 1]\n",
    "HIDDEN_SIZES=[256]\n",
    "\n",
    "# Setup directories\n",
    "train_dir = \"datasets/contest_TRAIN.h5\"\n",
    "#test_dir = \"datasets/test.h5\"\n",
    "\n",
    "# Setup target device\n",
    "#device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "# Create DataLoaders with help from data_setup.py\n",
    "train_dataloader, val_dataloader, train_labels = data_setup.create_dataloaders(\n",
    "    train_dir=train_dir,\n",
    "    #test_dir=test_dir,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    device = device\n",
    ")\n",
    "\n",
    "# Create model with help from siamese_net.py\n",
    "# margin = 1.0\n",
    "\n",
    "model = siamese_net.SiameseNetwork(\n",
    "    input_size=INPUT_SIZE, \n",
    "    output_size=len(np.unique(train_labels)), \n",
    "    channels=CHANNELS, \n",
    "    kernel_sizes=KERNEL_SIZES, \n",
    "    strides=STRIDES, \n",
    "    paddings=PADDINGS, \n",
    "    hidden_sizes=HIDDEN_SIZES\n",
    ").to(device)\n",
    "\n",
    "# Set loss and optimizer\n",
    "loss_fn = nn.TripletMarginLoss(margin=1.0)\n",
    "optimizer = torch.optim.Adam(model.parameters(),\n",
    "                             lr=LEARNING_RATE)\n",
    "\n",
    "# Start training with help from engine.py\n",
    "engine.train(model=model,\n",
    "             train_dataloader=train_dataloader,\n",
    "             val_dataloader=val_dataloader,\n",
    "             loss_fn=loss_fn,\n",
    "             optimizer=optimizer,\n",
    "             epochs=NUM_EPOCHS,\n",
    "             device=device)\n",
    "\n",
    "# Save the model with help from utils.py\n",
    "utils.save_model(model=model,\n",
    "                 target_dir=\"models\",\n",
    "                 model_name=\"modularity_test1.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%writefile src/test_data.py\n",
    "# \"\"\"\n",
    "# Contains functionality for creating PyTorch DataLoaders for \n",
    "# LIBS benchmark classification dataset.\n",
    "# \"\"\"\n",
    "\n",
    "# import os\n",
    "# import torch\n",
    "# from torch.utils.data import DataLoader\n",
    "# from load_libs_data import load_contest_train_dataset\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.preprocessing import Normalizer, MinMaxScaler\n",
    "# from siamese_net import prepare_triplets\n",
    "# import numpy as np\n",
    "\n",
    "\n",
    "# NUM_WORKERS = os.cpu_count()\n",
    "\n",
    "# def create_dataloaders(\n",
    "#     train_dir: str, \n",
    "#     #test_dir: str, \n",
    "#     batch_size: int, \n",
    "#     device: torch.device,\n",
    "#     num_workers: int=NUM_WORKERS, \n",
    "#     split_rate: float=0.6,\n",
    "#     random_st: int=102,\n",
    "#     spectra_count: int=100\n",
    "#     ):\n",
    "#     \"\"\"Creates training and validation DataLoaders.\n",
    "\n",
    "#     Takes in a training directory directory path and split the data\n",
    "#     to train/validation. After, it turns them into PyTorch Datasets and \n",
    "#     then into PyTorch DataLoaders.\n",
    "\n",
    "#     Args:\n",
    "#     train_dir: Path to training directory.\n",
    "#     batch_size: Number of samples per batch in each of the DataLoaders.\n",
    "#     num_workers: An integer for number of workers per DataLoader.\n",
    "\n",
    "#     Returns:\n",
    "#     A tuple of (train_dataloader, test_dataloader, class_labels).\n",
    "#     Example usage:\n",
    "#         train_dataloader, test_dataloader, class_labels, wavelengths = \\\n",
    "#         = create_dataloaders(train_dir=path/to/train_dir,\n",
    "#                                 test_dir=path/to/test_dir,\n",
    "#                                 transform=some_transform,\n",
    "#                                 batch_size=32,\n",
    "#                                 num_workers=4)\n",
    "#     \"\"\"\n",
    "\n",
    "\n",
    "\n",
    "#     X, y, samples = load_contest_train_dataset(train_dir, spectra_count)\n",
    "#     wavelengths = X.columns\n",
    "\n",
    "#     X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=split_rate, random_state=random_st, stratify=samples, shuffle = True)\n",
    "#     del X, y, samples\n",
    "\n",
    "#     if True:\n",
    "#       scaler =  Normalizer(norm = 'max')\n",
    "#       X_train = scaler.fit_transform(X_train)\n",
    "#       X_val = scaler.fit_transform(X_val)\n",
    "\n",
    "#     # Convert data to torch tensors\n",
    "#     X_train = torch.from_numpy(X_train).unsqueeze(1).float() # Add extra dimension for channels\n",
    "#     X_val = torch.from_numpy(X_val).unsqueeze(1).float() # Add extra dimension for channels\n",
    "#     y_train = torch.from_numpy(np.array(y_train)).long()\n",
    "#     y_val = torch.from_numpy(np.array(y_val)).long()\n",
    "\n",
    "#     #device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "#     # If available, move data to the GPU\n",
    "#     X_train.to(device)\n",
    "#     X_val.to(device) \n",
    "#     y_train.to(device)\n",
    "#     y_val.to(device)\n",
    "\n",
    "#     # Prepare triplets for the training, validation\n",
    "#     train_triplets = prepare_triplets(X_train, y_train)\n",
    "#     val_triplets = prepare_triplets(X_val, y_val)\n",
    "\n",
    "\n",
    "#     # Create PyTorch DataLoader objects for the training and validation sets\n",
    "#     train_dataloader = DataLoader(train_triplets, batch_size=batch_size, shuffle=True)\n",
    "#     val_dataloader = DataLoader(val_triplets, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "\n",
    "#     return train_dataloader, val_dataloader, y_train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# import funcs\n",
    "# import importlib\n",
    "# importlib.reload(funcs)\n",
    "# from funcs import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.load_libs_data import load_contest_test_dataset\n",
    "\n",
    "path = 'datasets/contest_TEST.h5'\n",
    "\n",
    "X_test = load_contest_test_dataset(path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x180d6e38850>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjEAAAGdCAYAAADjWSL8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTlElEQVR4nO3deXgT1cIG8LelC1DaSIG2FAqCVgXLoiBQREBZFUTlKiraT+9V0KuAXFGvyFVxo4gKKLgiFxHEqldxxcpeQMpiS6FsZSsUKF2ANt235Hx/lA5NszRpJsnM5P09D89TkpOZM5lk5s2Zc874CCEEiIiIiFTG19MVICIiImoKhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJT9PV8BVjEYjsrOzERwcDB8fH09Xh4iIiOwghEBxcTEiIyPh62u7rUWzISY7OxtRUVGergYRERE1wenTp9GxY0ebZTQbYoKDgwHUvgkhISEerg0RERHZo6ioCFFRUdJ53BbNhpi6S0ghISEMMURERCpjT1cQduwlIiIiVWKIISIiIlViiCEiIiJVYoghIiIiVWKIISIiIlViiCEiIiJVYoghIiIiVWKIISIiIlViiCEiIiJVcijE1NTU4D//+Q+6dOmCFi1aoGvXrnj99ddhNBqlMkIIzJ49G5GRkWjRogWGDh2KAwcOmCynsrISU6dORdu2bREUFIRx48bhzJkzJmUKCgoQFxcHnU4HnU6HuLg4FBYWNn1LiYiISFMcCjFvv/02PvnkEyxevBiHDh3CvHnz8M4772DRokVSmXnz5mH+/PlYvHgxdu/ejYiICIwYMQLFxcVSmenTp2P16tVISEjAtm3bUFJSgrFjx8JgMEhlJk6ciLS0NCQmJiIxMRFpaWmIi4uTYZOJiIhIE4QDxowZI/7xj3+YPDZ+/Hjx8MMPCyGEMBqNIiIiQsydO1d6vqKiQuh0OvHJJ58IIYQoLCwU/v7+IiEhQSpz9uxZ4evrKxITE4UQQhw8eFAAEDt27JDKJCcnCwDi8OHDdtVVr9cLAEKv1zuyiURERORBjpy/HWqJGTRoEDZs2IAjR44AAPbu3Ytt27bhjjvuAABkZmYiJycHI0eOlF4TGBiIIUOGYPv27QCAlJQUVFdXm5SJjIxETEyMVCY5ORk6nQ79+/eXygwYMAA6nU4q01BlZSWKiopM/hEBQI3BiM+3nsDBbH4miIi0xKG7WP/73/+GXq/Hddddh2bNmsFgMOCtt97Cgw8+CADIyckBAISHh5u8Ljw8HKdOnZLKBAQEoHXr1mZl6l6fk5ODsLAws/WHhYVJZRqKj4/Ha6+95sjmkJdYueMU3vztEADg5NwxHq4NERHJxaGWmG+++QYrV67EqlWrkJqaiuXLl+Pdd9/F8uXLTco1vH22EKLRW2o3LGOpvK3lzJw5E3q9Xvp3+vRpezeLNG4/W2CIiDTJoZaY559/Hi+++CIeeOABAECPHj1w6tQpxMfH45FHHkFERASA2paU9u3bS6/Ly8uTWmciIiJQVVWFgoICk9aYvLw8DBw4UCqTm5trtv78/HyzVp46gYGBCAwMdGRziIiISMUcaokpKyuDr6/pS5o1ayYNse7SpQsiIiKwbt066fmqqiokJSVJAaVPnz7w9/c3KXPu3Dns379fKhMbGwu9Xo9du3ZJZXbu3Am9Xi+VISIiIu/mUEvMnXfeibfeegudOnXC9ddfjz179mD+/Pn4xz/+AaD2EtD06dMxZ84cREdHIzo6GnPmzEHLli0xceJEAIBOp8Njjz2GGTNmoE2bNggNDcVzzz2HHj16YPjw4QCAbt26YfTo0Zg0aRI+/fRTAMDkyZMxduxYXHvttXJuPxEREamUQyFm0aJFePnll/HUU08hLy8PkZGReOKJJ/DKK69IZV544QWUl5fjqaeeQkFBAfr374+1a9ciODhYKrNgwQL4+flhwoQJKC8vx7Bhw/DFF1+gWbNmUpmvvvoK06ZNk0YxjRs3DosXL3Z2e4mIiEgjfIQQwtOVcIWioiLodDro9XqEhIR4ujrkQc99txf/S6mdEZqjk4iIlM2R8zfvnURERESqxBBDREREqsQQQ0RERKrEEENERESqxBBDREREqsQQQ0RERKrEEENERESqxBBDmqfNmZCIiIghhoiIiFSJIYY0z8fH0zUgIiJXYIghIiIiVWKIISIiIlViiCEiIiJVYoghIiIiVWKIIc3jEGsiIm1iiCEiIiJVYoghzeMQayIibWKIISIiIlViiCEiIiJVYoghIiIiVWKIISIiIlViiCHN4xBrIiJtYoghIiIiVWKIIc3jEGsiIm1iiCEiIiJVYoghIiIiVWKIISIiIlViiCEiIiJVYoghzTMYOcaaiEiLGGJI81bvOevpKhARkQswxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEkOaN6dHe01UgIiIXYIghzWvTKsDTVSAiIhdgiCHNE8LTNSBPqKwxYP9ZPQQ/AESaxRBDRJo0+csUjF20DSt2nPJ0VYjIRRhiiEiTko7kAwCWbz/p2YoQkcswxBAREZEqMcQQERGRKjHEEBERkSoxxBAREZEqMcQQERGRKjHEEBERkSo5HGLOnj2Lhx9+GG3atEHLli3Ru3dvpKSkSM8LITB79mxERkaiRYsWGDp0KA4cOGCyjMrKSkydOhVt27ZFUFAQxo0bhzNnzpiUKSgoQFxcHHQ6HXQ6HeLi4lBYWNi0rSQir8Wp7oi0y6EQU1BQgJtvvhn+/v74/fffcfDgQbz33nu44oorpDLz5s3D/PnzsXjxYuzevRsREREYMWIEiouLpTLTp0/H6tWrkZCQgG3btqGkpARjx46FwWCQykycOBFpaWlITExEYmIi0tLSEBcX5/wWExERkSb4OVL47bffRlRUFJYtWyY9duWVV0p/CyGwcOFCzJo1C+PHjwcALF++HOHh4Vi1ahWeeOIJ6PV6LF26FCtWrMDw4cMBACtXrkRUVBTWr1+PUaNG4dChQ0hMTMSOHTvQv39/AMCSJUsQGxuLjIwMXHvttc5uNxEREamcQy0xP//8M/r27Yv77rsPYWFhuOGGG7BkyRLp+czMTOTk5GDkyJHSY4GBgRgyZAi2b98OAEhJSUF1dbVJmcjISMTExEhlkpOTodPppAADAAMGDIBOp5PKNFRZWYmioiKTf0RERKRdDoWYEydO4OOPP0Z0dDT++OMPPPnkk5g2bRq+/PJLAEBOTg4AIDw83OR14eHh0nM5OTkICAhA69atbZYJCwszW39YWJhUpqH4+Hip/4xOp0NUVJQjm0YaJtgrgohIkxwKMUajETfeeCPmzJmDG264AU888QQmTZqEjz/+2KScj4+Pyf+FEGaPNdSwjKXytpYzc+ZM6PV66d/p06ft3SwiIiJSIYdCTPv27dG9e3eTx7p164asrCwAQEREBACYtZbk5eVJrTMRERGoqqpCQUGBzTK5ublm68/Pzzdr5akTGBiIkJAQk39ERLZ/PhGRmjkUYm6++WZkZGSYPHbkyBF07twZANClSxdERERg3bp10vNVVVVISkrCwIEDAQB9+vSBv7+/SZlz585h//79UpnY2Fjo9Xrs2rVLKrNz507o9XqpDJG9fHga82q8mEikXQ6NTvrXv/6FgQMHYs6cOZgwYQJ27dqFzz77DJ999hmA2ktA06dPx5w5cxAdHY3o6GjMmTMHLVu2xMSJEwEAOp0Ojz32GGbMmIE2bdogNDQUzz33HHr06CGNVurWrRtGjx6NSZMm4dNPPwUATJ48GWPHjuXIJHIY+8QQEWmTQyHmpptuwurVqzFz5ky8/vrr6NKlCxYuXIiHHnpIKvPCCy+gvLwcTz31FAoKCtC/f3+sXbsWwcHBUpkFCxbAz88PEyZMQHl5OYYNG4YvvvgCzZo1k8p89dVXmDZtmjSKady4cVi8eLGz20tEREQa4SOE0OTP1KKiIuh0Ouj1evaP8XL/+TEdK3fU9ts6OXeMh2tD7nLli78BALq2C8LGGUM9Wxkispsj52/eO4mIiIhUiSGGNI8de4mItIkhhjSPHXuJiLSJIYaIiIhUiSGGiLSNDXFEmsUQQ0RERKrEEENERESqxBBDREREqsQQQ0TaxhH2RJrFEENERESqxBBDREREqsQQQ5qnzbuDkd24/4k0iyGGiIiIVIkhhjTPhx07iYg0iSGGiIiIVIkhhjSPfWKIiLSJIYaIiIhUiSGGiIiIVIkhhoiIiFSJIYaINI1dooi0iyGGiIiIVIkhhog0jdMEEWkXQwwRERGpEkMMERHZ5du/TuPdPzI8XQ0iiZ+nK0BEROrwwv/2AQCGdw9H76grPFsZIrAlhoiIHKQvr/Z0FYgAMMQQkcZxiDWRdjHEEBERkSoxxJDm8Zc4EZE2McQQERGRKjHEEBGRQziBICkFQwxpHg+4RETaxBBDmsc+MURE2sQQQ0SaxpY4Iu1iiCEiTWNLHJF2McQQERGRKjHEEBERkSoxxBARWbD/rB4HsvWergYR2cC7WBMRNVBeZcDYRdsAABlvjkagXzMP14iILGFLDBFRA8UVl+/SXFFl9GBNiMgWhhgiInKID8etk0IwxBCRpgnBQdZEWsUQQ0RERKrEEENERESqxBBDREREqsQQQ0RERKrEEEOax36d3s2HQ2mINIshhoioAeZeInVgiCHNq/9D/Je92cguLPdcZcjtOMSaSLt42wHyKlO/3oNmvj44PucOT1eF1IJXo4gUiy0xpHkNf4gbjPxlTg7gx8XMEytS8Pjyv9jKRR7HEENE1AAbX2wrqzJg/aFcnC+p8nRVyMsxxBAREZEqMcQQERGRKjHEEBERkSoxxBARNcDuqkTqwBBDRJrGQEKkXQwxRGQir6gCx/KKPV0NIqJGMcQQkYl+czZg+PwtOKfnzMZkm2A7F3kYQwwRWXQ4RxutMZzzhUi7GGKIiGxhCrLKh28OeRhDDBGRDTn6Ck9XgYisYIghImqgfvvCqIVbsPFwrsfqQkTWMcSQF2DnQ7XSl1XjYqlz9+eRY+//d9tJGZZCRHJjiCGSSVlVDfKKeelBLkajQK/X1+LGN9ahotrg1nUz9hKpA0MMkUz6vbUB/d7agNwiBhk5VBmM0t98T4nIEoYY8gLuGUFRUlkDANiVedEt63M1jjuhxnCeGPI0hhjyAjzQEhFpEUMMebVz+nIcydXGpG5yY/SjxnCeGPI0hhjyarHxGzFywRb2uSDyEqlZBRj23mYkHcn3dFVIBk6FmPj4ePj4+GD69OnSY0IIzJ49G5GRkWjRogWGDh2KAwcOmLyusrISU6dORdu2bREUFIRx48bhzJkzJmUKCgoQFxcHnU4HnU6HuLg4FBYWOlNdIquO5ZV4ugpE5AZxn+/E8fxSPPLfXZ6uCsmgySFm9+7d+Oyzz9CzZ0+Tx+fNm4f58+dj8eLF2L17NyIiIjBixAgUF19usp8+fTpWr16NhIQEbNu2DSUlJRg7diwMhsvDKCdOnIi0tDQkJiYiMTERaWlpiIuLa2p1iUjFhIevbfnwqolmlFa5d7g+uVaTQkxJSQkeeughLFmyBK1bt5YeF0Jg4cKFmDVrFsaPH4+YmBgsX74cZWVlWLVqFQBAr9dj6dKleO+99zB8+HDccMMNWLlyJdLT07F+/XoAwKFDh5CYmIjPP/8csbGxiI2NxZIlS/Drr78iIyNDhs0mIrLO06GJiOzTpBDz9NNPY8yYMRg+fLjJ45mZmcjJycHIkSOlxwIDAzFkyBBs374dAJCSkoLq6mqTMpGRkYiJiZHKJCcnQ6fToX///lKZAQMGQKfTSWWIiIjIu/k5+oKEhASkpqZi9+7dZs/l5OQAAMLDw00eDw8Px6lTp6QyAQEBJi04dWXqXp+Tk4OwsDCz5YeFhUllGqqsrERlZaX0/6KiIge2ioga8uYrKA0vH7FlxjLOE0Oe5lBLzOnTp/HMM89g5cqVaN68udVyPg2OAEIIs8caaljGUnlby4mPj5c6Aet0OkRFRdlcHxEREambQyEmJSUFeXl56NOnD/z8/ODn54ekpCR88MEH8PPzk1pgGraW5OXlSc9FRESgqqoKBQUFNsvk5prfNTY/P9+slafOzJkzodfrpX+nT592ZNOISMHYsVaZOE8MeZpDIWbYsGFIT09HWlqa9K9v37546KGHkJaWhq5duyIiIgLr1q2TXlNVVYWkpCQMHDgQANCnTx/4+/ublDl37hz2798vlYmNjYVer8euXZeHwO3cuRN6vV4q01BgYCBCQkJM/hEREZF2OdQnJjg4GDExMSaPBQUFoU2bNtLj06dPx5w5cxAdHY3o6GjMmTMHLVu2xMSJEwEAOp0Ojz32GGbMmIE2bdogNDQUzz33HHr06CF1FO7WrRtGjx6NSZMm4dNPPwUATJ48GWPHjsW1117r9EYTNcQ+D+aU9JY4sn8Wrj+C/Wf1TXotEamLwx17G/PCCy+gvLwcTz31FAoKCtC/f3+sXbsWwcHBUpkFCxbAz88PEyZMQHl5OYYNG4YvvvgCzZo1k8p89dVXmDZtmjSKady4cVi8eLHc1SUijVm4/qjsy+TlLCJlcjrEbN682eT/Pj4+mD17NmbPnm31Nc2bN8eiRYuwaNEiq2VCQ0OxcuVKZ6tHREREGsV7J5HmyX054Zy+HBXVBmRdKIO+rFrehSuINzc+8BKUfTjEmjxN9stJRGpk7+WC4/klGPZeksljJ+eOcUGNiIioMWyJIc2Tsz/DxkN58i2MiIicwhBDRJrGTrmuw3liyNMYYkjz2L9BnZQUPvgZIlImhhgi0rSmBBAlBSgiso4hhgjy/tLmCVB7uE+JlIkhhogUiZdwiKgxDDFEMuPJV37ZheVuXR/3oX04Twx5GkMMkQN40PaMF39I93QViEiBGGKISPEullZ5ugpEpEAMMUQusvVoPr5MPunpaqhW/c60gtd3FInzxJCn8bYDRC4St3QXAKB7+xD0vTLUw7XxXrwESKRdbIkhgmuH0J51c6dULfLhGGcisoAhhsgB3tR87s3Bga035nhJj5SIIYYIHFKrdN4bp4jIFoYYIic1/IWqlQYMRf3ydvN76k0tbs5gixV5GkMMaZ6c52JLB+1JX6bItwInlFbW4I8DOSivMni6KrJjpPA8JWVaojoMMdRkNQYj8oorPF0Nj1t/KNfTVQAAPJOQhidWpODFH/Z5uiqyqH/S5PmTiCxhiKEmu+/TZPR7awP2n9V7uiqKMmXVHuQWuT/c1YWpn9Ky3b5uJWMLApF2McRQk+3JKgQAfJ96xrMVaYQn+qi8/stB6W+eRJ3n7l3Ivh7mLL0j7DtEnsYQQ5rniRChL692/0plpqQh1kqqCxEpB0MMEWkaW8KItIshhgi8fKB0bIhRJjV+b/hZ0haGGCIiapSi5g0iuoQhhpymhc59WtgGIiJvwxBDRBYp6Ze320cnKWfTicgGhhhyGq8xk6t5enRSVY0RpZU1Hq2DpzHXkRIxxBC5mBo7PwKeDw6e1HDTd2ZexPWv/qHJWzo4Q42XYdVXY7KFIYacxoOCObUGF7LtWF6Jp6tATuI3U1sYYkjzGCjUj0GZiCxhiCFyATU2s9NlaujYOy/xMD5JOu629Vl6T9T4A4HfTG1hiCHNyLpQhpRTF80etydQuPJgrIYToiucyC/BwewiT1cDZwvLZVuWUk7apy6U4qPNxzH398OergqRR/l5ugKkfkrp/zn4nU0AgA0zhuCqdq1csg57A8m2Y+ddsn41ue29JADAnpdHoHVQgIdroy3l1exgTASwJYZkoLRRLIfPFXu6ClTPOX2F08tQ2EfMKymlFYqoPoYY0jwefImItIkhhohUQBlNMd7av8kadmAnT2OIIacp/TBmz4HW3oPxf//MdLY6XkeOljB3X05iVjGnlQCntMvf5ByGGCIH5BZVeroKRJoJFETOYoghzSkoqzL5P/vEqB9/OysTv1vkaQwx5DyFnWFe/mm/p6tggr+a1cfaR1opu7L+FREl3W2cyN0YYkhzmnJM5y9KIiL1YYghp3GEAtnS1IYCNjCQK/BopS0MMURkkTcf7JmfiNSBIYYU7XBOkdX77+jLq91cm6bx9hOiHCNaOSrWlCdaqSytk62w5GkMMaRYlTUGjF64FXd8sBVlVTUmz63ecwa9XluLDzYclWdd1UaT/5dU1lgp6T2UFL54siQiSxhiyGmu+pVcUXU5WBSVm4aKF79PBwDMX3dElnU9/uVfKL0UXDYezkXMq3/wDsEyYd8W7WKHePI0hhhymlZ+I6dmFQAAXv/lIADgk6TjnqwO1cPLSab4fhDVYoghVaqsMTZeSCE4j4d2KHFfuqtKWml1YQDUFoYYIhfjvVrUR4lhheTBXastfp6uAKmf3OdoIQRm/bgfYcGB8i64EXWdR3mMq6Wk6KWkuhCRcjDEkOLszLyIVTuzPF0NUhB3t2YpvfVMKUOs1Ujhu5YcxMtJpDglFfIOb9bKwZdIaTj0nTyNIYacppUDmZp/oenL1DHxnyOU2JHU3hrNWXMIf/t4O6pU1AHdW9g6Xp2+WIZv/zqNagP3m1rwchI5TfY+MfIuTvPm/n6Yw8EV5rMtJwAA6w7mYkzP9h6ujTwsfS+VGDQbY6vOt8zbBAAoKq/G47d0dVeVyAlsiSHNczRkWbr8VFJZg7TThU1bf5NeZT9XBRj1nZ7kI9fopBqja37R1/9Me/N+cpUdJy56ugpkJ7bEkObZez6yFTbGLdqGE+dLm7b+Jr1K3ThEmZSq9nISP59awZYYIjs0NcComZK6CPkq5EillGymlHoQeZpCDg2kZko62Tnl0obI3cdHM+8PeTW2rpESMcSQKrhl5NClYzSP1fLy5vdT6fPNEKkdQww5jwdqIq+klekVSL0YYkhxLDVbu+XXPI/HvGRwCd8Gc3xLSIkYYohIsvlIvqerQHZQSuOnGueJ4Y8VbWGIIadp7ZigygOzTHL0FbIvU473092XLZQSEqyp31LE1jPyZgwx5LSzheWeroIseH1fuZQeKrwBsxIpEUMMOe1/KWc8XQWbeOz1rKaGQ2WeNBVZKSKvxRBDquCOX+K5RfJfSiHv5okGpF/2ZuOO97fipBdO0EjehyGGNKmsqkb6294TyfRv0gDI3wKg1kshzrwNJn02ZOkT417KbAWyz9Sv9+DguSL8+/t9nq6KIqn060hWMMSQ4shx/pi6ao8MS5GHmk6IPMBrR2m9IC8LFX2OyXswxJAmbTic1+TXqrXlRA5KPU8pZeZbpQRShbwdyv3AkNdgiCHFkfv47OnjrCtPONUGo+sWTk5z1b63J0xxtB15A4dCTHx8PG666SYEBwcjLCwMd999NzIyMkzKCCEwe/ZsREZGokWLFhg6dCgOHDhgUqayshJTp05F27ZtERQUhHHjxuHMGdMRLgUFBYiLi4NOp4NOp0NcXBwKCwubtpWkKpaOz+48HCvl13ZjzhSU4bqXE2VdplJPe0qtlxJY+7jKPd+RN8+fRMrlUIhJSkrC008/jR07dmDdunWoqanByJEjUVp6uRf8vHnzMH/+fCxevBi7d+9GREQERowYgeLiYqnM9OnTsXr1aiQkJGDbtm0oKSnB2LFjYTAYpDITJ05EWloaEhMTkZiYiLS0NMTFxcmwyeRttHoC/O+2kzAYlX9icWcoVPpwf1IL5X+vqJafI4UTE01/9S1btgxhYWFISUnB4MGDIYTAwoULMWvWLIwfPx4AsHz5coSHh2PVqlV44oknoNfrsXTpUqxYsQLDhw8HAKxcuRJRUVFYv349Ro0ahUOHDiExMRE7duxA//79AQBLlixBbGwsMjIycO2118qx7URkg9pmgq2sMeC57/a6dB1qekd4Ockyey7xqeyj79Wc6hOj1+sBAKGhoQCAzMxM5OTkYOTIkVKZwMBADBkyBNu3bwcApKSkoLq62qRMZGQkYmJipDLJycnQ6XRSgAGAAQMGQKfTSWUaqqysRFFRkck/UieLB5B6B55MB+e/4PHIO9hqlZqXeBgXS6vcWBtSMx4z1KPJIUYIgWeffRaDBg1CTEwMACAnJwcAEB4eblI2PDxcei4nJwcBAQFo3bq1zTJhYWFm6wwLC5PKNBQfHy/1n9HpdIiKimrqppHCfbDhqMn/5b6kopZfYWrpo9CUzq37zhTiq52n6i3EuTp8tPk4nndxK43WqeV7Qd6lySFmypQp2LdvH77++muz5xoOhxRCNDpEsmEZS+VtLWfmzJnQ6/XSv9OnT9uzGaRAjp70Pt96wjUV8XJynbPKqwyNF2pg3OI/MWfNYYcq09jlk5SsAofrYb4OIlKSJoWYqVOn4ueff8amTZvQsWNH6fGIiAgAMGstycvLk1pnIiIiUFVVhYKCAptlcnNzzdabn59v1spTJzAwECEhISb/yDvE/3648UIa5I4+DwajwJmCMqeW8d66I07X44QMU+jL0ZLg6CK03i+FjTPkaQ6FGCEEpkyZgh9++AEbN25Ely5dTJ7v0qULIiIisG7dOumxqqoqJCUlYeDAgQCAPn36wN/f36TMuXPnsH//fqlMbGws9Ho9du3aJZXZuXMn9Hq9VIY8a3qC62bEtXSy2XHiosvWp1ZyX046X1JptsSpX6di0Nub8Mve7CYvd1cm950nyD1HDQMLKZFDo5OefvpprFq1Cj/99BOCg4OlFhedTocWLVrAx8cH06dPx5w5cxAdHY3o6GjMmTMHLVu2xMSJE6Wyjz32GGbMmIE2bdogNDQUzz33HHr06CGNVurWrRtGjx6NSZMm4dNPPwUATJ48GWPHjuXIJIX4Ma3pJ7WmmPb1HozrFenWdcpFMbOr2rDlSD7+77+7zB5fk177Hf90y3Hc6cD7zxOe+1hrYdp3Ro93/8jAjJHXuGzGYzX2k9F665i3cSjEfPzxxwCAoUOHmjy+bNkyPProowCAF154AeXl5XjqqadQUFCA/v37Y+3atQgODpbKL1iwAH5+fpgwYQLKy8sxbNgwfPHFF2jWrJlU5quvvsK0adOkUUzjxo3D4sWLm7KNpDJrD1ruvF1HbUN/1VDdjzYf83QVnNbYeVqOz43aTn+LNx1Dz446jLw+wtNVURW1HWO8mUMhxp4d6+Pjg9mzZ2P27NlWyzRv3hyLFi3CokWLrJYJDQ3FypUrHakeacQPqWdlW9aBbD32ni6UbXlehcdxM2p8S3KLK2VZDk/spEQOhRgiJbDUNP5TmnnwMRgFxnywzR1VUj2en1xDDZcSnaGWYf6OUsoNR6lxvAEkacIzCWlmj9UYlXFzRC0cD7UQcjSwCU2jhZ1HZAVDDJGLueocwnOT61h7bzWQR2Wlxs+gFn5U0GUMMUTkFYoravDhpmOoMTS9hU6F52zZztqq3HYL7Ale7P+jHgwxpDr2HmCUMpSSv/yU450/MvD17sZn8+Y+sw9P9eRpDDFEDfBXWC2tdto8llvc5NcqMdtodT+5CgOqtjDEkOrYO3Lg3bUZLq6JdnjTadCbtlVOlrI9Az95GkMMadZnW3hjSG/ijvOpUk7ZWssOzvRTIu/GEENEFjlzolTyL3T7Ona6vh5uo/CNmflDOnrMXoscfYWnq0IqxBBDZIO+rBp/X2Z+TyFvo/DzoENW7DjV5F/+SulOoZR+HXJ8Lr7elYXyagOWJ590fmHkdRhiSHXc+Sv//Q1HsSkj323rI/fYevS8p6vgPrINsXbt904huYxUhiFGw8qrDCivMni6GrKpCy/uvIN2QVmV08tQRSuGGurYCEdOstXe1AdDFR9A97UuqeTtIDsxxGhUtcGIbq8kotsriZrpNPfst3vdsp7iyhq3rEdJ8kvkuUmgWvDeOGQLc456MMRo1In8Uunv4gptnJRX75Hv7tbWlFXVmLxfSu6g6mpevOmKV3/fuG0/WRxiLd/ilTI5JakLQ4xGbcrI83QVVKl++CPtOphdZPN5TiDnfkpqHFNQVagRDDFE9bD1Qb0c2Xfbj3tRx14X0mrY0+ZWaRNDjBdQ4hfybGE5sgvLPV0NMw0PyjtOXPRQTdzL0i9PJX5u5NLYtsl1acNVodgTrRau/jyw9YOaws/TFSDXUPIBoaLagJvnbgQAHH3rdvg3U06WbnjSKZWhk68WWnc0sAkmzumVF6C9npuSmVZbj7yVcs4e5DXqD1uuqHZsCLirO9qq6fDm7Hvxv5QzuGvxttqZUi2cP5QchJ11+qK2Qsz8dUcQv+aQ29crb8deIsexJYbcrv6Bz9GhrhlO3IHYHt40Gum572qHrL9l5eSntndCbfWVS2W1ER9sOAoA+PvNXTxcm6ZTUsdeUg+2xHiBE/klnq6CCWO9oODocctg9NZTlTm55jops+OSmaNrUvteUvolh/pZ21DvP66cxM/iXaxlXL67hlh70e8Ur8AQo1H1z28v/G+f5yrSCKX9+lLT8U3OVqPGdkP9NZVXGbyqxUrpuC/ImzHEeIHSKmVNdmdyOUlhV8K98XxgbZMtnRyP5haj2yuJmPGde2ZPtmTDoVzELd2JHH0FKqoNqKzRzq011IYBijyNIUajPDKjZxM0bInx/D1t5H+zXPX2L08+5aIlW7dk6wkAwA+pZ2E0Cuw/q3f7Pnts+V/YevQ8Jq/4C9e/+gf6vLEeQghFn1BrDEa8/ON+/LpPnvt+1f/euG/CXhffAFJZv2dIJRhiNMoTBzl72TrXlDcyWinrQpksdbB2P6mGdVPae1enTMbWNR803r/mTIH5+/7+hqMYu2gb/u2hy5X7zuhhMAqUVNagyuPh9zJLYerHtGys2HEKU1btcem63R0ElPr9cJaC8zA1wBBDbmfSsdfBg+4/v0qVpQ69XluLZxLMTyhqOXbVyNzBubHdYOn+W+9fGhHzg4V7WlXWuCZUWAufx/Mcv11EiY0OzXKdxOpaL/KLvesGm03BhhhqCoYYcpizzfb1X+2pPjGlVQb8lGbetN9w05R6YBUyZgRXBLdPNh93wVItByYAuOODrfh5r2OXasqrHO9Lk62vcPg1gPyXYjxxudibWid4aUs9GGLI7YQTLTGu1jCgKfW4bZTxjLLxcB4ulFaZPd5wDUUV1bC3AWjvmUKn62VJXpH1ELHCwT5CTfnsTbfQeteQJ0/2cg27t5esk90p6FjgTYFN7TjZnUa5soXD2S94/Zf7KunIpSJyH2MvWggxDfWcvVbmtdovwK/291ZGrvU5jxwNdk355HGaItdxW4uSe1ZDbsKWGI1qLBuUVtZgw6Fch6f9l4PpEGtlccUBzhWjZjLPO94HxF3eX38UW4+a3iXa2feg7nPyi4OXjGwu00UB2t0nSc9OzMdIQJ7FEKNRjZ0zpqxKxWPL/8Lrvx50fNlNrFMdOU9EclNLM/K/v3f9iKCmvBcHs4uwYP0Rs8df+emAc3Wxo8wRG600ltjT+qRUJvnLTT8KVPLVsOrk+VLsP6v3dDVIZgwxXsDSyWhTRj4AYNXOLDfX5vKoFsC8xWh1quWOm+6i9Onm65RYGC2kBIXlloPBih3OzWlTVWPE+oO5Ti2joQ/qfQ4bcuZToLT5alxZG4Vtqk1D392MsYu2ocpFI+fIMxhiNMqVXU1ceZB+9WfnfrE7reE8MTJsqyveLTk79lrTpM+QC6v1+Jd/ybo8T1xKlYvJ6KR6j7OLGXkbhhiS6MuqkXa6UHG/JN2p4Za7e7SHvdzRwbQpHwM1dXx11a5V2lsg12ZaOi64cluzLpTh8eW78dfJiy5cC6kdQ4wXKKqoxqINR3Hqgu3OoMPmb8bdH/6JzRn5uFhahW1Hz8t+4Jr6tekQVaWFBFfmt79OXsRbvx1s0vwk5pR2qlQfpd23y1XU+kl5alUK1h/Kw72fJHu6KqRgDDFeoKrGiPfWHcEd72+1We58SW1/hj8O5GDE/CQ8vHQnfkyTt4+Kkjv1Aq7tE3PvJ8lYsjUTH2465vSy3NNYZv9KhKi9j1Jjt43wBu5uyFTyDVWdcaag3NNVIBXgPDFepNSBFoC6yc/WHsjFPTd0NHlOy1eb3LFtJ847NorGEnf0iXHED6lnPXpn66Y42UjLpFrUD96WQrhLRyzJ+DFsuChPfsSV9e0iW9gSo3Jf7TyFgfEbcCzP+RNjHWu/pi+UVFq9d41WmB9IXXBXaxkW2dgijueX4Nd92W7r3+Ts6CNPOJxTbPU5NfULc1dLjIreEvIiDDEqte9MIUYt2IJZq/cjW1+Bl1an2/W6yprGW2Pq31Oo7sB1OKcIfd5cj/s+1fb16ayL8twl24QLDv7GRnrQDnsvCVNW7cHGw3lNXocjJ62004VNXo8nyXk38DruGKbvyF3qXTrEmm0W5GEMMSr1yH93ISP38i/Jhi0kb/52yOLrLN2N2Ja6g9Q7iRkAgD1ZhS47cOUVN+3menJ6+cf9Jv935BKcveToy2zvHth7pumTeymsz7XTJg/uavZYZbU6Wxbr75vGAm2NQcCgpmFjRA5giFEpR8NInZwm3oV3gxO/6BujL6+GwSjw3Heun4VWM+w8Jx3PL8HPe913WUnJLGUyVwQ1S3eY9uTb//qvB3Hbe5sVe2nUHfj51y527FWpmib+sqp2sE+Lpe++nMeDUxdKMeSdzYjUNUd2EwOWN7J3F/y27xx+23cOzf18MfL6CAfXou6mmIn9O+FgdpF0qcuvWePbU1ljQFmlukZY2fN9PHWhDJU1RjT3b+b6CikQM4x2sSVGhc4UNL3fhqPzsggAzyTsabTcih2ncLoJ/UlW76kdwu1NAUaOA2pJpWMtcU3ps+KpSxBy/Wqec08PtAy4fNKedEtXfPLwjSZlGnaEve3dJNzwxjrkFlXKUgdpPTLnwfr1tnekmitO5K4MB3K2njDDaBdDjAol7Drd5Nc6eixddzDXpKOvNS//uB8jFiQ5XJ8ag/YPL/9LOePpKpgoKK3C3R/+2Wi57ELPzNMh54lxzj09cHVYK7x7Xy9c0TIA/s1sH/LOXtrmP4+dt1nOUXKf7E36xNRbuCs72rI1g5SIIUaFnDlQ+bqwt2aFhU6SJ8+X4u/LdmG3lanDL5TK+4tXiXY12HZPd5hdtPGYXS0znpqLRs61Xtk2COufHYJ7+9TOdWS2SS7uE+MqVm5irWoN3zc5Z/NmnxjtYojRCHtnt5RlZIwDx4MnV6ZgU0Y+7rM6dbi6+100hSeOp/X3e6mdl6I8NaDFrSccDZzbLHUktljOBRvr2pYfXk6ixjHEqJCl77a9/Rfc3QpwtpFw9fWuLDfVxLs17WaO2j/0WzsJO/M9qb/MvGLzlsa8ogrc89Gf+F6my4zC5HISWeIFH2WvxRCjcPnFlXjj14M4mF2E4/m1s/Ja+j7a+x2VY0ZPR359ecOJ0BMC/Fz/1XVXi0jD9bjzE7P7ZIHFx+XK+m/8etDssbm/H8aerEKnbtNg2ifm8t+u3GeWvvdq+XpzUj7tYohRuGe/TcPSbZm444OtGPZeEv44kGNl2LMyW2I4x5ZrNDbBmSzrkHkVWRfKIIRAwq4sHMiunYTviz8zMSB+A/adKZRmk3blibGZr+kXYNKXfyHDxu0HXKHYwZFlll3eDp6gG6eWsEWOY4hRuIYdMBPsuPxScOnmjZb4+ACJ+885VSdHDghsiXENR99VfXm1w+uQe98NfmcT5q87ghd/SMeYD7YBAGb/chC5RZUYt/hPxMZvBODak/Kg6LZmj81flyHrOjx5F2t+3cjbMMQoXMOZeQWsNOvW+/uL7SetLm/b0fN4cmWqPJWzYtXOLNz+/lbkFlWgskad07q72tHcYiz7M9PhyQfrOBowvtrpeN8jV7T2LNp4zOpzFy+Fb0c27Z4bOji0fktDrP84kGs2x9EHNuqpBNaGWNvibMBRc0BytO4czaQeDDEqszkj3+LJxfTXmPUvoBw36mvs6/3S6nQcOleEub8fdnpdWjViwRa89stBfPHnSYvPL1h3BMPe2wx9meUWFGeOsfa2dLj6MP70KufC9HMjr8GC+3tj1aT+Ttfly+STDpWvNhitfs8sPerKy7iebImRc30NP5eMEWQPhhgVcuZOy3IcGJ771nqHxI83H5f+rpuNl6x7a43lG3W+v+EojueXYtn2TLPncouaNrvxtK8bn3m5PlutJnL4bZ/5Zc3afjP2vf7qsGAAwMCrzC8RvTK2u0N1ceRkXFZVg+hZv6PLzDUot/MGoXJ0qLeGjQaNc/QSpZxz1JBrMcSo0B8Hcm0+b/PrKsMBL/FAjtXn3k5k64slhWXW+ynZUn9G4xqDEfqy6ibf/PPnvdnYeNj2Z8fTBr+zya7+O2/cdT1GXR9u9Xl/G/dJqn8rgjonL5TZfWlv69HLs/l+tuWE2fOWWmh8XdkSU+9LfabQ+g8cZ7/6tl4vhPx3ypZzYk5eTtIuhhiNqP+lS9ht/bYE7GjrGb1fX2f1OXsPmHd/9Cd6vb62SfeoqpN6qlDxv9zv+3R7o2XiYq+0/WvZxnPPjbzW7LH1h3Lx8Oc77apfffbemqF+deTOM/X35+yfD8i89EbWfSnaPPVVKrq9koiyKvsCdlWNEROX7MD8tdY7VcvZGKLwjzw5gSFGg/ItTLBVx9oJrKktBeQ8W5dtVuw4Jf29/2wRAOCXvY3fy8qaxZuO4TuF3cupodMXnb9nk63zX6fQlhYf35lp+dYYDT2xIkX629fCEdRinxgXXk56f8NR6e9SG3fgdmXrwu/7c1BVY8TMH9LtKr/2YA62H79gswO1nO+Yo9tev7WNlI0hRiPs/YoaLHyZv9192mZLgVw8dVdkd0g5ZXnSNEs2Z+Sb/H/+uiNWy+rLq/FmgwnTLO1Db/LWPTGNlrH1K76ZjUtNjiqzt0+MCy8nbTyc57qFN6LhR9Gem8UCsOvSnaz3TpJtSaQ0DDFucjinCNuP2073W47kY9bqdJRXGWAwCvznR/t+1QDmQ7GtsfSL5IXv99m9Hme86Kb1eMLfPm78Ekid8mr7Tnx1Pt9m2rlXy2HQHvf3jWq0TNtWgVafs9YS0xTn9OadrC1lTHtPyEdyi/Hi9/uku2nLyZ5Pja1wIWdLjj39XeqXWLotE9/auEzeGC/P/Zrm5+kKeIvRC7cCALa+cCsAYNmfJ3FX70ikZhVg/I0doWvhj//77y4Atb/Unxt1DVbukP++Qp48/yn9MoYn6cuq8ceBHPSM0uG6iBCz59+r13cg+fgFd1ZNUd5/oDf8LMz10tCIbtY7/V7VrpVs9dll5yWo+h171x4071xtNNb2Lhm3eBsqqo04kF2EX6YOsrq8pgSKxl7y9a4szPwhHZ//X18M727+/hVYuOTsSC2EEHgmIQ3N/X0xKLqd2fNF5aY/xC7Um7Sz7vYN9/bpCN+m9JJmiNEstsQ00baj53Hvx9txNNfylOV5xRWI//0QTl0oNXk883wpHlyyA//9MxN3ffgnXvvlIJ5rcA+Vs4Xl+Nc3TbuvSmPNtPvOFDZpueRaEz5Nxgvf78PohVtx94d/mj1fv9/MBRszMmvdqOsj7CrXpBNdE/Wfsx7dXk7E3N8Po8JCK5vRKKz276g2GPFD6hl0fWkNhr23GRXVtd/f9LN6m+u09jW31YIz7es9uPLF36x2Rq7rz/LkyhSLz3+ZfMri4/bKLarEz3uz8e1fZ1Bm4dYL//3TtMWxdUt/szL2ZJE16edwMLu2/9i6g7mYl3i40QENpy6Umn3v6k8XQcrFENNEDy/dib9OFWDEgi1mE2VtPZqPfm9twKdJJ/C3j5NNnjMKgTMN7uy87mAuPt9qPlTTUZnnSxE963ebZc6XeO8JUMky6oVhOSYk1CpLM+42ZGvotSvkFlWivNqAT5KO46HPd+LEpRu11nl6VarVk2/0rN/x7KV5l05eaHzUmRACxRXV2JnpeGtc0pHavlgD5260Wa7+pZ76o42qLMy+7UiLUP2+XNZahOsvr+F9roDaS+5myzUKnDxf+2Nx98mLeOqrVNzxQW3L96Qv/8JHm4/jDxvTQqRmFWDIO5vNvnd100WUVNbg3T8ypGBEysIQ0wQNP8yv/HQApZU1OJFfghqDEXFLd0nPnS+pxN+XXf6/tVls3/zN8qRnjrj13c1OL4NIySyd2ADTyzWxXdu4qTbmUk4V4J6PTPtH/b4/B6/9Yn4368boy6vNQsLfv9iNHrPX4pWf5B1KvfXo5XBQZTAi60IZ7nh/K7q/8oc0bNtSN5Z7PtqOjzabjjBqOAVAjcEIQ4PWKGstI41lIksd6Kd/k4ah727GN7uzcNjKzTzzbIzYnLT8L5vrfCfxMBZvOiYFI1IWhhgHCSEsfpiHvLMZt72XhNvfN39uU73RKNa+ZERk21//GW71uR0vDZP+vteOjr9q0Ou1tZj1436TxxqObJPLp0mmLcGD39mEg+dqf6x9sf0kSitrLHZiBoB5iaZzvdwyb5P0t8EoMOSdzRj23ma8v/7yUHD7btlgnpoWbzqGvacLYTAKpJ/Ro8ZglKYc+GjzcauX7RL3W26JMRiFzfu75RVXYD9bYBSNHXsdZK0Z9HxJbdI/mldiuQARNdmvUwfZHHEUFtwcx+fcYbWlRq1W7cxCwq4sLH30JvjJvG0n8ktw23tJeGroVdh2zPbIyetf/aNJ68gpqpD66dS/XLbDSofo3Scvot+Voag2Gq0OS7/rwz/x1NCr8NHm4/jbjR2lx4WwPpS9LpA19MeBHJv9Zfq9tcHssS1H8hH/+2G8c29PxHTQWX1tUxSWVeF4fglOXShDTAcdrgkPlnX5WsQQ4yBtHSKJ1MGek4UjAebqsFY4ppIfHEYB/H3ZblmXeeWLv0l/f+TCDqxnCyx3IrZ03ywAeOCzHXYtt67O36daH/Foz+0r8osr7Z7rB6i9NFY3ivTvX+zG7lnWWwcdlXm+1KxLwMm5Y2RbvlbxchIRyWbGiGtkX2b8+B6yL7NVIH+/uVLCrtrpISZ8mtxISflkXSzDifzLo0F7vba20de86uBtGq6uN3CiuMI8JH2adFyaZbu8yoD0M3rpvlKVNQZ8sOEo9jboQJyjr8DohVss9mlce6lDcm5RhdnrqBa/yQ4qtfPeIETeZkDXUEwdFo17buyAQW9vavwFdnrgJvn7uGjsqpPivPhDuktnKbZmaYOJIV2pbjg8UDt1xbjFl4doj7+hAx5csgP7zujRJijAZFqEuhm6d700DM0DmmHRxqNW+0pOXpGCO3tFSv1+1ky7Bd0jzeeRsuaLPzMR2ioQ43pFIq+oAtuOncfYnpEI8PNFeZUBzf19LU7EeL6kEm2CAuDj4wMhBHx8fPDhpmMIae6HuNgr7V6/OzDEOIjDX4ksW/ZoPwBAx9byzYgb6Gf5IOssOe+QTJb9+3v7ZxxXq/MllThbUI67Gswx884fGdh3pnauH2vzOvWbY97fxpL690pbsP4IRnQLxx092zfamnjyfClmXxoVJ4TA89/tQ5XBiMzzpbivTxQGv7MJPTvq8POUQXjz14OIDm+F+2/qhK1H8xG3dBfu6h2Jf4++zmxI/idJJzD1tqtx89Vt0V7X3K7JJ13JR2j0nuNFRUXQ6XTQ6/UICbE/uTambgcT0WVd2wVh44yh0v/r97lwxtG3brdrbhhHLVx/BAvrjZaxZmT3cIsz7BJ5UqfQlrj12nZ4eEBnRF/q/FtSWYPtx84j8UAOfkg9i6m3XW3z5rJycUW/HUfO32yJcZCrhjgSqdnYHu1lX6YrRxv9c+hVaOHfDGvSz2HvGeuz4748tjtDDClO1sUyLE8+heXJp6QQEdNgBJk7AgwA/JR2Fnf17uCWdVmi+I69H330Ebp06YLmzZujT58+2LrVsxMOyT3MkUgLXHHJx5XDpQP9muGJIVfhpymD0LOj5ZFPPzw1EFGhLfHdk7EuqweRHPJtTObnas8kpHls3YDCQ8w333yD6dOnY9asWdizZw9uueUW3H777cjKkv/GiPbya8YQQ9TQPTc49kvs5Nwx6NxGvr4zzlj5eH/8Z0w3+Nf7bk/o2xE3dmoNALjpylCTJvNAP0UfNskL1c1T5o0U/W2cP38+HnvsMTz++OPo1q0bFi5ciKioKHz88cceq5Ofr6LfMiK32DhjiPT3wKva4Mq2QSbPW2qxvPXa2jsXXxNeexfpb5+IxZRbr8aUW682KzttWLSc1bUppLk/Hr+lK9Y/e3mbJliY9ffNu2Mw9Np2SHtlJMbf6Lnmc6L6DucU4YiVGxF7A8X2iamqqkJKSgpefPFFk8dHjhyJ7du3m5WvrKxEZeXlNFpU5Jqponk5SZ2mDYvGBxsa78hJ1lnrwGdpJt1fpg7Cx5uP42heCQ5dmi112d/7QV9ejaCAZgCA8JDmeG7UtQCA5cknUVxRO33B5//XF8O7u/cmjgDQuU0Qlj16E4xCoO+VoWbPPzygMx4e0BkA8O69vbD3dCGO55ealbPlwX6d8PUu05bk5v6+JsN1HXF/3yi8fW9PPPbFbmw4nIfpw6Pt6rBM2jF6oXff00mxzQrnz5+HwWBAeLjpwSw8PBw5Oeb3wYiPj4dOp5P+RUW55v4pZdX2z+5IyvHsiGvw/T8HIizY+tT13m7tvwbjp6dvNrmsYo/Yq8xvuNitfQg+ePAGTLqlCwCgdUt/AICuhb/FIZkLJvQGAPxnTDePBJg6t14XhmHdGl+/r68PEqcPxpBr2mGMhU7N/6vXj+aTh/vg9pgI3NEjAnPuiZEe79FBh+SZt+HwG7fjs7g+Jq+3NHz2xduvw4HXRpk89vzo2hD4SVwfrPvXYDwzLBppr4xAvy6hmH1nd8y7t2ej29KYRwdeiXfu7Ylbotti0NVtsXHGEDw55CqnlwsA/bqESi107jDo6rYAgCHXuG+drta2VaDNW3JonWKHWGdnZ6NDhw7Yvn07YmMvHxDeeustrFixAocPm94N2lJLTFRUlOxDrON/P2R2szRyv5Dmfvh5yiAMtePO3ddFBCNx+mAAwH9+TMfKHZ7rU+Uqv04dhMoaA/72seUZUr99ItZs9tRbotti69Hae+bsfXUkdC1qg8YHG45i/rojGNA1FH+7sSOe/98+/Hv0dfjnUNMT1+mLZUjNKsDYnpFWO+EajQJ/Hj+P7u1D0KaRA21FtQHN/ZvZtb1Ks3RbJt74tXZOjtVPDcQNnVojr7gC/r6+aB0UYFL2o83HkHWhDPHje5h0iJ61Oh1f7czC5MFd8dId3TB43iZkXbojdP1WsEPnijDh02QsfeQm9Oti3mLU0MQlO7D9+AWzxxc9eAP2Z+tNjmdbX7gVYSGBCPRrhrOF5ci6UGYxpAJAZY0Bx/JKMOaDbejVUWcyyqvh523FY/0Qt3QXurUPwfJ/3ISWAX5o6d8Mvpc+N3VD8juFtsT7D/RG76gr4OPjYzJUf/T1EUg8kIMJfTvi279Mbzew9l+DMXLBFgCmrV2xXdvgtuvCYBAC4SGBuLt3BxzOKcY14cE4dK4IYxdtk5ZxV+9ITLn1arydmIF+XVrj8UFdUVxRg16vNz7zLwB8/8+B6No2CLtPXkTLAD88vHSnyfNXtQuyq+Vu8cQbcGOn1mbzs1hT99mQa1qD+uy5Pcegq9ti5eP9ZV2vI0OsFRtiqqqq0LJlS3z33Xe45557pMefeeYZpKWlISkpyebrXTVPTPLxC3hwiX339yDXWXh/b9x9Qwfc9eGfJtNxD+8WjvWHTIfEDugaioTJtUHYUog59PpovJ14GF9sP9noekOa+6Googbtdc2t3tW3zo2drkBqVqHNMnKYcuvV0mWZhz7fgT+PmZ+wjs+5A1e9tAYAENzcD8UVNUh9eQR+3HMW/bqEmtybqMZgxO6TBegVpUPLAD8UVVQjpLm/y7dD7SqqDcgrqkSnJnZYFkLgQmmV9Kv6/fVHsWD9Efj4AJnxTZ+Lo6C0Cj+mnUXPjjop5F7ZpiU2P38rhBA4X1KFoopq+ADo2q5Vk9ax++RF3PfJ5dBycu4Y/HXyIj7YeAyv3tkdV7VrBYNRwNfH8ki21KwC/LbvHP49+joE1Os4/e4fGVi86RgeG9QFL4/tLs0eO+3rPThxvgTz/tYLHa5oAd2llr4agxF+zXyxJv0cPt1yAoseuMHm/kjNKsA7iRl4eWx3qzPhVlQbcN3LiTa3v7m/Lw6/cbvZ43XBokvbIGx6bijO6cvx275z6NO5NTpc0QL95mzA6OsjsOVoPsqqDNj50jCEhzQ3WcbRS/1djAKYvOIvnKp3I82EyQMwoGttyJyz5hA+21IbSIdc0w4v3n4djELg611Z0jHv0YFXokVAM3y8+TievvUqVFQbpVmOf506CKFBAYi8ogVO5JegotoovScN34NZd3TDW2sO4ear22DlY/1lH52oiRADAP3790efPn3w0UcfSY91794dd911F+Lj422+1lUh5lheCYbPtx2gXCUqtAVOX7R8QzVPmX1nd+QUVeKTJPluIjf+hg6oqDGguKJGailoqC7EHM0txohLv8AAYOkjffHY8r9Myj4xuCtm3tENgOUQc/iN0Qj080X6WT2uCQ/GifxSbDyci8dv6Wp28Fr3r8H4cNMxTB0WjVMXSpFfXGl1ZtJDr49Gt1esH/xGXR+OT+P64qXV6Vi103LrUGb8Hcg8X4rb3rP8mbslui1WPHb5V5DRKFBUUY3f9+dg5g+19doxcxgidM1x8nwpCsur0aODDpU1BrQMUGyXOEJtS8e3f53B4Oi26NwmqPEX2LnMxP05GHhVW7ST+dLqA58lY8eJ2rtTyzUBmhACx/NL0LVtK6nVpv5zrhjab83Xu7Kk71RDv00bhOsjzYfqF1VUo6C0Ch2uaGHxMmpd6LJXjcGIX/Zlo2/nUAQF+iG0QStfYVkV8osrpQnw6vxxIAct/JthcIPLaEIIvPHrIYSHBOKJRi4Rni+pxK3vbsZV7Vph9VMDXfrea2ayu2effRZxcXHo27cvYmNj8dlnnyErKwtPPvmkx+rkyrkrGvPN5Fg8k7AHu08WeKwO9fXvEopHb67t8zCmR3vcuXhbI68wNe22q/FBgwmZ5v2tJybUu1fOKz/tx5fJp8xe26dz7fDXhl/W264Lk/4e0DUUg65ui8cGdbVZj4BmtVPb9+x4BQCge2SI1V9l0eHBWPjADQCAqy79ap3QNwrFlTXoOdu02blFgPVLI79OHYRrLtV9zj098MrY7iaBKSw4EJufHwofHx90bdcKG2YMwbBLQaZdcKDVeSF8fX1wRcsAPNivE9q1CkRJZQ0idLW/7OqPIGKAUb5Av2aIu9SRWM5lumpisoTJsSivMtj83DvKx8cHV4cFW33One7vG4UubYPw7V+n8UPqWQDAgddGwb+Zr0nrUX0hzf1ttmI6OmW/XzNf3HNDR6vPX9EyAFe0DDB7fNT1ERbL+/j44JU7u9u17ratArH3lZHwsdKa5imK7dgLAPfffz8WLlyI119/Hb1798aWLVuwZs0adO4s7xdbDe7r0xGRV7TAd08ObNLr3/6bc3cCPvLm7RjfYC6QhMkDpL97dNRh7ysjbS7j+VHXSiHw/Qd649mR1+Ld+3rh0YFXSp1JB15tev391TuvR+L0WzDz9uukx7Y8fyuiQi83ET8+qIv0d/0vV48OOky5LdrqQXXf7JE48Noos1949U0efDkAWeuA6OPjg5Dm/nj3vl5Wl/Pj0zeb/D+mg87kwNewL8h7E3qZBI2r2rVC0vND8cBNUfhm8gBEh9UGKFsnpOHdw3G3g/O3EDlDzgCjNL6+PhjQtQ3euCsGL4y+FhtnDEFQoJ/VAKNFvr4+igowgMIvJznDVZeTLpRUos+b62Vbnr3qN8862oErpLkf9s0eZXIpxdKdUW+euxFnC8vR4YoWOFtoetnq5NwxKCyrQu/X11msUx2DUaCqxohXftqP71Iud76b2L8T5txTG6SqaoxmX/ziimoUVdSgwxUtLG6DwSjw+i8H0K9LG4zpaT4aZFNGHq5u1wpRoS2xZMsJ/LDnLFY93t+sU+XLP+7Hih2nrNbfkhqDEeln9egeGYJAP9sH6fpNzifnjkH6GT0Ky6twS3Q7/LI3G1O/3oM59/TAxP6dzF67OSMPa9LPYcqt0Y32qyiprEFGTjFu7HSF4g4qRETO0EyfGGe4KsQA9oWIzm1amnTActS79/XCuF6R2Jl5AdFhwdIlAQBYsO4I3t9wFG/eHYPNGflmHVnrpL0yAofOFePGzlcg0K8Z8oorMG7Rnxh8TVvMu7eXtB0HXhuFoEA/FJZVYf/ZIgy8qg22HjuP//yYjoBmvnjpjm7SsNOKagNe/nE/Rl4fgRE2hsLmFVXg//67Cw/176SoW7c3JcQ4wmgU+D71DHpHXWF2qQuo7ZPQWBAiIvJmDDHwfIiJCm2BkOb+OJBdO9FXTIcQvH5XDP48eh7jekdiyDubzV7To4MOY3u2x/7sIiy8v7fN/jcXSirRplUgJn35F9bVu0Hdsr/fhL8v2w0ASJ89EsE2rseWVNagqsZo1jlMy1wdYoiIyDma6dirVD9PuRnjFv9ps0z7kBb4799vku4s+uvUWwBAuh/LxhlD8OOesyYdW3+ZOsjuOtTNuVE/5nz80I24LuLyr//G0mmrQD/Ay+ZI6tHB8s3+iIhIfRhimqBnxyuw/tnBaNequclESG//rQeCm/vjh9SzmD2uO1oF+mHvKyPh72feotK1XSs8O/JaHDxXhPWH8kzu2+KIR2++EmvrtcTUTVgGAM152cLM3/p0RGWNweK08kREpC4MMU1UN+wvpkMI9p8twi3RbXH/TbWdNe+oNw153SRM1iz5v74wiqYP3R54VVtcHdYK2YXluOWadmgZ4Idfpw5CM18fr+o1b69mvj6K6qNDRERNxz4xTjpfUok16edwV+8OJq0g7iSEQI1RwN/BOQeIiIiUhn1i3Khtq0D8n4d/2fv4+Dh80z4iIiK14093IiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiXN3sVaCAGg9pbeREREpA515+2687gtmg0xxcXFAICoqCgP14SIiIgcVVxcDJ1OZ7OMj7An6qiQ0WhEdnY2goOD4ePjI+uyi4qKEBUVhdOnTyMkJETWZSsBt0/9tL6NWt8+QPvbyO1TP1dtoxACxcXFiIyMhK+v7V4vmm2J8fX1RceOHV26jpCQEM1+OAFunxZofRu1vn2A9reR26d+rtjGxlpg6rBjLxEREakSQwwRERGpEkNMEwQGBuLVV19FYGCgp6viEtw+9dP6Nmp9+wDtbyO3T/2UsI2a7dhLRERE2saWGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgHffTRR+jSpQuaN2+OPn36YOvWrZ6ukpnZs2fDx8fH5F9ERIT0vBACs2fPRmRkJFq0aIGhQ4fiwIEDJsuorKzE1KlT0bZtWwQFBWHcuHE4c+aMSZmCggLExcVBp9NBp9MhLi4OhYWFLtmmLVu24M4770RkZCR8fHzw448/mjzvzm3KysrCnXfeiaCgILRt2xbTpk1DVVWVS7fv0UcfNdunAwYMUM32xcfH46abbkJwcDDCwsJw9913IyMjw6SMmvehPdun9n348ccfo2fPntLEZrGxsfj999+l59W8/+zZPrXvv4bi4+Ph4+OD6dOnS4+pch8KsltCQoLw9/cXS5YsEQcPHhTPPPOMCAoKEqdOnfJ01Uy8+uqr4vrrrxfnzp2T/uXl5UnPz507VwQHB4vvv/9epKeni/vvv1+0b99eFBUVSWWefPJJ0aFDB7Fu3TqRmpoqbr31VtGrVy9RU1MjlRk9erSIiYkR27dvF9u3bxcxMTFi7NixLtmmNWvWiFmzZonvv/9eABCrV682ed5d21RTUyNiYmLErbfeKlJTU8W6detEZGSkmDJliku375FHHhGjR4822acXLlwwKaPk7Rs1apRYtmyZ2L9/v0hLSxNjxowRnTp1EiUlJVIZNe9De7ZP7fvw559/Fr/99pvIyMgQGRkZ4qWXXhL+/v5i//79Qgh17z97tk/t+6++Xbt2iSuvvFL07NlTPPPMM9LjatyHDDEO6Nevn3jyySdNHrvuuuvEiy++6KEaWfbqq6+KXr16WXzOaDSKiIgIMXfuXOmxiooKodPpxCeffCKEEKKwsFD4+/uLhIQEqczZs2eFr6+vSExMFEIIcfDgQQFA7NixQyqTnJwsAIjDhw+7YKsua3iSd+c2rVmzRvj6+oqzZ89KZb7++msRGBgo9Hq9S7ZPiNoD6F133WX1NWraPiGEyMvLEwBEUlKSEEJ7+7Dh9gmhvX0ohBCtW7cWn3/+ueb2X8PtE0I7+6+4uFhER0eLdevWiSFDhkghRq37kJeT7FRVVYWUlBSMHDnS5PGRI0di+/btHqqVdUePHkVkZCS6dOmCBx54ACdOnAAAZGZmIicnx2Q7AgMDMWTIEGk7UlJSUF1dbVImMjISMTExUpnk5GTodDr0799fKjNgwADodDq3vx/u3Kbk5GTExMQgMjJSKjNq1ChUVlYiJSXFpdu5efNmhIWF4ZprrsGkSZOQl5cnPae27dPr9QCA0NBQANrbhw23r45W9qHBYEBCQgJKS0sRGxuruf3XcPvqaGH/Pf300xgzZgyGDx9u8rha96FmbwApt/Pnz8NgMCA8PNzk8fDwcOTk5HioVpb1798fX375Ja655hrk5ubizTffxMCBA3HgwAGprpa249SpUwCAnJwcBAQEoHXr1mZl6l6fk5ODsLAws3WHhYW5/f1w5zbl5OSYrad169YICAhw6XbffvvtuO+++9C5c2dkZmbi5Zdfxm233YaUlBQEBgaqavuEEHj22WcxaNAgxMTESOutq2/D+qttH1raPkAb+zA9PR2xsbGoqKhAq1atsHr1anTv3l06Oal9/1nbPkAb+y8hIQGpqanYvXu32XNq/Q4yxDjIx8fH5P9CCLPHPO3222+X/u7RowdiY2Nx1VVXYfny5VJHtKZsR8Mylsp78v1w1zZ5Yrvvv/9+6e+YmBj07dsXnTt3xm+//Ybx48dbfZ0St2/KlCnYt28ftm3bZvacFvahte3Twj689tprkZaWhsLCQnz//fd45JFHkJSUZHW9att/1rave/fuqt9/p0+fxjPPPIO1a9eiefPmVsupbR/ycpKd2rZti2bNmpmlxLy8PLNEqTRBQUHo0aMHjh49Ko1SsrUdERERqKqqQkFBgc0yubm5ZuvKz893+/vhzm2KiIgwW09BQQGqq6vdut3t27dH586dcfToUaleati+qVOn4ueff8amTZvQsWNH6XGt7ENr22eJGvdhQEAArr76avTt2xfx8fHo1asX3n//fc3sP2vbZ4na9l9KSgry8vLQp08f+Pn5wc/PD0lJSfjggw/g5+cnLVt1+9ChHjRerl+/fuKf//ynyWPdunVTXMfehioqKkSHDh3Ea6+9JnXeevvtt6XnKysrLXbe+uabb6Qy2dnZFjtv7dy5UyqzY8cOj3bsdcc21XVIy87OlsokJCS4vGNvQ+fPnxeBgYFi+fLlqtg+o9Eonn76aREZGSmOHDli8Xk178PGts8Ste1DS2677TbxyCOPqH7/NbZ9lqht/xUVFYn09HSTf3379hUPP/ywSE9PV+0+ZIhxQN0Q66VLl4qDBw+K6dOni6CgIHHy5ElPV83EjBkzxObNm8WJEyfEjh07xNixY0VwcLBUz7lz5wqdTid++OEHkZ6eLh588EGLw+g6duwo1q9fL1JTU8Vtt91mcRhdz549RXJyskhOThY9evRw2RDr4uJisWfPHrFnzx4BQMyfP1/s2bNHGt7urm2qGxo4bNgwkZqaKtavXy86duzo9PBHW9tXXFwsZsyYIbZv3y4yMzPFpk2bRGxsrOjQoYNqtu+f//yn0Ol0YvPmzSZDVMvKyqQyat6HjW2fFvbhzJkzxZYtW0RmZqbYt2+feOmll4Svr69Yu3atEELd+6+x7dPC/rOk/ugkIdS5DxliHPThhx+Kzp07i4CAAHHjjTeaDKFUirqx/f7+/iIyMlKMHz9eHDhwQHreaDSKV199VURERIjAwEAxePBgkZ6ebrKM8vJyMWXKFBEaGipatGghxo4dK7KyskzKXLhwQTz00EMiODhYBAcHi4ceekgUFBS4ZJs2bdokAJj9q/uV5M5tOnXqlBgzZoxo0aKFCA0NFVOmTBEVFRUu276ysjIxcuRI0a5dO+Hv7y86deokHnnkEbO6K3n7LG0bALFs2TKpjJr3YWPbp4V9+I9//EM69rVr104MGzZMCjBCqHv/NbZ9Wth/ljQMMWrchz5CCOHYBSgiIiIiz2PHXiIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUiWGGCIiIlIlhhgiIiJSJYYYIiIiUqX/B/YZdzKoUzMsAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_test.shape\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(X_test.iloc[1,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO: Successfully saved requirements file in c:\\Users\\kubov\\Documents\\git_projects\\similarity_metrics\\requirements.txt\n"
     ]
    }
   ],
   "source": [
    "# !pip install pipreqs\n",
    "import pipreqs\n",
    "!python -m  pipreqs.pipreqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/prediction_engine.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/prediction_engine.py\n",
    "\"\"\"\n",
    "Contains functionality for creating PyTorch DataLoaders for \n",
    "LIBS benchmark classification dataset.\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from load_libs_data import load_contest_test_dataset, load_contest_train_dataset\n",
    "from sklearn.preprocessing import Normalizer, MinMaxScaler\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def create_dataloaders(\n",
    "    test_dir: str, \n",
    "    test_labels_dir: str, \n",
    "    batch_size: int, \n",
    "    device: torch.device,\n",
    "    pred_test: bool,\n",
    "    ):\n",
    "    \"\"\"Creates training and validation DataLoaders.\n",
    "\n",
    "    Takes in a training directory directory path and split the data\n",
    "    to train/validation. After, it turns them into PyTorch Datasets and \n",
    "    then into PyTorch DataLoaders.\n",
    "\n",
    "    Args:\n",
    "    train_dir: Path to training directory.\n",
    "    batch_size: Number of samples per batch in each of the DataLoaders.\n",
    "    num_workers: An integer for number of workers per DataLoader.\n",
    "\n",
    "    Returns:\n",
    "    A tuple of (train_dataloader, test_dataloader, class_labels).\n",
    "    Example usage:\n",
    "        train_dataloader, test_dataloader, class_labels, wavelengths = \\\n",
    "        = create_dataloaders(train_dir=path/to/train_dir,\n",
    "                                test_dir=path/to/test_dir,\n",
    "                                transform=some_transform,\n",
    "                                batch_size=32,\n",
    "                                num_workers=4)\n",
    "    \"\"\"\n",
    "\n",
    "    if pred_test:\n",
    "        X_test = load_contest_test_dataset(test_dir)\n",
    "        y_test = np.loadtxt(test_labels_dir, delimiter = ',')\n",
    "    else: # use with caution, only for predicting training embeddings\n",
    "        X_test, y_test, _ = load_contest_train_dataset(test_dir)\n",
    "\n",
    "    if True:\n",
    "      scaler =  Normalizer(norm = 'max')\n",
    "      X_test = scaler.fit_transform(X_test)\n",
    "\n",
    "    # Convert data to torch tensors\n",
    "    X_test = torch.from_numpy(X_test).unsqueeze(1).float() # Add extra dimension for channels\n",
    "    y_test = torch.from_numpy(np.array(y_test)).long()\n",
    "\n",
    "\n",
    "    #device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    # If available, move data to the GPU\n",
    "    X_test.to(device)\n",
    "    y_test.to(device)\n",
    "\n",
    "\n",
    "\n",
    "    # Create PyTorch DataLoader objects for the training and validation sets\n",
    "    pred_test_loader = DataLoader(X_test, batch_size=batch_size)\n",
    "\n",
    "\n",
    "    return pred_test_loader, y_test\n",
    "\n",
    "def predict_test(\n",
    "                model: torch.nn.Module, \n",
    "                dataloader: torch.utils.data.DataLoader,\n",
    "                device: torch.device,\n",
    "                test_dir: str, \n",
    "                test_labels_dir: str,\n",
    "                batch_size: int,\n",
    "                y_test\n",
    "                ):\n",
    "    X_test_pred=[]\n",
    "    with torch.no_grad():\n",
    "        for data in dataloader:\n",
    "            input = data.to(device)\n",
    "            output = (model.forward_once(input)).cpu()\n",
    "            output = np.array(output)\n",
    "            X_test_pred.append(output)\n",
    "    X_test_pred = np.concatenate(X_test_pred, axis = 0)\n",
    "    return X_test_pred\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/prediction.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/prediction.py\n",
    "\n",
    "import torch\n",
    "import prediction_engine\n",
    "import siamese_net\n",
    "import numpy as np\n",
    "\n",
    "NUM_EPOCHS = 50\n",
    "BATCH_SIZE = 128\n",
    "LEARNING_RATE = 0.0001\n",
    "INPUT_SIZE = 40000\n",
    "OUTPUT_SIZE = 12\n",
    "CHANNELS=50\n",
    "KERNEL_SIZES=[50, 10]\n",
    "STRIDES=[2, 2]\n",
    "PADDINGS=[1, 1]\n",
    "HIDDEN_SIZES=[256]\n",
    "\n",
    "# Setup directories\n",
    "#test_dir = \"datasets/contest_TEST.h5\"\n",
    "test_labels_dir = \"datasets/test_labels.csv\"\n",
    "model_dir = 'models/modularity_test1.pth'\n",
    "test_dir = \"datasets/contest_TRAIN.h5\"\n",
    "\n",
    "\n",
    "# Setup target device\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "# Create DataLoaders with help from data_setup.py\n",
    "test_dataloader, y_test = prediction_engine.create_dataloaders(\n",
    "    test_dir=test_dir,\n",
    "    test_labels_dir=test_labels_dir,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    device = device,\n",
    "    pred_test = True # USE WITH CAUTION, turn to 'False' if you want to get embeddings of the training data\n",
    ")\n",
    "\n",
    "\n",
    "saved_state_dict = torch.load(model_dir, map_location=torch.device('cpu'))\n",
    "\n",
    "# Create a new instance of your model\n",
    "model = siamese_net.SiameseNetwork(\n",
    "    input_size=INPUT_SIZE, \n",
    "    output_size=OUTPUT_SIZE, \n",
    "    channels=CHANNELS, \n",
    "    kernel_sizes=KERNEL_SIZES, \n",
    "    strides=STRIDES, \n",
    "    paddings=PADDINGS, \n",
    "    hidden_sizes=HIDDEN_SIZES\n",
    ").to(device)\n",
    "# Load the saved state into the new model instance\n",
    "model.load_state_dict(saved_state_dict)\n",
    "\n",
    "#todo save this to a file\n",
    "prediction_X_test = prediction_engine.predict_test(\n",
    "                    model=model, \n",
    "                    dataloader=test_dataloader,\n",
    "                    device=device,\n",
    "                    test_dir=test_dir, \n",
    "                    test_labels_dir=test_labels_dir,\n",
    "                    batch_size=BATCH_SIZE,\n",
    "                    y_test=y_test\n",
    "                    )\n",
    "\n",
    "\n",
    "np.save('datasets/prediction_X_test2.npy', prediction_X_test)        \n",
    "np.save('datasets/y_test.npy', y_test)          \n",
    "\n",
    "#https://colab.research.google.com/drive/15D5vAYkhbAs5-txhYTCb_Fp2jiCnHXVN#scrollTo=82F_qINOBbkL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 12)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "test = np.load('datasets/prediction_X_test.npy')\n",
    "test.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "376601425dfe99d085b94bbaa5bb028434fb406f20f17f98957b701f4150938c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}