Multi_Horizon_Quantile_Recurrent_Forecaster.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wUZTZP_DDOfm",
        "outputId": "e002528d-84e1-4c72-a384-6fe52009be96"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from tqdm import trange, tqdm\n",
        "\n",
        "from io import BytesIO\n",
        "from urllib.request import urlopen\n",
        "from zipfile import ZipFile\n",
        "\n",
        "from pandas import read_csv\n",
        "from scipy import stats\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "import torch.nn as nn\n",
        "\n",
        "from torch.utils.data import DataLoader\n",
        "from tqdm import tqdm, trange\n"
      ],
      "metadata": {
        "id": "aepWjup9DrqP"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "name = 'LD2011_2014.txt'\n",
        "save_name = 'elect'\n",
        "num_covariates = 3\n",
        "train_start = '2011-01-01 00:00:00'\n",
        "train_end = '2014-08-31 23:00:00'\n",
        "test_start = '2014-08-24 00:00:00' #need additional 7 days as given info\n",
        "test_end = '2014-09-07 23:00:00'\n",
        "\n",
        "save_path = os.path.join('data', save_name)\n",
        "if not os.path.exists(save_path):\n",
        "    os.makedirs(save_path)\n",
        "csv_path = os.path.join(save_path, name)\n",
        "if not os.path.exists(csv_path):\n",
        "    zipurl = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00321/LD2011_2014.txt.zip'\n",
        "    with urlopen(zipurl) as zipresp:\n",
        "        with ZipFile(BytesIO(zipresp.read())) as zfile:\n",
        "            zfile.extractall(save_path)"
      ],
      "metadata": {
        "id": "B4lKH0IkDjdf"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Read the time series data from the CSV file, using the first column as the index and parsing dates.\n",
        "target_df = pd.read_csv(csv_path, sep=\";\", index_col=0, parse_dates=True, decimal=',')\n",
        "\n",
        "# Resample the data to 12-hour intervals, summing values within each period, and define the label and closure of intervals.\n",
        "target_df = target_df.resample('12H', label='left', closed='right').sum()[train_start:test_end]\n",
        "\n",
        "# Replace any missing values with zeros to avoid gaps in the data.\n",
        "target_df.fillna(0, inplace=True)\n",
        "\n",
        "# Split the dataset into training and test sets based on the defined date ranges.\n",
        "train_target_df = target_df[train_start:train_end]\n",
        "test_target_df = target_df[test_start:test_end]\n",
        "\n",
        "# Determine the horizon size from the test dataset's duration.\n",
        "horizon_size = test_target_df.shape[0]\n",
        "\n",
        "# Initialize the MinMaxScaler to scale the data between 0 and 1.\n",
        "scaler = MinMaxScaler()\n",
        "\n",
        "# Fit the scaler on the training data to learn the scaling parameters.\n",
        "scaler.fit(train_target_df)\n",
        "\n",
        "# Transform the training data using the learned scaling parameters and convert it back to a DataFrame.\n",
        "# Preserve the original index and column names.\n",
        "train_target_df = pd.DataFrame(scaler.transform(train_target_df), index=train_target_df.index, columns=train_target_df.columns)\n",
        "\n",
        "# Transform the test data using the same scaling parameters and convert it back to a DataFrame.\n",
        "# Preserve the original index and column names.\n",
        "test_target_df = pd.DataFrame(scaler.transform(test_target_df), index=test_target_df.index, columns=test_target_df.columns)\n",
        "\n",
        "# Create a DataFrame to hold covariate features with the same index as the target DataFrame.\n",
        "covariate_df = pd.DataFrame(index=target_df.index,\n",
        "                            data={\n",
        "                                'hour': target_df.index.hour,  # Extract the hour from the index.\n",
        "                                'dayofweek': target_df.index.dayofweek,  # Extract the day of the week from the index.\n",
        "                                'month': target_df.index.month  # Extract the month from the index.\n",
        "                            })\n",
        "\n",
        "# Standardize each covariate column to have a mean of 0 and standard deviation of 1.\n",
        "for col in covariate_df.columns:\n",
        "    covariate_df[col] = (covariate_df[col] - np.mean(covariate_df[col])) / np.std(covariate_df[col])\n",
        "\n",
        "# Slice the covariate DataFrame to create a training covariate DataFrame corresponding to the training period.\n",
        "train_covariate_df = covariate_df[train_start:train_end]\n",
        "\n",
        "# Slice the covariate DataFrame to create a testing covariate DataFrame corresponding to the testing period.\n",
        "test_covariate_df = covariate_df[test_start:test_end]\n"
      ],
      "metadata": {
        "id": "ax_riM7DDxZj"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##  Multi-Horizon Quantile Recurrent Forecaster"
      ],
      "metadata": {
        "id": "OKsHWkOjl7jt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Define the quantiles for which predictions are desired.\n",
        "desired_quantiles = [0.25, 0.5, 0.75, 0.95]\n",
        "\n",
        "# Create a class for the MQRNN dataset, inheriting from PyTorch's Dataset class.\n",
        "class MQRNN_dataset(Dataset):\n",
        "\n",
        "    # Initialize the dataset with time series data, covariates, and other parameters.\n",
        "    def __init__(self, series_df: pd.DataFrame, covariate_df: pd.DataFrame, horizon_size: int=horizon_size, quantile_size: int=len(desired_quantiles)):\n",
        "        # Store the series data and covariates.\n",
        "        self.series_df = series_df\n",
        "        self.covariate_df = covariate_df\n",
        "        self.horizon_size = horizon_size\n",
        "        self.quantile_size = quantile_size\n",
        "\n",
        "        # Prepare and store future covariate data.\n",
        "        full_covariate = []\n",
        "        covariate_size = self.covariate_df.shape[1]\n",
        "        for i in range(1, self.covariate_df.shape[0] - horizon_size + 1):\n",
        "            cur_covariate = []\n",
        "            cur_covariate.append(self.covariate_df.iloc[i:i+horizon_size, :].to_numpy())\n",
        "            full_covariate.append(cur_covariate)\n",
        "        full_covariate = np.array(full_covariate)\n",
        "        full_covariate = full_covariate.reshape(-1, horizon_size * covariate_size)\n",
        "        self.next_covariate = full_covariate\n",
        "\n",
        "    # Define the method to get the length of the dataset.\n",
        "    def __len__(self):\n",
        "        return self.series_df.shape[1]\n",
        "\n",
        "    # Define the method to retrieve an item from the dataset.\n",
        "    def __getitem__(self, idx):\n",
        "        # Extract the current time series and covariates.\n",
        "        cur_series = np.array(self.series_df.iloc[: -self.horizon_size, idx])\n",
        "        cur_covariate = np.array(self.covariate_df.iloc[:-self.horizon_size, :])\n",
        "\n",
        "        # Convert covariate size to a variable for readability.\n",
        "        covariate_size = self.covariate_df.shape[1]\n",
        "\n",
        "        # Prepare real values for the loss calculation.\n",
        "        real_vals_list = []\n",
        "        for i in range(1, self.horizon_size + 1):\n",
        "            real_vals_list.append(np.array(self.series_df.iloc[i: self.series_df.shape[0] - self.horizon_size + i, idx]))\n",
        "        real_vals_array = np.array(real_vals_list)  # [horizon_size, seq_len]\n",
        "        real_vals_array = real_vals_array.T  # [seq_len, horizon_size]\n",
        "\n",
        "        # Convert current series and covariates to tensors.\n",
        "        cur_series_tensor = torch.tensor(cur_series)\n",
        "        cur_series_tensor = torch.unsqueeze(cur_series_tensor, dim=1)  # [seq_len, 1]\n",
        "        cur_covariate_tensor = torch.tensor(cur_covariate)  # [seq_len, covariate_size]\n",
        "        cur_series_covariate_tensor = torch.cat([cur_series_tensor, cur_covariate_tensor], dim=1)\n",
        "        next_covariate_tensor = torch.tensor(self.next_covariate)  # [seq_len, horizon_size * covariate_size]\n",
        "\n",
        "        # Convert real values to a tensor.\n",
        "        cur_real_vals_tensor = torch.tensor(real_vals_array)\n",
        "\n",
        "        # Return the combined current series-covariate tensor, next covariate tensor, and real values tensor.\n",
        "        return cur_series_covariate_tensor, next_covariate_tensor, cur_real_vals_tensor\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# Define the MQRNN model class, inheriting from PyTorch's nn.Module.\n",
        "class MQRNN(nn.Module):\n",
        "    # Initialize the model with given parameters.\n",
        "    def __init__(self,\n",
        "                horizon_size:int=horizon_size,  # Define the forecast horizon size.\n",
        "                hidden_size:int=100,  # Set the size of hidden layers.\n",
        "                quantiles:list=desired_quantiles,  # Specify the quantiles for prediction.\n",
        "                dropout:float=0.3,  # Set dropout rate for regularization.\n",
        "                layer_size:int=3,  # Define the number of layers in the LSTM.\n",
        "                context_size:int=50,  # Set the size of the context vectors.\n",
        "                covariate_size:int=num_covariates,  # Define the number of covariates.\n",
        "                bidirectional=False,  # Set bidirectional LSTM if True.\n",
        "                device=torch.device('cuda')):  # Specify the device (GPU or CPU) for computation.\n",
        "        super(MQRNN, self).__init__()  # Initialize the superclass (nn.Module).\n",
        "\n",
        "        # Store the provided attributes in the instance.\n",
        "        self.quantiles = desired_quantiles\n",
        "        self.quantile_size = len(quantiles)\n",
        "        self.bidirectional = bidirectional\n",
        "        self.hidden_size = hidden_size\n",
        "        self.horizon_size = horizon_size\n",
        "        self.device = device\n",
        "        self.covariate_size = covariate_size\n",
        "\n",
        "        # Define the encoder as an LSTM network.\n",
        "        self.encoder = nn.LSTM(input_size=covariate_size+1,\n",
        "                            hidden_size=hidden_size,\n",
        "                            num_layers=layer_size,\n",
        "                            dropout=dropout,\n",
        "                            bidirectional=bidirectional)\n",
        "\n",
        "        # Define the global decoder as a series of linear layers with ReLU activations.\n",
        "        self.global_decoder = nn.Sequential(nn.Linear(in_features=hidden_size + covariate_size*horizon_size, out_features=horizon_size*hidden_size*3),\n",
        "                                           nn.ReLU(),\n",
        "                                           nn.Linear(in_features=horizon_size*hidden_size*3, out_features=horizon_size*hidden_size*2),\n",
        "                                           nn.ReLU(),\n",
        "                                           nn.Linear(in_features=horizon_size*hidden_size*2, out_features=(horizon_size+1)*context_size),\n",
        "                                           nn.ReLU())\n",
        "\n",
        "        # Define the local decoder similarly with linear layers and ReLU activations.\n",
        "        self.local_decoder = nn.Sequential(nn.Linear(in_features=horizon_size*context_size + horizon_size*covariate_size + context_size, out_features=horizon_size*context_size),\n",
        "                                           nn.ReLU(),\n",
        "                                           nn.Linear(in_features=horizon_size*context_size, out_features=horizon_size*self.quantile_size),\n",
        "                                           nn.ReLU())\n",
        "\n",
        "        # Set the data type to double for all layers and move them to the specified device (GPU/CPU).\n",
        "        self.encoder.double().to(self.device)\n",
        "        self.global_decoder.double().to(self.device)\n",
        "        self.local_decoder.double().to(self.device)\n",
        "\n",
        "    # Define the forward pass of the model.\n",
        "    def forward(self, cur_series_covariate_tensor, next_covariate_tensor):\n",
        "        # Extract sequence length and batch size from the input tensor.\n",
        "        seq_len, batch_size = cur_series_covariate_tensor.shape[0], cur_series_covariate_tensor.shape[1]\n",
        "\n",
        "        # Determine the direction size based on whether the LSTM is bidirectional.\n",
        "        direction_size = 2 if self.bidirectional else 1\n",
        "\n",
        "        # Pass the input through the encoder LSTM.\n",
        "        outputs, _ = self.encoder(cur_series_covariate_tensor)\n",
        "\n",
        "        # Reshape the outputs to separate the direction and hidden size.\n",
        "        outputs_reshape = outputs.view(seq_len, batch_size, direction_size, self.hidden_size)[:,:,-1,:]\n",
        "        encoder_outputs = outputs_reshape.view(seq_len, batch_size, self.hidden_size)\n",
        "\n",
        "        # If not training, use only the last encoder output for all future time steps.\n",
        "        if not self.training:\n",
        "            encoder_outputs = torch.unsqueeze(encoder_outputs[-1], dim=0)  # [1, 1, hidden_size]\n",
        "\n",
        "        # Pass the encoder outputs and next covariates through the global decoder.\n",
        "        global_decoder_output = self.global_decoder(torch.cat([encoder_outputs, next_covariate_tensor], dim=2))\n",
        "\n",
        "        # Pass the global decoder output and next covariates through the local decoder.\n",
        "        local_decoder_output = self.local_decoder(torch.cat([global_decoder_output, next_covariate_tensor], dim=2))\n",
        "\n",
        "        # Extract sequence length and batch size from the local decoder output.\n",
        "        seq_len = local_decoder_output.shape[0]\n",
        "        batch_size = local_decoder_output.shape[1]\n",
        "\n",
        "        # Reshape the local decoder output for the specified quantiles.\n",
        "        return local_decoder_output.view(seq_len, batch_size, self.horizon_size, self.quantile_size)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "# Define the training function for the MQRNN model.\n",
        "def train(model, train_dataset, desired_quantiles=desired_quantiles, batch_size=1, num_epochs=1, lr=1e-3, device=torch.device(\"cuda\")):\n",
        "    # Initialize the optimizer with the Adam algorithm.\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "    # Create a DataLoader to iterate over the training dataset.\n",
        "    data_iter = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "    # Set the model to training mode.\n",
        "    model.train()\n",
        "    # Iterate over the specified number of epochs.\n",
        "    for epoch in range(num_epochs):\n",
        "        epoch_loss_sum = 0.0  # Initialize the sum of loss for this epoch.\n",
        "        total_sample = 0  # Initialize the total number of samples processed.\n",
        "        pbar = tqdm(data_iter)  # Initialize a progress bar for the data iterator.\n",
        "\n",
        "        # Iterate over batches of data.\n",
        "        for (cur_series_tensor, cur_covariate_tensor, cur_real_vals_tensor) in pbar:\n",
        "            # Extract batch size, sequence length, and horizon size from the tensors.\n",
        "            batch_size, seq_len, horizon_size = cur_series_tensor.shape[0], cur_series_tensor.shape[1], cur_covariate_tensor.shape[-1]\n",
        "            total_sample += batch_size * seq_len * horizon_size  # Update total sample count.\n",
        "            optimizer.zero_grad()  # Reset gradients to zero before starting backpropagation.\n",
        "\n",
        "            # Prepare tensors for input to the model and move them to the specified device.\n",
        "            cur_series_covariate_tensor = cur_series_tensor.double().permute(1, 0, 2).to(device)  # Rearrange and convert to double.\n",
        "            next_covariate_tensor = cur_covariate_tensor.double().permute(1, 0, 2).to(device)  # Rearrange and convert to double.\n",
        "            cur_real_vals_tensor = cur_real_vals_tensor.double().permute(1, 0, 2).to(device)  # Rearrange and convert to double.\n",
        "\n",
        "            # Pass tensors through the model to get the output.\n",
        "            model_output = model(cur_series_covariate_tensor, next_covariate_tensor)\n",
        "\n",
        "            # Calculate losses for each quantile.\n",
        "            losses = []\n",
        "            for i, p in enumerate(desired_quantiles):\n",
        "                errors = cur_real_vals_tensor - model_output[:, :, :, i]  # Calculate errors for each quantile.\n",
        "                losses.append(torch.max((p - 1) * errors, p * errors))  # Apply quantile loss formula.\n",
        "            total_loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1)).to(device)  # Calculate mean of total loss.\n",
        "\n",
        "            # Update the progress bar with the current loss.\n",
        "            pbar.set_description(f\"Loss: {total_loss.item()}\")\n",
        "            total_loss.backward()  # Perform backpropagation.\n",
        "            optimizer.step()  # Update model parameters.\n",
        "            epoch_loss_sum += total_loss.item()  # Accumulate loss for the epoch.\n",
        "\n",
        "        # Calculate and print the average loss for the epoch.\n",
        "        epoch_loss_mean = epoch_loss_sum / total_sample\n",
        "        if (epoch + 1) % 5 == 0:\n",
        "            print(f\"Epoch {epoch + 1}/{num_epochs}, Current Loss: {epoch_loss_mean}\")\n",
        "\n",
        "### evaluate Function\n",
        "\n",
        "# Define the evaluate function for the MQRNN model.\n",
        "def evaluate(model, device=torch.device('cuda'), covariate_size=3):\n",
        "    # Convert training covariate DataFrame to tensor and move to the specified device.\n",
        "    full_covariate_tensor = torch.tensor(train_covariate_df.to_numpy()).to(device)\n",
        "    # Convert test covariate DataFrame to tensor, reshape, unsqueeze, and move to the specified device.\n",
        "    next_covariate_tensor = torch.tensor(test_covariate_df.to_numpy().reshape(-1, horizon_size * covariate_size)).unsqueeze(dim=0).to(device)\n",
        "    results = []  # Initialize a list to store results.\n",
        "\n",
        "    # Set the model to evaluation mode.\n",
        "    model.eval()\n",
        "    # Disable gradient calculations.\n",
        "    with torch.no_grad():\n",
        "        # Iterate over columns in the training target DataFrame.\n",
        "        for colname in tqdm(train_target_df.columns):\n",
        "            # Convert the column to tensor and move to the specified device.\n",
        "            input_target_tensor = torch.tensor(train_target_df[[colname]].to_numpy()).to(device)\n",
        "            # Combine target tensor with covariate tensor, unsqueeze, and move to the specified device.\n",
        "            input_target_covariate_tensor = torch.unsqueeze(torch.cat([input_target_tensor, full_covariate_tensor], dim=1), dim=0).to(device)\n",
        "            # Rearrange the tensor dimensions.\n",
        "            input_target_covariate_tensor = input_target_covariate_tensor.permute(1, 0, 2).to(device)\n"
      ],
      "metadata": {
        "id": "sKgQE3vllgTO"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = MQRNN()\n",
        "train_dataset = MQRNN_dataset(train_target_df, train_covariate_df)\n",
        "train(model, train_dataset, batch_size=16, num_epochs=15, lr=0.0001)\n",
        "# Save the model state\n",
        "model_save_path = '/content/drive/MyDrive/Colab_Notebooks/ads_506/mqrnn2.pth'\n",
        "torch.save(model.state_dict(), model_save_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MnUrEF0bMJJo",
        "outputId": "e12be6a6-7549-4b75-aca6-d2d20045b739"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss:0.8355442649751902: 100%|██████████| 24/24 [01:14<00:00,  3.11s/it]\n",
            "Loss:0.7289806183880713: 100%|██████████| 24/24 [01:12<00:00,  3.04s/it]\n",
            "Loss:0.45107804422925: 100%|██████████| 24/24 [01:11<00:00,  3.00s/it]\n",
            "Loss:0.650795937595124: 100%|██████████| 24/24 [01:12<00:00,  3.01s/it]\n",
            "Loss:0.29195850184528627: 100%|██████████| 24/24 [01:11<00:00,  2.99s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch_num 5, current loss is: 1.0988836889952198e-06\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss:0.2491680562709323: 100%|██████████| 24/24 [01:11<00:00,  2.99s/it]\n",
            "Loss:0.2601942242261246: 100%|██████████| 24/24 [01:11<00:00,  3.00s/it]\n",
            "Loss:0.24870597028499428: 100%|██████████| 24/24 [01:12<00:00,  3.02s/it]\n",
            "Loss:0.1801676619723639: 100%|██████████| 24/24 [01:13<00:00,  3.05s/it]\n",
            "Loss:0.177961324489008: 100%|██████████| 24/24 [01:13<00:00,  3.06s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch_num 10, current loss is: 4.23120875475864e-07\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loss:0.177108150255517: 100%|██████████| 24/24 [01:12<00:00,  3.01s/it]\n",
            "Loss:0.21538066589021831: 100%|██████████| 24/24 [01:12<00:00,  3.03s/it]\n",
            "Loss:0.20035888852840025: 100%|██████████| 24/24 [01:12<00:00,  3.02s/it]\n",
            "Loss:0.21567029552323666: 100%|██████████| 24/24 [01:12<00:00,  3.01s/it]\n",
            "Loss:0.3538721460684165: 100%|██████████| 24/24 [01:12<00:00,  3.02s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch_num 15, current loss is: 4.1205895706314956e-07\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_rmse = evaluate(model)\n",
        "print(test_rmse)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m4_iFaYphrnH",
        "outputId": "ba159a1c-6a2e-4e1b-efdd-e91c02587f57"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 370/370 [00:27<00:00, 13.57it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.5520811276694852\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(test_rmse)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MoxEfWvMh4eh",
        "outputId": "10da31aa-40ef-4b26-8254-192acca2968b"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.5520811276694852\n"
          ]
        }
      ]
    }
  ]
}