diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 36bb390d..4a1a0900 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -29,12 +29,11 @@ jobs: with: path: | ~/.cache/huggingface - key: ${{ runner.os }}-huggingface-cache-v1 # increment this key to invalidate the cache when new models/datasets are added + key: ${{ runner.os }}-hf-cache-v0.2 # increment this key to invalidate the cache when new models/datasets are added - name: dependencies run: | python -m pip install --upgrade pip - pip install -r requirements-nocuda.txt - pip install -e . + pip install -e .[dev,notebooks] - name: black run: black --check . - name: isort diff --git a/README.md b/README.md index f6569099..5d04ac5c 100644 --- a/README.md +++ b/README.md @@ -1,99 +1,196 @@ -# Delphi +# delphi -Interpreting Small Language Models Across Time and Scale +delphi is a set of tools for standardized and (mostly) reproducible training of small language models. You can use delphi to train a custom tokenizer, tokenize your dataset, and train your model. We build on top of HuggingFace, supporting every `CausalLM` architecture. Datasets, tokenizers and models (including checkpoints!) can be downloaded from and uploaded to HuggingFace automatically, with no need to manage local files. -# Training Models -See [`scripts/run_training.py`](scripts/run_training.py): -```bash - ./scripts/run_training.py --config_file /path/to/my/training/config.json + +# Setup + +1. Clone the repo +```shell +git clone https://github.com/delphi-suite/delphi.git +cd delphi +``` +2. Make & activate python >= 3.10 virtual env +```shell +python3.10 -m venv .venv +source .venv/bin/activate +``` +3. Install the project in editable state +`pip install -e .` +See `[project.optional-dependencies]` section in `pyproject.toml` for additional dependencies, e.g. you may want to `pip install -e ."[dev,mamba_cuda]"` +4. get your HuggingFace and W&B tokens and put them in the environment variables +```shell +export HF_TOKEN=... +export WANDB_API_KEY=... ``` -See [`scripts/sample_config.json`](scripts/sample_config.json) for an example of a training run json. +# Training a tokenizer -## Features -### Uploading to HuggingFace -With `huggingface.push_checkpoints_to_hub` set to `True`, the model and all associated -training run data will be uploaded to HuggingFace repo specified by `huggingface.repo_id` -every checkpoint. Every upload will be in a new folder named by the current iteration (e.g. `iter_1`). -### Resuming model training -With `init_from` set to `'resume'`, training will resume from `output_dir`. -### Deterministic, Reproducible* Training -Delphi aims to be deterministic and as reproducible as possible. However, there is one major caveat: hardware. CUDA algorithms are not always 100% isomorphic to CPU algorithms. We do record the hardware device type each training run uses, -to enable reproduction *given the same class of hardware*. -### Different Model Architectures -`model_config.model_type` can specify currently supported architectures. At time of writing, these are `'llama2'` and `'mamaba`'. Config for the selected model type should -be in `model_config.` (e.g. `model_config.llama2`) and correspond to the -arguments for that model type. See [`model_types.py`](src/delphi/train/config/models/model_types.py) -### Weights and Biases Integration +If you want to train a small and efficient model on a narrow dataset, then we recommend using a custom tokenizer with a small vocabulary. To train a reversible, GPT2-style, BPE tokenizer you can use `scripts/train_tokenizer.py`. +Script usage: -# Analyzing Models -TODO +``` +> scripts/train_tokenizer.py --help +usage: train_tokenizer.py [-h] --in-dataset IN_DATASET --feature FEATURE --split SPLIT + --vocab-size VOCAB_SIZE + [--out-dir OUT_DIR] [--out-repo OUT_REPO] + +Train a custom, reversible, BPE tokenizer (GPT2-like). You need to provide --out-repo or --out-dir. + +options: + -h, --help show this help message and exit + --in-dataset IN_DATASET, -i IN_DATASET + Dataset you want to train the tokenizer on. Local path or HF repo id + --feature FEATURE, -f FEATURE + Name of the feature (column) containing text documents in the input dataset + --split SPLIT, -s SPLIT + Split of the dataset to be used for tokenizer training, supports slicing like 'train[:10%]' + --vocab-size VOCAB_SIZE, -v VOCAB_SIZE + Vocabulary size of the tokenizer + --out-dir OUT_DIR Local directory to save the resulting tokenizer + --out-repo OUT_REPO HF repo id to upload the resulting tokenizer +``` -# Development +Here's how we trained the tokenizer for our `stories-*` suite of models. Please note that you can use single letter abbreviations for most arguments. + +``` +> scripts/train_tokenizer.py \ + --in-dataset delphi-suite/stories \ + --feature story \ + --split train \ + --vocab-size 4096 \ + --out-repo delphi-suite/stories-tokenizer +``` + +We use the only feature named `story` in the `train` split of [delphi-suite/stories](https://huggingface.co/datasets/delphi-suite/stories). We train a tokenizer with a vocabulary of 4096 tokens, and upload it to HF model repo [delphi-suite/stories-tokenizer](https://huggingface.co/delphi-suite/stories-tokenizer). -## Setup -1. Clone this repo and submodules: `git clone https://github.com/delphi-suite/delphi.git --recurse-submodules` -2. make python 3.10 virtual env in `.venv` -3. install dependencies `pip install -r requirements.txt` -4. install the project in editable state `pip install -e .` -5. run tests `pytest` +# Tokenizing a dataset -### Submodule Setup -If you cloned without `--recurse-submodules`, you can still install the submodules later with: -```bash -git submodule init -git submodule update +To turn a collection of text documents into sequences of tokens required for model training, you can use `scripts/tokenize_dataset.py`. All documents are tokenized and concatenated, with the `` token as a separator, e.g. ``` +doc1_tok1, doc1_tok2, ..., doc1_tokX, , doc2_tok1, doc2_tok2, ..., doc2_tokX, , doc3_tok1, ... +``` +Then this is divided into chunks, and the `` token is inserted at the begining of each chunk, e.g. +``` + doc1_tok1, doc1_tok2, ..., doc1_tokX, , doc2_tok1 + doc2_tok2, ..., doc2_tok511 + doc2_tok512, doc2_tok513, ..., doc2_tokX , doc3_tok1, ... +... +``` +It will produce sequences of specified size, by discarding the last chunk if it's too short. We don't use padding. -## Formatting +Script usage: -We're using black & isort to format the code. To make sure your changes adhere to the rules: +``` +> scripts/tokenize_dataset.py --help +usage: tokenize_dataset.py [-h] --in-dataset IN_DATASET --feature FEATURE --split SPLIT + --tokenizer TOKENIZER --seq-len SEQ_LEN + [--batch-size BATCH_SIZE] [--chunk-size CHUNK_SIZE] + [--out-dir OUT_DIR] [--out-repo OUT_REPO] + +Tokenize a text dataset using a specific tokenizer + +options: + -h, --help show this help message and exit + --in-dataset IN_DATASET, -i IN_DATASET + Dataset you want to tokenize. Local path or HF repo id + --feature FEATURE, -f FEATURE + Name of the feature (column) containing text documents in the input dataset + --split SPLIT, -s SPLIT + Split of the dataset to be tokenized, supports slicing like 'train[:10%]' + --tokenizer TOKENIZER, -t TOKENIZER + HF repo id or local directory containing the tokenizer + --seq-len SEQ_LEN, -l SEQ_LEN + Length of the tokenized sequences + --batch-size BATCH_SIZE, -b BATCH_SIZE + How many text documents to tokenize at once (default: 50) + --chunk-size CHUNK_SIZE, -c CHUNK_SIZE + Maximum number of tokenized sequences in a single parquet file (default: 200_000) + --out-dir OUT_DIR Local directory to save the resulting dataset + --out-repo OUT_REPO HF repo id to upload the resulting dataset +``` -1. follow setup instructions above -2. install pre-commit `pre-commit install` -3. install recommended vscode extensions +Here's how we tokenized the dataset for our `stories-*` suite of models. Please note that you can use single letter abbreviations for most arguments. -When you save a file vscode should automatically format it. Otherwise, pre-commit will do that, but you will need to add the changes and commit again. +For `train` split: +``` +> scripts/tokenize_dataset.py \ + --in-dataset delphi-suite/stories \ + --feature story \ + --split train \ + --tokenizer delphi-suite/stories-tokenizer \ + --seq-len 512 \ + --out-repo delphi-suite/stories-tokenized +``` +For `validation` split, repeated arguments omitted: +``` +> scripts/tokenize_dataset.py \ + ... + --split validation \ + ... +``` -## Pull Requests - -1. make a branch - - if it relates to an existing issue - - go to the issue page and click _Create a branch_ under _Development_ - - if the default name is not very long, keep it; otherwise, make it shorter, but keep the issue number in the front - - otherwise pick a short but descriptive name, a few hyphen-separated-words -2. make your changes - - include unit tests - - update README if needed - - if new huggingface datasets/models are added to testing, increment the cache number in `.github/workflows/checks.yml` -3. make a pull request - - if it isn't ready for review yet, mark it as draft - - check if CI is passing - - if the change is big, try to keep the commit history clean using interactive rebase - - don't push more often than it's needed, we're running github actions on a free tier - - if there were any changes to the main branch, rebase on top of it - - explain the change - - provide short description; focus on things that were not mentioned in the relevant issue - - comment important sections of the code in _Files changed_ tab - - when it's ready, add the relevant stakeholders as reviewers -4. after the comments are resolved and PR is approved, merge it using _Squash and merge_ - -## Incrementing Versions -When making a new release, increment the version in `delphi/__init__.py` +The input dataset is the same as in tokenizer training example above. We tokenize it with our custom [delphi-suite/stories-tokenizer](https://huggingface.co/delphi-suite/stories-tokenizer) into sequences of length 512. We upload it to HF dataset repo [delphi-suite/stories-tokenized](https://huggingface.co/datasets/delphi-suite/stories-tokenized). + +Please note that you can use any HuggingFace tokenizer, you don't need to train a custom one. + +# Training a model + +To train a model, you'll need to create a config file. For examples see `configs/`, and for field descriptions see `delphi/train/config/training_config.py`. The training script is located in `scripts/train_model.py`. + +Script usage: + +``` +> scripts/train_model.py --help +usage: train_model.py [-h] [--overrides [OVERRIDES ...]] [-v | -s] [config_files ...] + +Train a delphi model + +positional arguments: + config_files Path to json file(s) containing config values, e.g. 'primary_config.json secondary_config.json'. + +options: + -h, --help show this help message and exit + --overrides [OVERRIDES ...] + Override config values with space-separated declarations. e.g. `--overrides model_config.hidden_size=42 run_name=foo` + -v, --verbose Increase verbosity level, repeatable (e.g. -vvv). Mutually exclusive with --silent, --loglevel + -s, --silent Silence all logging. Mutually exclusive with --verbose, --loglevel +``` + +You can specify primary config and secondary config, which is useful if you're training a suite of models that only differ in a few parameters. Additionally, you can override specific fields using the `--overrides` flag. If you don't want to push the model and its checkpoints to HF, you need to explicitly set `out_repo=""`. If you don't want to log to W&B, you need to set `wandb=""`. Please note that by default we save the optimizer state (2x model size) with every checkpoint. + +Here is how we trained our `stories-mamba-100k` model +``` +> scripts/train_model.py \ + configs/stories/mamba/base.json \ + configs/stories/mamba/100k.json \ + --overrides \ + out_repo="delphi-suite/stories-mamba-100k" \ + wandb="delphi-suite/delphi" +``` + +# Development + +1. Install the `dev` and `notebooks` dependencies `pip install -e ."[dev,notebooks]"`. +2. Run the tests `pytest`. +3. Install pre-commit `pre-commit install`. +4. Install the recommended vscode extensions. + +When you save a file vscode should automatically format it. Otherwise, pre-commit will do that, but you will need to add the changes and commit again. # Citation -If you use `delphi` in your research, please cite using the following +If you use delphi in your research, please cite using the following ```bibtex @software{delphi, title = {delphi: small language models training made easy}, - author = {Jett Janiak, Jai Dhyani, Jannik Brinkmann, Gonçalo Paulo, Joshua Wendland, Víctor Abia Alonso, Siwei Li, Rai (Phan Anh Duong), Alice Rigg}, + author = {Jett Janiak, Jai Dhyani, Jannik Brinkmann, Gonçalo Paulo, Joshua Wendland, Víctor Abia Alonso, Siwei Li, Phan Anh Duong, Alice Rigg}, year = 2024, url = {https://github.com/delphi-suite/delphi}, license = {apache-2.0} } -``` +``` \ No newline at end of file diff --git a/configs/debug.json b/configs/debug.json deleted file mode 100644 index bdfd6308..00000000 --- a/configs/debug.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "max_seq_len": 512, - "max_epochs": 2, - "eval_iters": 1, - "batch_ordering_seed": 42, - "torch_seed": 1337, - "batch_size": 64, - "model_config": { - "model_class": "LlamaForCausalLM", - "hidden_size": 48, - "intermediate_size": 48, - "num_attention_heads": 2, - "num_hidden_layers": 2, - "num_key_value_heads": 2, - "vocab_size": 4096 - }, - "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" - } -} \ No newline at end of file diff --git a/configs/sample_config.json b/configs/sample_config.json deleted file mode 100644 index ac538399..00000000 --- a/configs/sample_config.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "run_name": "2024_03_15_17_28_14", - "output_dir": "/Users/jaidhyani/Library/Application Support/delphi", - "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" - }, - "device": "auto", - "log_interval": 1, - "eval_iters": 100, - "wandb": { - "project": "delphi", - "entity": "set_wandb.entity_to_your_wandb_username_to_make_wandb_logging_work" - }, - "batch_size": 64, - "max_seq_len": 512, - "model_config": { - "model_class": "LlamaForCausalLM", - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": -1, - "eos_token_id": -2, - "hidden_act": "silu", - "hidden_size": 288, - "initializer_range": 0.02, - "intermediate_size": 288, - "max_position_embeddings": 512, - "num_attention_heads": 6, - "num_hidden_layers": 6, - "num_key_value_heads": 6, - "pretraining_tp": 1, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 10000.0, - "tie_word_embeddings": false, - "use_cache": true, - "vocab_size": 4096 - }, - "max_epochs": 10, - "gradient_accumulation_steps": 1, - "grad_clip": 1.0, - "adam": { - "learning_rate": 0.0005, - "weight_decay": 0.1, - "beta1": 0.9, - "beta2": 0.95, - "decay_lr": true, - "warmup_iters": 1000, - "min_lr": 0.0 - }, - "batch_ordering_seed": 42, - "torch_seed": 1337 -} \ No newline at end of file diff --git a/configs/sample_mamba.json b/configs/sample_mamba.json deleted file mode 100644 index 7fddcb26..00000000 --- a/configs/sample_mamba.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "max_seq_len": 512, - "max_epochs": 2, - "log_interval": 1, - "eval_iters": 10, - "batch_size": 8, - "model_config": { - "model_class": "MambaForCausalLM", - "vocab_size": 4096, - "hidden_size": 48, - "state_size": 16, - "num_hidden_layers": 2, - "conv_kernel": 2, - "expand": 2, - "time_step_rank": 2 - }, - "batch_ordering_seed": 42, - "torch_seed": 1337, - "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" - } -} \ No newline at end of file diff --git a/configs/sample_transformers_bloom.json b/configs/sample_transformers_bloom.json deleted file mode 100644 index 793f6a8a..00000000 --- a/configs/sample_transformers_bloom.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "max_seq_len": 512, - "max_epochs": 2, - "eval_iters": 1, - "batch_size": 64, - "model_config": { - "model_class": "BloomForCausalLM", - "apply_residual_connection_post_layernorm": false, - "attention_dropout": 0.0, - "bos_token_id": 1, - "eos_token_id": 2, - "hidden_dropout": 0.0, - "hidden_size": 8, - "initializer_range": 0.02, - "layer_norm_epsilon": 1e-05, - "n_head": 2, - "n_layer": 2, - "pretraining_tp": 1, - "slow_but_exact": false, - "use_cache": true, - "vocab_size": 4096 - }, - "batch_ordering_seed": 42, - "torch_seed": 1337, - "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" - } -} \ No newline at end of file diff --git a/configs/stories/llama2/README.md b/configs/stories/llama2/README.md index be1f976e..6192ef38 100644 --- a/configs/stories/llama2/README.md +++ b/configs/stories/llama2/README.md @@ -1,7 +1,6 @@ -not using padding, so pad_token_id not set -use_cache - using default -pretraining_tp - experimental parallelization we're not using, which is the default -tie_word_embeddings - llama2 used False and this is better for interpretability, note that llama2.c is using True by default, which is probably more efficient use of parameters for very small models -rope settings are widely used defaults -attention_bias - no biases on QKV and output projection is the default and that's what we're using -attention_dropout - this is the only dropout llama2 can use, it's set to prob=0 by default and that's what we're using \ No newline at end of file +- use_cache - using default +- pretraining_tp - experimental parallelization we're not using, which is the default +- tie_word_embeddings - llama2 used False and this is better for interpretability, note that llama2.c is using True by default, which is probably more efficient use of parameters for very small models +- rope settings are widely used defaults +- attention_bias - no biases on QKV and output projection is the default and that's what we're using +- attention_dropout - this is the only dropout llama2 can use, it's set to prob=0 by default and that's what we're using \ No newline at end of file diff --git a/configs/stories/llama2/base.json b/configs/stories/llama2/base.json index 4a2394d5..870ddf2f 100644 --- a/configs/stories/llama2/base.json +++ b/configs/stories/llama2/base.json @@ -48,7 +48,7 @@ "batch_ordering_seed": 1337, "torch_seed": 42, "dataset": { - "name": "delphi-suite/stories-tokenized" + "path": "delphi-suite/stories-tokenized" }, "tokenizer": "delphi-suite/stories-tokenizer" } \ No newline at end of file diff --git a/configs/stories/mamba/README.md b/configs/stories/mamba/README.md index 3e83bccc..7e30ceb7 100644 --- a/configs/stories/mamba/README.md +++ b/configs/stories/mamba/README.md @@ -1,10 +1,8 @@ -pad_token_id - we're not using pad tokens, do we don't set it -layer_norm_eps - different than rms norm eps in mamba -initializer_range - different in mamba & llama -residual_in_fp32 - mamba specific parameter -time_step_* - mamba specific, sane defaults -there is no way to untie embeddings and unembeddings in mamba, they're tied by default -https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/mamba/modeling_mamba.py#L602-L610 -rescale_prenorm_residual was True in original paper, so we set it to True, despite HF default being false -using default for use_cache -state_size is default \ No newline at end of file +- layer_norm_eps - different than rms norm eps in llama +- initializer_range - different in mamba & llama +- residual_in_fp32 - mamba specific parameter +- time_step_* - mamba specific, sane defaults +- there is no way to untie embeddings and unembeddings in mamba, they're tied by default https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/mamba/modeling_mamba.py#L602-L610 +- rescale_prenorm_residual was True in original paper, so we set it to True, despite HF default being false +- using default for use_cache +- state_size is default \ No newline at end of file diff --git a/configs/stories/mamba/base.json b/configs/stories/mamba/base.json index 4e151a61..83bac807 100644 --- a/configs/stories/mamba/base.json +++ b/configs/stories/mamba/base.json @@ -49,7 +49,7 @@ "batch_ordering_seed": 1337, "torch_seed": 42, "dataset": { - "name": "delphi-suite/stories-tokenized" + "path": "delphi-suite/stories-tokenized" }, "tokenizer": "delphi-suite/stories-tokenizer" } \ No newline at end of file diff --git a/src/delphi/constants.py b/delphi/__init__.py similarity index 53% rename from src/delphi/constants.py rename to delphi/__init__.py index c86e97fb..0f0ca210 100644 --- a/src/delphi/constants.py +++ b/delphi/__init__.py @@ -2,7 +2,9 @@ from pathlib import Path from typing import cast -TEST_CONFIGS_DIR = cast(Path, files("delphi.test_configs")) +from beartype.claw import beartype_this_package # <-- hype comes + +beartype_this_package() # <-- hype goes -CORPUS_DATASET = "delphi-suite/stories" -TINYSTORIES_TOKENIZED_HF_DATASET = "delphi-suite/v0-tinystories-v2-clean-tokenized" +__version__ = "0.2" +TEST_CONFIGS_DIR = cast(Path, files("delphi.test_configs")) diff --git a/delphi/eval.py b/delphi/eval.py new file mode 100644 index 00000000..c7ffaf2c --- /dev/null +++ b/delphi/eval.py @@ -0,0 +1,379 @@ +import math +import random +import uuid +from typing import Any, Optional, cast + +import numpy as np +import panel as pn +import plotly.graph_objects as go +import torch +from datasets import Dataset +from IPython.core.display import HTML +from IPython.core.display_functions import display +from jaxtyping import Float, Int +from transformers import PreTrainedTokenizerBase + + +def single_loss_diff_to_color(loss_diff: float) -> str: + # if loss_diff is negative, we want the color to be red + # if loss_diff is positive, we want the color to be green + # if loss_diff is 0, we want the color to be white + # the color should be more intense the larger the absolute value of loss_diff + + def sigmoid(x: float) -> float: + return 1 / (1 + math.exp(-x)) + + scaled_loss_diff = sigmoid(loss_diff) # scale to 0-1 + + if scaled_loss_diff < 0.5: # red + red_val = 255 + green_blue_val = min(int(255 * 2 * scaled_loss_diff), 255) + return f"rgb({red_val}, {green_blue_val}, {green_blue_val})" + else: # green + green_val = 255 + red_blue_val = min(int(255 * 2 * (1 - scaled_loss_diff)), 255) + return f"rgb({red_blue_val}, {green_val}, {red_blue_val})" + + +def token_to_html( + token: int, + tokenizer: PreTrainedTokenizerBase, + bg_color: str, + data: dict, + class_name: str = "token", +) -> str: + data = data or {} # equivalent to if not data: data = {} + # non-breakable space, w/o it leading spaces wouldn't be displayed + str_token = tokenizer.decode(token).replace(" ", " ") + + # background or user-select (for \n) goes here + specific_styles = {} + # for now just adds line break or doesn't + br = "" + + if bg_color: + specific_styles["background-color"] = bg_color + if str_token == "\n": + # replace new line character with two characters: \ and n + str_token = r"\n" + # add line break in html + br += "
" + # this is so we can copy the prompt without "\n"s + specific_styles["user-select"] = "none" + str_token = str_token.replace("<", "<").replace(">", ">") + + style_str = data_str = "" + # converting style dict into the style attribute + if specific_styles: + inside_style_str = "; ".join(f"{k}: {v}" for k, v in specific_styles.items()) + style_str = f" style='{inside_style_str}'" + if data: + data_str = "".join( + f" data-{k}='{v.replace(' ', ' ')}'" for k, v in data.items() + ) + return f"
{str_token}
{br}" + + +_token_style = { + "border": "1px solid #888", + "display": "inline-block", + # each character of the same width, so we can easily spot a space + "font-family": "monospace", + "font-size": "14px", + "color": "black", + "background-color": "white", + "margin": "1px 0px 1px 1px", + "padding": "0px 1px 1px 1px", +} +_token_emphasized_style = { + "border": "3px solid #888", + "display": "inline-block", + "font-family": "monospace", + "font-size": "14px", + "color": "black", + "background-color": "white", + "margin": "1px 0px 1px 1px", + "padding": "0px 1px 1px 1px", +} +_token_style_str = " ".join([f"{k}: {v};" for k, v in _token_style.items()]) +_token_emphasized_style_str = " ".join( + [f"{k}: {v};" for k, v in _token_emphasized_style.items()] +) + + +def vis_pos_map( + pos_list: list[tuple[int, int]], + selected_tokens: list[int], + metrics: Float[torch.Tensor, "prompt pos"], + token_ids: Int[torch.Tensor, "prompt pos"], + tokenizer: PreTrainedTokenizerBase, +): + """ + Randomly sample from pos_map and visualize the loss diff at the corresponding position. + """ + + token_htmls = [] + unique_id = str(uuid.uuid4()) + token_class = f"pretoken_{unique_id}" + selected_token_class = f"token_{unique_id}" + hover_div_id = f"hover_info_{unique_id}" + + # choose a random keys from pos_map + key = random.choice(pos_list) + + prompt, pos = key + all_toks = token_ids[prompt][: pos + 1] + + for i in range(all_toks.shape[0]): + token_id = cast(int, all_toks[i].item()) + value = metrics[prompt][i].item() + token_htmls.append( + token_to_html( + token_id, + tokenizer, + bg_color="white" + if np.isnan(value) + else single_loss_diff_to_color(value), + data={"loss-diff": f"{value:.2f}"}, + class_name=token_class + if token_id not in selected_tokens + else selected_token_class, + ) + ) + + # add break line + token_htmls.append("

") + + html_str = f""" + + {"".join(token_htmls)}
+ + """ + display(HTML(html_str)) + + +def token_selector( + vocab_map: dict[str, int] +) -> tuple[pn.widgets.MultiChoice, list[int]]: + tokens = list(vocab_map.keys()) + token_selector_ = pn.widgets.MultiChoice(name="Tokens", options=tokens) + token_ids = [vocab_map[token] for token in cast(list[str], token_selector_.value)] + + def update_tokens(event): + token_ids.clear() + token_ids.extend([vocab_map[token] for token in event.new]) + + token_selector_.param.watch(update_tokens, "value") + return token_selector_, token_ids + + +def calc_model_group_stats( + tokenized_corpus_dataset: Dataset, + logprobs_by_dataset: dict[str, torch.Tensor], + selected_tokens: list[int], +) -> dict[str, dict[str, float]]: + """ + For each (model, token group) pair, calculate useful stats (for visualization) + + args: + - tokenized_corpus_dataset: a list of the tokenized corpus datasets, e.g. load_dataset(constants.tokenized_corpus_dataset))["validation"] + - logprob_datasets: a dict of lists of logprobs, e.g. {"llama2": load_dataset("transcendingvictor/llama2-validation-logprobs")["validation"]["logprobs"]} + - selected_tokens: a list of selected token IDs, e.g. [46, 402, ...] + + returns: a dict of model names as keys and stats dict as values + e.g. {"100k": {"mean": -0.5, "median": -0.4, "min": -0.1, "max": -0.9, "25th": -0.3, "75th": -0.7}, ...} + + Stats calculated: mean, median, min, max, 25th percentile, 75th percentile + """ + model_group_stats = {} + for model in logprobs_by_dataset: + model_logprobs = [] + print(f"Processing model {model}") + dataset = logprobs_by_dataset[model] + for ix_doc_lp, document_lps in enumerate(dataset): + tokens = tokenized_corpus_dataset[ix_doc_lp]["tokens"] + for ix_token, token in enumerate(tokens): + if ix_token == 0: # skip the first token, which isn't predicted + continue + logprob = document_lps[ix_token].item() + if token in selected_tokens: + model_logprobs.append(logprob) + + if model_logprobs: + model_group_stats[model] = { + "mean": np.mean(model_logprobs), + "median": np.median(model_logprobs), + "min": np.min(model_logprobs), + "max": np.max(model_logprobs), + "25th": np.percentile(model_logprobs, 25), + "75th": np.percentile(model_logprobs, 75), + } + return model_group_stats + + +def dict_filter_quantile( + d: dict[Any, float], q_start: float, q_end: float +) -> dict[Any, float]: + if not (0 <= q_start < q_end <= 1): + raise ValueError("Invalid quantile range") + q_start_val = np.nanquantile(list(d.values()), q_start) + q_end_val = np.nanquantile(list(d.values()), q_end) + return { + k: v for k, v in d.items() if q_start_val <= v <= q_end_val and not np.isnan(v) + } + + +def get_all_tok_metrics_in_label( + token_ids: Int[torch.Tensor, "prompt pos"], + selected_tokens: list[int], + metrics: torch.Tensor, + q_start: Optional[float] = None, + q_end: Optional[float] = None, +) -> dict[tuple[int, int], float]: + """ + From the token_map, get all the positions of the tokens that have a certain label. + We don't use the token_map because for sampling purposes, iterating through token_ids is more efficient. + Optionally, filter the tokens based on the quantile range of the metrics. + + Args: + - token_ids (Dataset): token_ids dataset e.g. token_ids[0] = {"tokens": [[1, 2, ...], [2, 5, ...], ...]} + - selected_tokens (list[int]): list of token IDs to search for e.g. [46, 402, ...] + - metrics (torch.Tensor): tensor of metrics to search through e.g. torch.tensor([[0.1, 0.2, ...], [0.3, 0.4, ...], ...]) + - q_start (float): the start of the quantile range to filter the metrics e.g. 0.1 + - q_end (float): the end of the quantile range to filter the metrics e.g. 0.9 + + Returns: + - tok_positions (dict[tuple[int, int], Number]): dictionary of token positions and their corresponding metrics + """ + + # check if metrics have the same dimensions as token_ids + if metrics.shape != token_ids.shape: + raise ValueError( + f"Expected metrics to have the same shape as token_ids, but got {metrics.shape} and {token_ids.shape} instead." + ) + + tok_positions = {} + for prompt_pos, prompt in enumerate(token_ids.numpy()): + for tok_pos, tok in enumerate(prompt): + if tok in selected_tokens: + tok_positions[(prompt_pos, tok_pos)] = metrics[ + prompt_pos, tok_pos + ].item() + + if q_start is not None and q_end is not None: + tok_positions = dict_filter_quantile(tok_positions, q_start, q_end) + + return tok_positions + + +def visualize_selected_tokens( + input: dict[str | int, tuple[float, float, float]], + log_scale=False, + line_metric="Means", + checkpoint_mode=True, + shade_color="rgba(68, 68, 68, 0.3)", + line_color="rgb(31, 119, 180)", + bar_color="purple", + marker_color="SkyBlue", + background_color="AliceBlue", +) -> go.FigureWidget: + input_x = list(input.keys()) + + def get_hovertexts(mid: np.ndarray, lo: np.ndarray, hi: np.ndarray) -> list[str]: + return [f"Loss: {m:.3f} ({l:.3f}, {h:.3f})" for m, l, h in zip(mid, lo, hi)] + + def get_plot_values() -> tuple[np.ndarray, np.ndarray, np.ndarray]: + x = np.array([input[x] for x in input_x]).T + means, err_lo, err_hi = x[0], x[1], x[2] + return means, err_lo, err_hi + + means, err_lo, err_hi = get_plot_values() + + if checkpoint_mode: + scatter_plot = go.Figure( + [ + go.Scatter( + name="Upper Bound", + x=input_x, + y=means + err_hi, + mode="lines", + marker=dict(color=shade_color), + line=dict(width=0), + showlegend=False, + ), + go.Scatter( + name="Lower Bound", + x=input_x, + y=means - err_lo, + marker=dict(color=shade_color), + line=dict(width=0), + mode="lines", + fillcolor=shade_color, + fill="tonexty", + showlegend=False, + ), + go.Scatter( + name=line_metric, + x=input_x, + y=means, + mode="lines", + marker=dict( + color=line_color, + size=0, + line=dict(color=line_color, width=1), + ), + ), + ] + ) + else: + scatter_plot = go.Scatter( + x=input_x, + y=means, + error_y=dict( + type="data", + symmetric=False, + array=err_hi, + arrayminus=err_lo, + color=bar_color, + ), + marker=dict( + color=marker_color, + size=15, + line=dict(color=line_color, width=2), + ), + hovertext=get_hovertexts(means, err_lo, err_hi), + hoverinfo="text+x", + ) + g = go.FigureWidget( + data=scatter_plot, + layout=go.Layout( + yaxis=dict( + title="Loss", + type="log" if log_scale else "linear", + ), + plot_bgcolor=background_color, + ), + ) + + return g diff --git a/src/delphi/dataset/__init__.py b/delphi/test_configs/__init__.py similarity index 100% rename from src/delphi/dataset/__init__.py rename to delphi/test_configs/__init__.py diff --git a/src/delphi/test_configs/debug.json b/delphi/test_configs/debug.json similarity index 83% rename from src/delphi/test_configs/debug.json rename to delphi/test_configs/debug.json index 03d5c35d..f6e3360a 100644 --- a/src/delphi/test_configs/debug.json +++ b/delphi/test_configs/debug.json @@ -15,7 +15,8 @@ "vocab_size": 4096 }, "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" + "path": "delphi-suite/stories-tokenized" }, - "out_repo_id": "" + "out_repo": "", + "wandb": "" } \ No newline at end of file diff --git a/src/delphi/dataset/tokenization.py b/delphi/tokenization.py similarity index 100% rename from src/delphi/dataset/tokenization.py rename to delphi/tokenization.py diff --git a/src/delphi/eval/__init__.py b/delphi/train/__init__.py similarity index 100% rename from src/delphi/eval/__init__.py rename to delphi/train/__init__.py diff --git a/src/delphi/train/checkpoint_step.py b/delphi/train/checkpoint_step.py similarity index 100% rename from src/delphi/train/checkpoint_step.py rename to delphi/train/checkpoint_step.py diff --git a/src/delphi/train/config/__init__.py b/delphi/train/config/__init__.py similarity index 85% rename from src/delphi/train/config/__init__.py rename to delphi/train/config/__init__.py index fe0e825b..dc5a504d 100644 --- a/src/delphi/train/config/__init__.py +++ b/delphi/train/config/__init__.py @@ -6,4 +6,3 @@ dot_notation_to_dict, get_user_config_path, ) -from .wandb_config import WandbConfig diff --git a/src/delphi/train/config/adam_config.py b/delphi/train/config/adam_config.py similarity index 100% rename from src/delphi/train/config/adam_config.py rename to delphi/train/config/adam_config.py diff --git a/delphi/train/config/dataset_config.py b/delphi/train/config/dataset_config.py new file mode 100644 index 00000000..8f3fbe47 --- /dev/null +++ b/delphi/train/config/dataset_config.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass, field + +from beartype import beartype +from datasets import Dataset + +from delphi import utils + + +@beartype +@dataclass(frozen=True) +class DatasetConfig: + # tokenized dataset; HF repo id or local directory + path: str + + # feature in the dataset; should be a list of <= max_seq_len token ints + feature: str = "tokens" + + # split of the dataset to use for training + train_split: str = "train" + + # split of the dataset to use for validation + validation_split: str = "validation" + + def _load(self, split) -> Dataset: + ds = utils.load_dataset_split_sequence_int32_feature( + self.path, split, self.feature + ) + ds.set_format("torch") + return ds + + def load_train(self) -> Dataset: + return self._load(self.train_split) + + def load_validation(self) -> Dataset: + return self._load(self.validation_split) diff --git a/src/delphi/train/config/debug_config.py b/delphi/train/config/debug_config.py similarity index 100% rename from src/delphi/train/config/debug_config.py rename to delphi/train/config/debug_config.py diff --git a/delphi/train/config/training_config.py b/delphi/train/config/training_config.py new file mode 100644 index 00000000..1e4d3730 --- /dev/null +++ b/delphi/train/config/training_config.py @@ -0,0 +1,82 @@ +import os +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional + +import platformdirs +from beartype import beartype + +from .adam_config import AdamConfig +from .dataset_config import DatasetConfig +from .debug_config import DebugConfig + + +@beartype +@dataclass(frozen=True, kw_only=True) +class TrainingConfig: + # model config; class_name=name of model class in transformers, everything else is kwargs for the corresponding model config + model_config: dict[str, Any] + + max_seq_len: int + run_name: str = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + out_dir: str = os.path.join(platformdirs.user_data_dir(appname="delphi"), run_name) + + # device to use (cuda, mps, cpu) + device: str = "auto" + + # checkpoint every N iters + checkpoint_interval: int = 2000 + + # manually list iterations to save checkpoints on + extra_checkpoint_iters: list[int] = field(default_factory=list) + + # log to the console every N iters; this doesn't control wandb logging which is done only on checkpoints + log_interval: int = 1 + + # FIXME: there is a bug in the current implementation, and eval loss is computed on the + # entire dataset. In this implementation, eval_iters controls the number of minibatches + # the dataset is split into for evaluation. + eval_iters: int = 100 + + # path to a checkpoint to resume from + resume_from_path: Optional[str] = None + + # number of samples used to compute the gradient for a single optimizer step + batch_size: int = 64 + + # total number of training epochs + max_epochs: int = 10 + + # clip gradients at this value, or disable if == 0.0 + grad_clip: float = 1.0 + + # if > 1 reduces memory usage by computing gradient in microbatches + gradient_accumulation_steps: int = 1 + + # AdamW optimizer + adam: AdamConfig = field(default_factory=AdamConfig) + + # seed used for pseudorandomly sampling data during training + batch_ordering_seed: int + + # seed used for torch + torch_seed: int + + # whether to save the optimizer state with each checkpoint + # this is twice as large as the model, but allows to resume training in a reproducible way + save_optimizer: bool = True + + # specify training and validation data + dataset: DatasetConfig + + # HF repo id or local directory containing the tokenizer. Used only to upload it to HF with the model, not for training + tokenizer: str = "" + + # wandb config in 'entity/project' form. Set to empty string to not use wandb. + wandb: str + + # HF repo id. Set to empty string to not push to repo. + out_repo: str + + # debug config + debug_config: DebugConfig = field(default_factory=DebugConfig) diff --git a/src/delphi/train/config/utils.py b/delphi/train/config/utils.py similarity index 81% rename from src/delphi/train/config/utils.py rename to delphi/train/config/utils.py index b645cd5e..7fda9cf1 100644 --- a/src/delphi/train/config/utils.py +++ b/delphi/train/config/utils.py @@ -65,28 +65,6 @@ def build_config_dict_from_files(config_files: list[Path]) -> dict[str, Any]: return combined_config -def set_backup_vals(config: dict[str, Any], config_files: list[Path]): - """ - Convenience default values for run_name and output_dir based on config file (if exactly one passed) - - If the user is using 1 config file and has not set a run_name, we set it to the filename. - Likewise for output_dir, we set it to a user-specific directory based on the run_name. - """ - if len(config_files) == 1: - prefix = f"{config_files[0].stem}__" - else: - prefix = "" - if "run_name" not in config: - run_time = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - config["run_name"] = f"{prefix}{run_time}" - logging.info(f"Setting run_name to {config['run_name']}") - if "output_dir" not in config: - config["output_dir"] = os.path.join( - platformdirs.user_data_dir(appname="delphi"), config["run_name"] - ) - logging.info(f"Setting output_dir to {config['output_dir']}") - - def cast_types(config: dict[str, Any], target_dataclass: Type): """ user overrides are passed in as strings, so we need to cast them to the correct type @@ -118,13 +96,11 @@ def build_config_from_files_and_overrides( (we expect this to be passed as strings w/o type hints from a script argument: e.g. `--overrides model_config.hidden_size=42 run_name=foo`) 3. Merge in overrides to config_dict, taking precedence over all config_files values. - 4. Set backup values (for run_name and output_dir) if they are not already set. - 5. Build the TrainingConfig object from the final config dict and return it. + 4. Build the TrainingConfig object from the final config dict and return it. """ combined_config = build_config_dict_from_files(config_files) cast_types(overrides, TrainingConfig) merge_two_dicts(merge_into=combined_config, merge_from=overrides) - set_backup_vals(combined_config, config_files) return from_dict(TrainingConfig, combined_config, config=dacite_config(strict=True)) diff --git a/src/delphi/train/run_context.py b/delphi/train/run_context.py similarity index 100% rename from src/delphi/train/run_context.py rename to delphi/train/run_context.py diff --git a/src/delphi/train/shuffle.py b/delphi/train/shuffle.py similarity index 100% rename from src/delphi/train/shuffle.py rename to delphi/train/shuffle.py diff --git a/src/delphi/train/train_step.py b/delphi/train/train_step.py similarity index 100% rename from src/delphi/train/train_step.py rename to delphi/train/train_step.py diff --git a/src/delphi/train/training.py b/delphi/train/training.py similarity index 93% rename from src/delphi/train/training.py rename to delphi/train/training.py index e1d65adc..575f9844 100644 --- a/src/delphi/train/training.py +++ b/delphi/train/training.py @@ -5,6 +5,7 @@ from pathlib import Path import torch +from huggingface_hub import HfApi from tqdm import tqdm from transformers import AutoTokenizer @@ -25,22 +26,23 @@ def setup_training(config: TrainingConfig): logging.info("Setting up training...") - os.makedirs(config.output_dir, exist_ok=True) + os.makedirs(config.out_dir, exist_ok=True) - # torch misc - TODO: check if this is actually needed torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn - # determinism setup_determinism(config.torch_seed) - # wandb setup + if config.out_repo: + api = HfApi() + api.create_repo(config.out_repo, exist_ok=True) + if config.wandb: - init_wandb(config=config) + init_wandb(config) if config.tokenizer: tokenizer = AutoTokenizer.from_pretrained(config.tokenizer) - tokenizer.save_pretrained(Path(config.output_dir) / "tokenizer") + tokenizer.save_pretrained(Path(config.out_dir) / "tokenizer") def run_training(config: TrainingConfig) -> tuple[ModelTrainingState, RunContext]: diff --git a/src/delphi/train/utils.py b/delphi/train/utils.py similarity index 93% rename from src/delphi/train/utils.py rename to delphi/train/utils.py index 4eadb2c6..af55c0c7 100644 --- a/src/delphi/train/utils.py +++ b/delphi/train/utils.py @@ -17,9 +17,10 @@ from torch.optim import AdamW from transformers import PreTrainedModel +from delphi.train.config import dot_notation_to_dict + from .config import TrainingConfig from .run_context import RunContext -from .shuffle import shuffle_list @dataclass @@ -198,8 +199,8 @@ def save_results( config, context (e.g. hardware), training step, etc """ iter_name = "main" if final else f"iter{train_results.iter_num}" - output_dir = Path(config.output_dir) - results_path = output_dir / iter_name + out_dir = Path(config.out_dir) + results_path = out_dir / iter_name logging.info(f"saving checkpoint to {results_path}") results_path.mkdir(parents=True, exist_ok=True) with open(results_path / "training_config.json", "w") as file: @@ -220,19 +221,18 @@ def save_results( json.dump(training_state_dict, file, indent=2) with open(results_path / "run_context.json", "w") as file: json.dump(run_context.asdict(), file, indent=2) - if (tokenizer_dir := output_dir / "tokenizer").exists(): + if (tokenizer_dir := out_dir / "tokenizer").exists(): for src_file in tokenizer_dir.iterdir(): if src_file.is_file(): dest_file = results_path / src_file.name shutil.copy2(src_file, dest_file) - if config.out_repo_id: + if config.out_repo: try: api = HfApi() - api.create_repo(config.out_repo_id, exist_ok=True) - api.create_branch(config.out_repo_id, branch=iter_name, exist_ok=True) + api.create_branch(config.out_repo, branch=iter_name, exist_ok=True) api.upload_folder( folder_path=results_path, - repo_id=config.out_repo_id, + repo_id=config.out_repo, revision=iter_name, ) except Exception as e: @@ -255,3 +255,9 @@ def init_model(model_config_dict: dict[str, Any], seed: int) -> PreTrainedModel: model_params_dict = model_config_dict.copy() model_params_dict.pop("model_class") return model_class(config_class(**(model_params_dict))) + + +def overrides_to_dict(overrides: list[str]) -> dict[str, Any]: + # ["a.b.c=4", "foo=false"] to {"a": {"b": {"c": 4}}, "foo": False} + config_vars = {k: v for k, v in [x.split("=") for x in overrides if "=" in x]} + return dot_notation_to_dict(config_vars) diff --git a/src/delphi/train/wandb_utils.py b/delphi/train/wandb_utils.py similarity index 68% rename from src/delphi/train/wandb_utils.py rename to delphi/train/wandb_utils.py index 83c53912..f420b5da 100644 --- a/src/delphi/train/wandb_utils.py +++ b/delphi/train/wandb_utils.py @@ -1,5 +1,4 @@ import logging -import os from dataclasses import asdict import wandb @@ -8,19 +7,12 @@ from .utils import ModelTrainingState -def silence_wandb(): - logging.info("silencing wandb output") - os.environ["WANDB_SILENT"] = "true" - - def init_wandb(config: TrainingConfig): - # if log level < debug, silence wandb - assert config.wandb is not None - if logging.getLogger().level > logging.INFO or config.wandb.silence: - silence_wandb() + assert "/" in config.wandb, "wandb should be in the 'entity/project' form" + wandb_entity, wandb_project = config.wandb.split("/") wandb.init( - entity=config.wandb.entity, - project=config.wandb.project, + entity=wandb_entity, + project=wandb_project, name=config.run_name, config=asdict(config), ) diff --git a/src/delphi/utils.py b/delphi/utils.py similarity index 50% rename from src/delphi/utils.py rename to delphi/utils.py index 0ceb059a..15f79545 100644 --- a/src/delphi/utils.py +++ b/delphi/utils.py @@ -1,20 +1,22 @@ +from collections.abc import Callable from typing import cast +import torch from datasets import Dataset, Features, Sequence, Value, load_dataset +from jaxtyping import Float, Int def hf_split_to_split_name(split: str) -> str: return split.split("[")[0] -# TODO: test load_dataset functions def load_dataset_split_features( - repo_id: str, + path: str, split: str, features: Features, ) -> Dataset: dataset = load_dataset( - repo_id, + path, split=split, features=features, ) @@ -23,28 +25,28 @@ def load_dataset_split_features( def load_dataset_split_string_feature( - repo_id: str, + path: str, split: str, feature_name: str, ) -> Dataset: print("Loading string dataset") - print(f"{repo_id=}, {split=}, {feature_name=}") + print(f"{path=}, {split=}, {feature_name=}") return load_dataset_split_features( - repo_id, + path, split, Features({feature_name: Value("string")}), ) def load_dataset_split_sequence_int32_feature( - repo_id: str, + path: str, split: str, feature_name: str, ) -> Dataset: print("Loading sequence int32 dataset") - print(f"{repo_id=}, {split=}, {feature_name=}") + print(f"{path=}, {split=}, {feature_name=}") return load_dataset_split_features( - repo_id, + path, split, Features({feature_name: Sequence(Value("int32"))}), ) @@ -56,3 +58,30 @@ def get_all_hf_branch_names(repo_id: str) -> list[str]: api = HfApi() refs = api.list_repo_refs(repo_id) return [branch.name for branch in refs.branches] + + +def gather_logprobs( + logprobs: Float[torch.Tensor, "batch seq vocab"], + tokens: Int[torch.Tensor, "batch seq"], +) -> Float[torch.Tensor, "batch seq"]: + return torch.gather(logprobs, -1, tokens.unsqueeze(-1)).squeeze(-1) + + +def get_all_logprobs( + model: Callable, input_ids: Int[torch.Tensor, "batch seq"] +) -> Float[torch.Tensor, "batch seq vocab"]: + # batch, seq, vocab + logits = model(input_ids).logits + return torch.log_softmax(logits, dim=-1) + + +def get_all_and_next_logprobs( + model: Callable, + input_ids: Int[torch.Tensor, "batch seq"], +) -> tuple[ + Float[torch.Tensor, "batch shorter_seq vocab"], + Float[torch.Tensor, "batch shorter_seq"], +]: + logprobs = get_all_logprobs(model, input_ids[:, :-1]) + next_tokens = input_ids[:, 1:] + return logprobs, gather_logprobs(logprobs, next_tokens) diff --git a/notebooks/end2end_demo.ipynb b/notebooks/end2end_demo.ipynb deleted file mode 100644 index 3f8e938d..00000000 --- a/notebooks/end2end_demo.ipynb +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from typing import cast\n", - "import pickle\n", - "from collections import defaultdict\n", - "\n", - "from datasets import load_dataset, Dataset\n", - "\n", - "from delphi.constants import STATIC_ASSETS_DIR\n", - "from delphi.eval import utils\n", - "from delphi.eval import constants\n", - "from delphi.eval.vis_per_token_model import visualize_per_token_category\n", - "\n", - "# from delphi.eval.calc_model_group_stats import calc_model_group_stats\n", - "from delphi.eval.spacy_token_labelling import TOKEN_LABELS" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# load data\n", - "tokenized_corpus_dataset = cast(Dataset, load_dataset(\n", - " constants.tokenized_corpus_dataset,\n", - " split=\"validation\"\n", - "))\n", - "\n", - "# TODO: convert to use static paths\n", - "# with open(\"../src/delphi/eval/labelled_token_ids_dict.pkl\", \"rb\") as f:\n", - "# token_groups = pickle.load(f)\n", - "# model_group_stats = calc_model_group_stats(\n", - "# tokenized_corpus_dataset, logprob_datasets, token_groups, token_groups[0].keys()\n", - "# )\n", - "with open(f\"{STATIC_ASSETS_DIR}/model_group_stats.pkl\", \"rb\") as f:\n", - " model_group_stats = pickle.load(f)\n", - "\n", - "logprob_datasets = utils.load_logprob_datasets(\"validation\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Visualization" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d6c18c9588f3499b94e89ccea5954780", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(Dropdown(description='Token Category:', options=('Capitalized', 'Is Determiner', 'Is Interjunct…" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "performance_data = defaultdict(dict)\n", - "for model in constants.LLAMA2_MODELS:\n", - " for token_group_desc in TOKEN_LABELS:\n", - " if (model, token_group_desc) not in model_group_stats:\n", - " continue\n", - " stats = model_group_stats[(model, token_group_desc)]\n", - " performance_data[model][token_group_desc] = (\n", - " -stats[\"median\"],\n", - " -stats[\"75th\"],\n", - " -stats[\"25th\"],\n", - " )\n", - "\n", - "visualize_per_token_category(\n", - " performance_data,\n", - " log_scale=True,\n", - " bg_color=\"LightGrey\",\n", - " line_color=\"Red\",\n", - " marker_color=\"Orange\",\n", - " bar_color=\"Green\",\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "tinyevals", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/eval_notebook.ipynb b/notebooks/eval_notebook.ipynb new file mode 100644 index 00000000..daed1f52 --- /dev/null +++ b/notebooks/eval_notebook.ipynb @@ -0,0 +1,480 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# colab cells (only run if on colab)\n", + "# TODO: experiment on colab to see how to set up the environment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Important\n", + "\n", + "Run this cell by cell. The token selecter cell needs to be ran first so the later cells work." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": "(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n var py_version = '3.4.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n var reloading = false;\n var Bokeh = root.Bokeh;\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks;\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n if (js_modules == null) js_modules = [];\n if (js_exports == null) js_exports = {};\n\n root._bokeh_onload_callbacks.push(callback);\n\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n run_callbacks();\n return null;\n }\n if (!reloading) {\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n }\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n window._bokeh_on_load = on_load\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n var skip = [];\n if (window.requirejs) {\n window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n root._bokeh_is_loading = css_urls.length + 0;\n } else {\n root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n }\n\n var existing_stylesheets = []\n var links = document.getElementsByTagName('link')\n for (var i = 0; i < links.length; i++) {\n var link = links[i]\n if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n }\n }\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n }\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n } var existing_scripts = []\n var scripts = document.getElementsByTagName('script')\n for (var i = 0; i < scripts.length; i++) {\n var script = scripts[i]\n if (script.src != null) {\n\texisting_scripts.push(script.src)\n }\n }\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (var i = 0; i < js_modules.length; i++) {\n var url = js_modules[i];\n if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n for (const name in js_exports) {\n var url = js_exports[name];\n if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t on_load();\n\t}\n\tcontinue;\n }\n var element = document.createElement('script');\n element.onerror = on_error;\n element.async = false;\n element.type = \"module\";\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n element.textContent = `\n import ${name} from \"${url}\"\n window.${name} = ${name}\n window._bokeh_on_load()\n `\n document.head.appendChild(element);\n }\n if (!js_urls.length && !js_modules.length) {\n on_load()\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.4.1.min.js\", \"https://cdn.holoviz.org/panel/1.4.0/dist/panel.min.js\"];\n var js_modules = [];\n var js_exports = {};\n var css_urls = [\"https://cdn.holoviz.org/panel/1.4.0/dist/bundled/font-awesome/css/all.min.css\"];\n var inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n ];\n\n function run_inline_js() {\n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n\ttry {\n inline_js[i].call(root, root.Bokeh);\n\t} catch(e) {\n\t if (!reloading) {\n\t throw e;\n\t }\n\t}\n }\n // Cache old bokeh versions\n if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n }\n root._bokeh_is_initializing = false\n }\n\n function load_or_wait() {\n // Implement a backoff loop that tries to ensure we do not load multiple\n // versions of Bokeh and its dependencies at the same time.\n // In recent versions we use the root._bokeh_is_initializing flag\n // to determine whether there is an ongoing attempt to initialize\n // bokeh, however for backward compatibility we also try to ensure\n // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n // before older versions are fully initialized.\n if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n root._bokeh_is_initializing = false;\n root._bokeh_onload_callbacks = undefined;\n console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n load_or_wait();\n } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n setTimeout(load_or_wait, 100);\n } else {\n root._bokeh_is_initializing = true\n root._bokeh_onload_callbacks = []\n var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n if (!reloading && !bokeh_loaded) {\n\troot.Bokeh = undefined;\n }\n load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n });\n }\n }\n // Give older versions of the autoload script a head-start to ensure\n // they initialize before we start loading newer version.\n setTimeout(load_or_wait, 100)\n}(window));", + "application/vnd.holoviews_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "\nif ((window.PyViz === undefined) || (window.PyViz instanceof HTMLElement)) {\n window.PyViz = {comms: {}, comm_status:{}, kernels:{}, receivers: {}, plot_index: []}\n}\n\n\n function JupyterCommManager() {\n }\n\n JupyterCommManager.prototype.register_target = function(plot_id, comm_id, msg_handler) {\n if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n comm_manager.register_target(comm_id, function(comm) {\n comm.on_msg(msg_handler);\n });\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n window.PyViz.kernels[plot_id].registerCommTarget(comm_id, function(comm) {\n comm.onMsg = msg_handler;\n });\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n google.colab.kernel.comms.registerTarget(comm_id, (comm) => {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n console.log(message)\n var content = {data: message.data, comm_id};\n var buffers = []\n for (var buffer of message.buffers || []) {\n buffers.push(new DataView(buffer))\n }\n var metadata = message.metadata || {};\n var msg = {content, buffers, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n })\n }\n }\n\n JupyterCommManager.prototype.get_client_comm = function(plot_id, comm_id, msg_handler) {\n if (comm_id in window.PyViz.comms) {\n return window.PyViz.comms[comm_id];\n } else if (window.comm_manager || ((window.Jupyter !== undefined) && (Jupyter.notebook.kernel != null))) {\n var comm_manager = window.comm_manager || Jupyter.notebook.kernel.comm_manager;\n var comm = comm_manager.new_comm(comm_id, {}, {}, {}, comm_id);\n if (msg_handler) {\n comm.on_msg(msg_handler);\n }\n } else if ((plot_id in window.PyViz.kernels) && (window.PyViz.kernels[plot_id])) {\n var comm = window.PyViz.kernels[plot_id].connectToComm(comm_id);\n comm.open();\n if (msg_handler) {\n comm.onMsg = msg_handler;\n }\n } else if (typeof google != 'undefined' && google.colab.kernel != null) {\n var comm_promise = google.colab.kernel.comms.open(comm_id)\n comm_promise.then((comm) => {\n window.PyViz.comms[comm_id] = comm;\n if (msg_handler) {\n var messages = comm.messages[Symbol.asyncIterator]();\n function processIteratorResult(result) {\n var message = result.value;\n var content = {data: message.data};\n var metadata = message.metadata || {comm_id};\n var msg = {content, metadata}\n msg_handler(msg);\n return messages.next().then(processIteratorResult);\n }\n return messages.next().then(processIteratorResult);\n }\n }) \n var sendClosure = (data, metadata, buffers, disposeOnDone) => {\n return comm_promise.then((comm) => {\n comm.send(data, metadata, buffers, disposeOnDone);\n });\n };\n var comm = {\n send: sendClosure\n };\n }\n window.PyViz.comms[comm_id] = comm;\n return comm;\n }\n window.PyViz.comm_manager = new JupyterCommManager();\n \n\n\nvar JS_MIME_TYPE = 'application/javascript';\nvar HTML_MIME_TYPE = 'text/html';\nvar EXEC_MIME_TYPE = 'application/vnd.holoviews_exec.v0+json';\nvar CLASS_NAME = 'output';\n\n/**\n * Render data to the DOM node\n */\nfunction render(props, node) {\n var div = document.createElement(\"div\");\n var script = document.createElement(\"script\");\n node.appendChild(div);\n node.appendChild(script);\n}\n\n/**\n * Handle when a new output is added\n */\nfunction handle_add_output(event, handle) {\n var output_area = handle.output_area;\n var output = handle.output;\n if ((output.data == undefined) || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n return\n }\n var id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n if (id !== undefined) {\n var nchildren = toinsert.length;\n var html_node = toinsert[nchildren-1].children[0];\n html_node.innerHTML = output.data[HTML_MIME_TYPE];\n var scripts = [];\n var nodelist = html_node.querySelectorAll(\"script\");\n for (var i in nodelist) {\n if (nodelist.hasOwnProperty(i)) {\n scripts.push(nodelist[i])\n }\n }\n\n scripts.forEach( function (oldScript) {\n var newScript = document.createElement(\"script\");\n var attrs = [];\n var nodemap = oldScript.attributes;\n for (var j in nodemap) {\n if (nodemap.hasOwnProperty(j)) {\n attrs.push(nodemap[j])\n }\n }\n attrs.forEach(function(attr) { newScript.setAttribute(attr.name, attr.value) });\n newScript.appendChild(document.createTextNode(oldScript.innerHTML));\n oldScript.parentNode.replaceChild(newScript, oldScript);\n });\n if (JS_MIME_TYPE in output.data) {\n toinsert[nchildren-1].children[1].textContent = output.data[JS_MIME_TYPE];\n }\n output_area._hv_plot_id = id;\n if ((window.Bokeh !== undefined) && (id in Bokeh.index)) {\n window.PyViz.plot_index[id] = Bokeh.index[id];\n } else {\n window.PyViz.plot_index[id] = null;\n }\n } else if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n var bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n var script_attrs = bk_div.children[0].attributes;\n for (var i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].childNodes[1].setAttribute(script_attrs[i].name, script_attrs[i].value);\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n}\n\n/**\n * Handle when an output is cleared or removed\n */\nfunction handle_clear_output(event, handle) {\n var id = handle.cell.output_area._hv_plot_id;\n var server_id = handle.cell.output_area._bokeh_server_id;\n if (((id === undefined) || !(id in PyViz.plot_index)) && (server_id !== undefined)) { return; }\n var comm = window.PyViz.comm_manager.get_client_comm(\"hv-extension-comm\", \"hv-extension-comm\", function () {});\n if (server_id !== null) {\n comm.send({event_type: 'server_delete', 'id': server_id});\n return;\n } else if (comm !== null) {\n comm.send({event_type: 'delete', 'id': id});\n }\n delete PyViz.plot_index[id];\n if ((window.Bokeh !== undefined) & (id in window.Bokeh.index)) {\n var doc = window.Bokeh.index[id].model.document\n doc.clear();\n const i = window.Bokeh.documents.indexOf(doc);\n if (i > -1) {\n window.Bokeh.documents.splice(i, 1);\n }\n }\n}\n\n/**\n * Handle kernel restart event\n */\nfunction handle_kernel_cleanup(event, handle) {\n delete PyViz.comms[\"hv-extension-comm\"];\n window.PyViz.plot_index = {}\n}\n\n/**\n * Handle update_display_data messages\n */\nfunction handle_update_output(event, handle) {\n handle_clear_output(event, {cell: {output_area: handle.output_area}})\n handle_add_output(event, handle)\n}\n\nfunction register_renderer(events, OutputArea) {\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n var toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[0]);\n element.append(toinsert);\n return toinsert\n }\n\n events.on('output_added.OutputArea', handle_add_output);\n events.on('output_updated.OutputArea', handle_update_output);\n events.on('clear_output.CodeCell', handle_clear_output);\n events.on('delete.Cell', handle_clear_output);\n events.on('kernel_ready.Kernel', handle_kernel_cleanup);\n\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n safe: true,\n index: 0\n });\n}\n\nif (window.Jupyter !== undefined) {\n try {\n var events = require('base/js/events');\n var OutputArea = require('notebook/js/outputarea').OutputArea;\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n } catch(err) {\n }\n}\n", + "application/vnd.holoviews_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.holoviews_exec.v0+json": "", + "text/html": [ + "
\n", + "
\n", + "
\n", + "" + ] + }, + "metadata": { + "application/vnd.holoviews_exec.v0+json": { + "id": "2ac0543f-58a1-4b93-9570-a2c0e6f09501" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "# imports\n", + "import torch\n", + "import panel as pn\n", + "from delphi.eval import token_selector, vis_pos_map, calc_model_group_stats, visualize_selected_tokens, get_all_tok_metrics_in_label\n", + "from datasets import load_dataset, Dataset\n", + "from transformers import AutoTokenizer\n", + "from typing import cast\n", + "import ipywidgets as widgets\n", + "\n", + "# refer to https://panel.holoviz.org/reference/panes/IPyWidget.html to integrate ipywidgets with panel\n", + "pn.extension('ipywidgets')\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# specify model names (or checkpoints)\n", + "prefix = \"delphi-suite/v0-next-logprobs-llama2-\"\n", + "suffixes = [\n", + " \"100k\",\n", + " \"200k\",\n", + " \"400k\",\n", + "] # , \"800k\", \"1.6m\", \"3.2m\", \"6.4m\", \"12.8m\", \"25.6m\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# load next logprobs data for all models\n", + "split = \"validation[:100]\"\n", + "next_logprobs = {\n", + " suffix: cast(\n", + " Dataset,\n", + " load_dataset(f\"{prefix}{suffix}\", split=split),\n", + " )\n", + " .with_format(\"torch\")\n", + " .map(lambda x: {\"logprobs\": x[\"logprobs\"].to(device)})\n", + " for suffix in suffixes\n", + "}\n", + "next_logprobs_plot = {k: d[\"logprobs\"] for k, d in next_logprobs.items()}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# load the tokenized dataset\n", + "tokenized_corpus_dataset = (\n", + " cast(\n", + " Dataset,\n", + " load_dataset(\"delphi-suite/stories-tokenized\", split=split),\n", + " )\n", + " .with_format(\"torch\")\n", + " .map(lambda x: {\"tokens\": x[\"tokens\"].to(device)})\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run this notebook until the following cell, then the rest should work." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jett/Documents/jett/delphi/.venv/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "29c01ed2f022418ebc6a7c4f2d8210b4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "BokehModel(combine_events=True, render_bundle={'docs_json': {'f8a47e67-7cc8-4e1f-bc8e-c10325c540a2': {'version…" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# specific token specification\n", + "tokenizer = AutoTokenizer.from_pretrained(\"delphi-suite/stories-tokenizer\")\n", + "\n", + "# Count the frequency of each token using torch.bincount\n", + "token_counts = torch.bincount(tokenized_corpus_dataset[\"tokens\"].view(-1))\n", + "\n", + "# Get the indices that would sort the token counts in descending order\n", + "sorted_indices = torch.argsort(token_counts, descending=True)\n", + "\n", + "# Get the token IDs in descending order of frequency\n", + "valid_tok_ids = sorted_indices.tolist()\n", + "def format_fix(s):\n", + " if s.startswith(\" \"):\n", + " return \"_\" + s[1:]\n", + " return s\n", + "vocab = {format_fix(tokenizer.decode(t, clean_up_tokenization_spaces=True)): t for t in sorted_indices.tolist() if token_counts[t] > 0}\n", + "\n", + "\n", + "selector, selected_ids = token_selector(vocab) # use selected_ids as a dynamic variable\n", + "pn.Row(selector, height=500).servable()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Selected IDs: [40, 2, 14]\n" + ] + } + ], + "source": [ + "if not selected_ids:\n", + " selected_ids = [40, 2, 14]\n", + "print(\"Selected IDs:\", selected_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([100, 512])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(next_logprobs_plot.values())[0].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing model 100k\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing model 200k\n", + "Processing model 400k\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2084f5f7ca5e4aeca85b0f5c821eaced", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FigureWidget({\n", + " 'data': [{'line': {'width': 0},\n", + " 'marker': {'color': 'rgba(68, 68, 68, 0.3)'},\n", + " 'mode': 'lines',\n", + " 'name': 'Upper Bound',\n", + " 'showlegend': False,\n", + " 'type': 'scatter',\n", + " 'uid': '3348f11d-9719-4274-9954-43a9dc8f2ce1',\n", + " 'x': [100k, 200k, 400k],\n", + " 'y': array([4.6017912 , 4.03893679, 3.46496367])},\n", + " {'fill': 'tonexty',\n", + " 'fillcolor': 'rgba(68, 68, 68, 0.3)',\n", + " 'line': {'width': 0},\n", + " 'marker': {'color': 'rgba(68, 68, 68, 0.3)'},\n", + " 'mode': 'lines',\n", + " 'name': 'Lower Bound',\n", + " 'showlegend': False,\n", + " 'type': 'scatter',\n", + " 'uid': '2a90892b-69a9-49d6-bad0-086ee4837fcc',\n", + " 'x': [100k, 200k, 400k],\n", + " 'y': array([1.00667199, 0.88813308, 0.735852 ])},\n", + " {'marker': {'color': 'rgb(31, 119, 180)', 'line': {'color': 'rgb(31, 119, 180)', 'width': 1}, 'size': 0},\n", + " 'mode': 'lines',\n", + " 'name': 'Means',\n", + " 'type': 'scatter',\n", + " 'uid': '24ce8dc7-90cf-48f7-9722-de6cc02ba5a1',\n", + " 'x': [100k, 200k, 400k],\n", + " 'y': array([1.39094847, 1.15670866, 0.93012363])}],\n", + " 'layout': {'template': '...'}\n", + "})" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_group_stats = calc_model_group_stats( # i'm not sure if tokenized_corpus_dataset.tolist() is the right input, it was list(tokenized_corpus_dataset) before\n", + " tokenized_corpus_dataset, next_logprobs_plot, selected_ids\n", + ")\n", + "performance_data = {}\n", + "for suffix in suffixes:\n", + " stats = model_group_stats[suffix]\n", + " performance_data[suffix] = (\n", + " -stats[\"median\"],\n", + " -stats[\"75th\"],\n", + " -stats[\"25th\"],\n", + " )\n", + "\n", + "visualize_selected_tokens(performance_data, log_scale=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3dee1d95570945f3a119c830cdd6d9b6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "interactive(children=(FloatRangeSlider(value=(0.25, 0.75), description='Quantiles', max=1.0, step=0.05), Dropd…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def show_pos_map(\n", + " quantile: tuple[float, float],\n", + " model_name_1: str,\n", + " model_name_2: str,\n", + "):\n", + " logprobs_diff = next_logprobs[model_name_2][\"logprobs\"] - next_logprobs[model_name_1][\"logprobs\"] # type: ignore\n", + " pos_to_diff = get_all_tok_metrics_in_label(tokenized_corpus_dataset[\"tokens\"], selected_tokens=selected_ids, metrics=logprobs_diff, q_start=quantile[0], q_end=quantile[1]) # type: ignore\n", + " try:\n", + " _ = vis_pos_map(list(pos_to_diff.keys()), selected_ids, logprobs_diff, tokenized_corpus_dataset[\"tokens\"], tokenizer) # type: ignore\n", + " except ValueError:\n", + " if pos_to_diff == {}:\n", + " print(\"No tokens found in this label\")\n", + " return\n", + "\n", + "\n", + "widgets.interact_manual(\n", + " show_pos_map,\n", + " quantile=widgets.FloatRangeSlider(\n", + " min=0.0, max=1.0, step=0.05, description=\"Quantiles\"\n", + " ),\n", + " samples=widgets.IntSlider(min=1, max=5, description=\"Samples\", value=2),\n", + " model_name_1=widgets.Dropdown(\n", + " options=suffixes,\n", + " description=\"Model 1\",\n", + " value=\"100k\",\n", + " ),\n", + " model_name_2=widgets.Dropdown(\n", + " options=suffixes,\n", + " description=\"Model 2\",\n", + " value=\"200k\",\n", + " ),\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/model_diff.ipynb b/notebooks/model_diff.ipynb deleted file mode 100644 index c3ea4429..00000000 --- a/notebooks/model_diff.ipynb +++ /dev/null @@ -1,171 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import pickle\n", - "\n", - "\n", - "from datasets import load_dataset, Dataset\n", - "\n", - "\n", - "from typing import cast\n", - "from ipywidgets import interact\n", - "import ipywidgets as widgets\n", - "\n", - "\n", - "from transformers import AutoTokenizer\n", - "from delphi.constants import STATIC_ASSETS_DIR\n", - "from delphi.eval.token_positions import get_all_tok_metrics_in_label\n", - "from delphi.eval.vis import vis_pos_map\n", - "from delphi.eval.constants import LLAMA2_NEXT_LOGPROBS_DATASETS_MAP\n", - "\n", - "# from delphi.train.utils import get_device\n", - "\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer = AutoTokenizer.from_pretrained(\"delphi-suite/stories-tokenizer\")\n", - "token_ids = (\n", - " cast(\n", - " Dataset,\n", - " load_dataset(\n", - " \"delphi-suite/v0-tinystories-v2-clean-tokenized\", split=\"validation\"\n", - " ),\n", - " )\n", - " .with_format(\"torch\")\n", - " .map(lambda x: {\"tokens\": x[\"tokens\"].to(device)})\n", - ")\n", - "\n", - "next_logprobs = { # preloading all the logprobs datasets for interactive use\n", - " model_name: (\n", - " cast(\n", - " Dataset,\n", - " load_dataset(f\"{dataset_name}\", split=\"validation\"),\n", - " )\n", - " .with_format(\"torch\")\n", - " .map(lambda x: {\"logprobs\": x[\"logprobs\"].to(device)})\n", - " )\n", - " for model_name, dataset_name in LLAMA2_NEXT_LOGPROBS_DATASETS_MAP.items()\n", - "}\n", - "\n", - "token_labels_filename = \"labelled_token_ids_dict.pkl\"\n", - "with open(f\"{STATIC_ASSETS_DIR.joinpath(token_labels_filename)}\", \"rb\") as f:\n", - " token_labels = pickle.load(f)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8e6f7079bf3b43bcb4b1afb904b36d11", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(FloatRangeSlider(value=(0.25, 0.75), description='Start quantile', max=1.0, step=0.01), …" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def show_pos_map(\n", - " quantile: tuple[float, float],\n", - " model_name_1: str,\n", - " model_name_2: str,\n", - " label: str,\n", - " samples: int,\n", - "):\n", - " token_id_t = token_ids[\"tokens\"]\n", - " logprobs_diff = next_logprobs[model_name_2][\"logprobs\"] - next_logprobs[model_name_1][\"logprobs\"] # type: ignore\n", - " pos_to_diff = get_all_tok_metrics_in_label(token_id_t, token_labels=token_labels, metrics=logprobs_diff, label=label, q_start=quantile[0], q_end=quantile[1]) # type: ignore\n", - " try:\n", - " _ = vis_pos_map(pos_to_diff, token_id_t, tokenizer, sample=samples) # type: ignore\n", - " except ValueError:\n", - " print(\"No tokens found in this label\")\n", - " return\n", - "\n", - "\n", - "interact(\n", - " show_pos_map,\n", - " quantile=widgets.FloatRangeSlider(\n", - " min=0.0, max=1.0, step=0.01, description=\"Start quantile\"\n", - " ),\n", - " samples=widgets.IntSlider(min=1, max=5, description=\"Samples\", value=2),\n", - " model_name_1=widgets.Dropdown(\n", - " options=LLAMA2_NEXT_LOGPROBS_DATASETS_MAP.keys(),\n", - " description=\"Model 1\",\n", - " value=\"llama2-100k\",\n", - " ),\n", - " model_name_2=widgets.Dropdown(\n", - " options=LLAMA2_NEXT_LOGPROBS_DATASETS_MAP.keys(),\n", - " description=\"Model 2\",\n", - " value=\"llama2-200k\",\n", - " ),\n", - " label=widgets.Dropdown(\n", - " options=token_labels[0].keys(), description=\"Label\", value=\"Is Noun\"\n", - " ),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/per_token_plot.ipynb b/notebooks/per_token_plot.ipynb deleted file mode 100644 index 12e09926..00000000 --- a/notebooks/per_token_plot.ipynb +++ /dev/null @@ -1,165 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fbda6a916fe84814be64a40423196d76", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FigureWidget({\n", - " 'data': [{'line': {'width': 0},\n", - " 'marker': {'color': 'rgba(68, 68, 68, 0.3)'},\n", - " 'mode': 'lines',\n", - " 'name': 'Upper Bound',\n", - " 'showlegend': False,\n", - " 'type': 'scatter',\n", - " 'uid': 'a3590fcd-466d-4a73-b167-194ab728efcd',\n", - " 'x': [0, 1, 2, ..., 497, 498, 499],\n", - " 'y': array([2.34006592, 2.41241021, 2.57781922, ..., 2.56474203, 2.59573629,\n", - " 2.43304471])},\n", - " {'fill': 'tonexty',\n", - " 'fillcolor': 'rgba(68, 68, 68, 0.3)',\n", - " 'line': {'width': 0},\n", - " 'marker': {'color': 'rgba(68, 68, 68, 0.3)'},\n", - " 'mode': 'lines',\n", - " 'name': 'Lower Bound',\n", - " 'showlegend': False,\n", - " 'type': 'scatter',\n", - " 'uid': 'fda82808-c8ff-4b6c-878d-c76d66c8ce17',\n", - " 'x': [0, 1, 2, ..., 497, 498, 499],\n", - " 'y': array([0.93626447, 0.9302987 , 0.99836227, ..., 0.95607835, 0.76146911,\n", - " 0.81709211])},\n", - " {'marker': {'color': 'rgb(31, 119, 180)', 'line': {'color': 'rgb(31, 119, 180)', 'width': 1}, 'size': 0},\n", - " 'mode': 'lines',\n", - " 'name': 'Means',\n", - " 'type': 'scatter',\n", - " 'uid': 'b11dfbd0-c130-4a97-a8ff-b8c753b95035',\n", - " 'x': [0, 1, 2, ..., 497, 498, 499],\n", - " 'y': array([1.3701917 , 1.4372206 , 1.53251235, ..., 1.55583357, 1.50179179,\n", - " 1.45715223])}],\n", - " 'layout': {'template': '...'}\n", - "})" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from collections import defaultdict\n", - "import math\n", - "import random\n", - "import numpy as np\n", - "\n", - "from delphi.eval.vis_per_token_model import visualize_per_token_category\n", - "\n", - "\n", - "random.seed(0)\n", - "\n", - "# generate mock data\n", - "# model_names = ['llama2-100k', 'llama2-200k', 'llama2-1m', 'llama2-10m', \"0\"]\n", - "model_names = list(range(500))\n", - "categories = ['nouns', 'verbs', 'prepositions', 'adjectives']\n", - "entries = [200, 100, 150, 300, 100]*100\n", - "performance_data = defaultdict()\n", - "for i, model in enumerate(model_names):\n", - " performance_data[model] = defaultdict()\n", - " for cat in categories:\n", - " x = [math.log2(random.random()) for _ in range(entries[i])]\n", - " means = np.mean(x)\n", - " err_low = means - np.percentile(x, 25)\n", - " err_hi = np.percentile(x, 75) - means\n", - " performance_data[model][cat] = (-means, err_low, err_hi)\n", - "\n", - "\n", - "visualize_per_token_category(performance_data, log_scale=True, checkpoint_mode=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "993e5d66ae56462a8eeec2c9ac6bd972", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FigureWidget({\n", - " 'data': [{'line': {'width': 0},\n", - " 'marker': {'color': 'wheat'},\n", - " 'mode': 'lines',\n", - " 'name': 'Upper Bound',\n", - " 'showlegend': False,\n", - " 'type': 'scatter',\n", - " 'uid': '56999008-205c-4592-a3f7-ea61e3e09d8e',\n", - " 'x': [0, 1, 2, ..., 497, 498, 499],\n", - " 'y': array([2.34006592, 2.41241021, 2.57781922, ..., 2.56474203, 2.59573629,\n", - " 2.43304471])},\n", - " {'fill': 'tonexty',\n", - " 'fillcolor': 'wheat',\n", - " 'line': {'width': 0},\n", - " 'marker': {'color': 'wheat'},\n", - " 'mode': 'lines',\n", - " 'name': 'Lower Bound',\n", - " 'showlegend': False,\n", - " 'type': 'scatter',\n", - " 'uid': 'be8a04f1-b8c4-46af-bf5e-03c942eff19f',\n", - " 'x': [0, 1, 2, ..., 497, 498, 499],\n", - " 'y': array([0.93626447, 0.9302987 , 0.99836227, ..., 0.95607835, 0.76146911,\n", - " 0.81709211])},\n", - " {'marker': {'color': 'Orange', 'line': {'color': 'Orange', 'width': 1}, 'size': 0},\n", - " 'mode': 'lines',\n", - " 'name': 'Median',\n", - " 'type': 'scatter',\n", - " 'uid': '85fe5113-70fb-4aa7-9821-947287d84e1d',\n", - " 'x': [0, 1, 2, ..., 497, 498, 499],\n", - " 'y': array([1.3701917 , 1.4372206 , 1.53251235, ..., 1.55583357, 1.50179179,\n", - " 1.45715223])}],\n", - " 'layout': {'template': '...'}\n", - "})" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "visualize_per_token_category(performance_data, log_scale=True, checkpoint_mode=True, line_metric=\"Median\", line_color='Orange' , shade_color=\"wheat\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/token_labelling.ipynb b/notebooks/token_labelling.ipynb deleted file mode 100644 index a0c8cf48..00000000 --- a/notebooks/token_labelling.ipynb +++ /dev/null @@ -1,760 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Giving tokens a label - How to categorize tokens\n", - "\n", - "\n", - "The first part of this Notebook contains elements that explain how to label tokens and how the functions work.\n", - "\n", - "The second part shows how all tokens are labelled that are used for our delphi language models.3\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], - "source": [ - "# autoreload\n", - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "from pprint import pprint \n", - "\n", - "import spacy\n", - "from tqdm.auto import tqdm\n", - "\n", - "import delphi\n", - "\n", - "from delphi.eval import token_labelling" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "# 1) How to use the token labelling functions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We analyze a simple sentence and receive the respective tokens with their analyzed attributes. \n", - "The grammatical/linguistic analysis is done by a model provided by spaCy for the English language." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Peter \t PROPN \t nsubj \t PERSON\n", - "is \t AUX \t ROOT \t \n", - "a \t DET \t det \t \n", - "person \t NOUN \t attr \t \n" - ] - } - ], - "source": [ - "# Load the english model\n", - "nlp = spacy.load(\"en_core_web_sm\")\n", - "\n", - "# Create a Doc object from a given text\n", - "doc = nlp(\"Peter is a person\")\n", - "\n", - "token = doc[0]\n", - "for tok in doc:\n", - " print(tok,\"\\t\", tok.pos_, \"\\t\", tok.dep_, \"\\t\", tok.ent_type_)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's get the label for our custom token that we just printed." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Capitalized': True,\n", - " 'Is Adjective': False,\n", - " 'Is Adposition': False,\n", - " 'Is Adverb': False,\n", - " 'Is Auxiliary': False,\n", - " 'Is Coordinating conjuction': False,\n", - " 'Is Determiner': False,\n", - " 'Is Interjunction': False,\n", - " 'Is Named Entity': True,\n", - " 'Is Noun': False,\n", - " 'Is Numeral': False,\n", - " 'Is Other': False,\n", - " 'Is Particle': False,\n", - " 'Is Pronoun': False,\n", - " 'Is Proper Noun': True,\n", - " 'Is Punctuation': False,\n", - " 'Is Subordinating conjuction': False,\n", - " 'Is Symbol': False,\n", - " 'Is Verb': False,\n", - " 'Starts with space': False}\n" - ] - } - ], - "source": [ - "from delphi.eval import token_labelling\n", - "\n", - "label = token_labelling.label_single_token(token)\n", - "pprint(label)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's get an understanding of what the labels acutally mean.\n", - "Use this function to receive an explanation for a single token." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "-------- Explanation of token labels --------\n", - "Token text: Peter\n", - "Token dependency: nominal subject\n", - "Token POS: proper noun\n", - "---------------- Token labels ---------------\n", - " 0 Starts with space False\n", - " 1 Capitalized True\n", - " 2 Is Adjective False\n", - " 3 Is Adposition False\n", - " 4 Is Adverb False\n", - " 5 Is Auxiliary False\n", - " 6 Is Coordinating conjuction False\n", - " 7 Is Determiner False\n", - " 8 Is Interjunction False\n", - " 9 Is Noun False\n", - " 10 Is Numeral False\n", - " 11 Is Particle False\n", - " 12 Is Pronoun False\n", - " 13 Is Proper Noun True\n", - " 14 Is Punctuation False\n", - " 15 Is Subordinating conjuction False\n", - " 16 Is Symbol False\n", - " 17 Is Verb False\n", - " 18 Is Other False\n", - " 19 Is Named Entity True\n" - ] - } - ], - "source": [ - "token_labelling.explain_token_labels(token)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you are interested in all the possible labels a token can have, that spaCy is capable of assigning, then call the same function but without any argument:\n", - "```Python\n", - ">>> token_labelling.explain_token_labels()\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Batched token labelling\n", - "Next, let us analyze a batch of sentences and have them labelled.\n", - "> In the example below the input sentences are not yet tokenized, so spaCy uses its internal tokenizer." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Token: Peter\n", - "Starts with space | Capitalized | Is Adjective | Is Adposition | Is Adverb | Is Auxiliary | Is Coordinating conjuction | Is Determiner | Is Interjunction | Is Noun | Is Numeral | Is Particle | Is Pronoun | Is Proper Noun | Is Punctuation | Is Subordinating conjuction | Is Symbol | Is Verb | Is Other | Is Named Entity\n", - "False | True | False | False | False | False | False | False | False | False | False | False | False | True | False | False | False | False | False | True \n", - "---\n", - "Token: is\n", - "Starts with space | Capitalized | Is Adjective | Is Adposition | Is Adverb | Is Auxiliary | Is Coordinating conjuction | Is Determiner | Is Interjunction | Is Noun | Is Numeral | Is Particle | Is Pronoun | Is Proper Noun | Is Punctuation | Is Subordinating conjuction | Is Symbol | Is Verb | Is Other | Is Named Entity\n", - "False | False | False | False | False | True | False | False | False | False | False | False | False | False | False | False | False | False | False | False \n", - "---\n", - "Token: a\n", - "Starts with space | Capitalized | Is Adjective | Is Adposition | Is Adverb | Is Auxiliary | Is Coordinating conjuction | Is Determiner | Is Interjunction | Is Noun | Is Numeral | Is Particle | Is Pronoun | Is Proper Noun | Is Punctuation | Is Subordinating conjuction | Is Symbol | Is Verb | Is Other | Is Named Entity\n", - "False | False | False | False | False | False | False | True | False | False | False | False | False | False | False | False | False | False | False | False \n", - "---\n", - "Token: person\n", - "Starts with space | Capitalized | Is Adjective | Is Adposition | Is Adverb | Is Auxiliary | Is Coordinating conjuction | Is Determiner | Is Interjunction | Is Noun | Is Numeral | Is Particle | Is Pronoun | Is Proper Noun | Is Punctuation | Is Subordinating conjuction | Is Symbol | Is Verb | Is Other | Is Named Entity\n", - "False | False | False | False | False | False | False | False | False | True | False | False | False | False | False | False | False | False | False | False \n", - "---\n", - "Token: .\n", - "Starts with space | Capitalized | Is Adjective | Is Adposition | Is Adverb | Is Auxiliary | Is Coordinating conjuction | Is Determiner | Is Interjunction | Is Noun | Is Numeral | Is Particle | Is Pronoun | Is Proper Noun | Is Punctuation | Is Subordinating conjuction | Is Symbol | Is Verb | Is Other | Is Named Entity\n", - "False | False | False | False | False | False | False | False | False | False | False | False | False | False | True | False | False | False | False | False \n", - "---\n", - "\n", - "\n", - "5\n", - "[{'Starts with space': False, 'Capitalized': True, 'Is Adjective': False, 'Is Adposition': False, 'Is Adverb': False, 'Is Auxiliary': False, 'Is Coordinating conjuction': False, 'Is Determiner': False, 'Is Interjunction': False, 'Is Noun': False, 'Is Numeral': False, 'Is Particle': False, 'Is Pronoun': False, 'Is Proper Noun': True, 'Is Punctuation': False, 'Is Subordinating conjuction': False, 'Is Symbol': False, 'Is Verb': False, 'Is Other': False, 'Is Named Entity': True}, {'Starts with space': False, 'Capitalized': False, 'Is Adjective': False, 'Is Adposition': False, 'Is Adverb': False, 'Is Auxiliary': True, 'Is Coordinating conjuction': False, 'Is Determiner': False, 'Is Interjunction': False, 'Is Noun': False, 'Is Numeral': False, 'Is Particle': False, 'Is Pronoun': False, 'Is Proper Noun': False, 'Is Punctuation': False, 'Is Subordinating conjuction': False, 'Is Symbol': False, 'Is Verb': False, 'Is Other': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Adjective': False, 'Is Adposition': False, 'Is Adverb': False, 'Is Auxiliary': False, 'Is Coordinating conjuction': False, 'Is Determiner': True, 'Is Interjunction': False, 'Is Noun': False, 'Is Numeral': False, 'Is Particle': False, 'Is Pronoun': False, 'Is Proper Noun': False, 'Is Punctuation': False, 'Is Subordinating conjuction': False, 'Is Symbol': False, 'Is Verb': False, 'Is Other': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Adjective': False, 'Is Adposition': False, 'Is Adverb': False, 'Is Auxiliary': False, 'Is Coordinating conjuction': False, 'Is Determiner': False, 'Is Interjunction': False, 'Is Noun': True, 'Is Numeral': False, 'Is Particle': False, 'Is Pronoun': False, 'Is Proper Noun': False, 'Is Punctuation': False, 'Is Subordinating conjuction': False, 'Is Symbol': False, 'Is Verb': False, 'Is Other': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Adjective': False, 'Is Adposition': False, 'Is Adverb': False, 'Is Auxiliary': False, 'Is Coordinating conjuction': False, 'Is Determiner': False, 'Is Interjunction': False, 'Is Noun': False, 'Is Numeral': False, 'Is Particle': False, 'Is Pronoun': False, 'Is Proper Noun': False, 'Is Punctuation': True, 'Is Subordinating conjuction': False, 'Is Symbol': False, 'Is Verb': False, 'Is Other': False, 'Is Named Entity': False}]\n" - ] - } - ], - "source": [ - "sentences = [\n", - " \"Peter is a person.\"\n", - "]\n", - "labels = token_labelling.label_batch_sentences(sentences, tokenized=False, verbose=True)\n", - "\n", - "print(len(labels[0]))\n", - "print(labels[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now with our own tokenization. E.g. the one from our TinyStories models." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "5\n", - "[{'Starts with space': False, 'Capitalized': True, 'Is Noun': True, 'Is Pronoun': False, 'Is Adjective': False, 'Is Verb': False, 'Is Adverb': False, 'Is Preposition': False, 'Is Conjunction': False, 'Is Interjunction': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Noun': False, 'Is Pronoun': False, 'Is Adjective': False, 'Is Verb': False, 'Is Adverb': True, 'Is Preposition': False, 'Is Conjunction': False, 'Is Interjunction': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Noun': False, 'Is Pronoun': False, 'Is Adjective': True, 'Is Verb': False, 'Is Adverb': False, 'Is Preposition': False, 'Is Conjunction': False, 'Is Interjunction': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Noun': True, 'Is Pronoun': False, 'Is Adjective': False, 'Is Verb': False, 'Is Adverb': False, 'Is Preposition': False, 'Is Conjunction': False, 'Is Interjunction': False, 'Is Named Entity': False}, {'Starts with space': False, 'Capitalized': False, 'Is Noun': False, 'Is Pronoun': False, 'Is Adjective': False, 'Is Verb': False, 'Is Adverb': False, 'Is Preposition': False, 'Is Conjunction': False, 'Is Interjunction': False, 'Is Named Entity': False}]\n" - ] - } - ], - "source": [ - "sentences = [\n", - " [\"This \", \"is \", \"a \", \"sentence\", \".\"]\n", - "]\n", - "labelled_sentences = token_labelling.label_batch_sentences(sentences, tokenized=True, verbose=False)\n", - "\n", - "print(len(labelled_sentences[0]))\n", - "print(labelled_sentences[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 2) Labelling all tokens in the dataset\n", - "\n", - "Now we want to label all the tokens that our tokenizer knows - its entire vocabulary.\n", - "\n", - "Using thy script in `scripts/label_all_tokens.py` we get the files:\n", - "- `src\\delphi\\eval\\all_tokens_list.txt`\n", - "- `src\\delphi\\eval\\labelled_token_ids_dict.pkl`\n", - "\n", - "Let's load the tokenizer so that we can look at the labelled tokens.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\joshu\\anaconda3\\envs\\delphi2\\lib\\site-packages\\transformers\\utils\\generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", - " _torch_pytree._register_pytree_node(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The vocab size is: 4096\n" - ] - } - ], - "source": [ - "# Get all the tokens of the tokenizer\n", - "from transformers import AutoTokenizer, PreTrainedTokenizer\n", - "\n", - "\n", - "# Decode a sentence\n", - "def decode(tokenizer: PreTrainedTokenizer, token_ids: list[int]) -> str:\n", - " return tokenizer.decode(token_ids, skip_special_tokens=True)\n", - "\n", - "model = \"delphi-suite/delphi-llama2-100k\"\n", - "tokenizer = AutoTokenizer.from_pretrained(model)\n", - "vocab_size = tokenizer.vocab_size\n", - "print(\"The vocab size is:\", vocab_size)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Load the pickle." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import pickle\n", - "path = \"../src/delphi/eval/labelled_token_ids_dict.pkl\"\n", - "# load \n", - "with open(path, \"rb\") as f:\n", - " labelled_token_ids_dict = pickle.load(f)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Look at some random tokens and their labels" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The token id is: 2980\n", - "The decoded token is: four\n", - "The label is:\n", - "{'Capitalized': False,\n", - " 'Is Adjective': False,\n", - " 'Is Adposition': False,\n", - " 'Is Adverb': False,\n", - " 'Is Auxiliary': False,\n", - " 'Is Coordinating conjuction': False,\n", - " 'Is Determiner': False,\n", - " 'Is Interjunction': False,\n", - " 'Is Named Entity': False,\n", - " 'Is Noun': True,\n", - " 'Is Numeral': False,\n", - " 'Is Other': False,\n", - " 'Is Particle': False,\n", - " 'Is Pronoun': False,\n", - " 'Is Proper Noun': False,\n", - " 'Is Punctuation': False,\n", - " 'Is Subordinating conjuction': False,\n", - " 'Is Symbol': False,\n", - " 'Is Verb': False,\n", - " 'Starts with space': False}\n" - ] - } - ], - "source": [ - "import random\n", - "from pprint import pprint\n", - "# Get a random token id between 0 and 4000\n", - "token_id = random.randint(0, 4000)\n", - "# decode the token id\n", - "decoded_token = decode(tokenizer, [token_id])\n", - "# get the corresponding label\n", - "label = labelled_token_ids_dict[token_id]\n", - "# print the results\n", - "print(\"The token id is:\", token_id)\n", - "print(\"The decoded token is:\", decoded_token)\n", - "print(\"The label is:\")\n", - "pprint(label)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 3) Visualize the token label stats\n", - "\n", - "Let's have a look at the statistics." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import matplotlib.pyplot as plt # install matplotlib, if necessary\n", - "from tqdm.autonotebook import tqdm" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
token_idStarts with spaceCapitalizedIs AdjectiveIs AdpositionIs AdverbIs AuxiliaryIs Coordinating conjuctionIs DeterminerIs Interjunction...Is NumeralIs ParticleIs PronounIs Proper NounIs PunctuationIs Subordinating conjuctionIs SymbolIs VerbIs OtherIs Named Entity
00000000000...0000000010
11000000000...0000000010
22000000000...0000000010
33000000000...0000100000
44000000000...0000100000
\n", - "

5 rows × 21 columns

\n", - "
" - ], - "text/plain": [ - " token_id Starts with space Capitalized Is Adjective Is Adposition \\\n", - "0 0 0 0 0 0 \n", - "1 1 0 0 0 0 \n", - "2 2 0 0 0 0 \n", - "3 3 0 0 0 0 \n", - "4 4 0 0 0 0 \n", - "\n", - " Is Adverb Is Auxiliary Is Coordinating conjuction Is Determiner \\\n", - "0 0 0 0 0 \n", - "1 0 0 0 0 \n", - "2 0 0 0 0 \n", - "3 0 0 0 0 \n", - "4 0 0 0 0 \n", - "\n", - " Is Interjunction ... Is Numeral Is Particle Is Pronoun Is Proper Noun \\\n", - "0 0 ... 0 0 0 0 \n", - "1 0 ... 0 0 0 0 \n", - "2 0 ... 0 0 0 0 \n", - "3 0 ... 0 0 0 0 \n", - "4 0 ... 0 0 0 0 \n", - "\n", - " Is Punctuation Is Subordinating conjuction Is Symbol Is Verb Is Other \\\n", - "0 0 0 0 0 1 \n", - "1 0 0 0 0 1 \n", - "2 0 0 0 0 1 \n", - "3 1 0 0 0 0 \n", - "4 1 0 0 0 0 \n", - "\n", - " Is Named Entity \n", - "0 0 \n", - "1 0 \n", - "2 0 \n", - "3 0 \n", - "4 0 \n", - "\n", - "[5 rows x 21 columns]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\"\"\"\n", - " Create a pandas dataframe from the labelled token ids dictionary\n", - "\"\"\"\n", - "# each item in the dictionary is a tuple (token_id, label), where label: dict[str, float]\n", - "# the dataframe should have the columns: token_id, label1, label2 ... labelN\n", - "# label1, label2 ... labelN are the keys of the label dictionary\n", - "# the values of the label dictionary are the probabilities of the label\n", - "# here we go:\n", - "df = pd.DataFrame(labelled_token_ids_dict.items(), columns=[\"token_id\", \"label\"])\n", - "# split the label column into multiple columns\n", - "df = df.join(pd.DataFrame(df.pop('label').tolist()))\n", - "# Change datatype of columns to float\n", - "df = df.astype(int)\n", - "\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We perform a **sanity check** to assure that the code above was correct." - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "aac7894b3f61477bb96b9818757be9f4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "0it [00:00, ?it/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Perform sanity check, that the table was created correctly\n", - "for (row_index, row_values) in tqdm(df.iterrows()):\n", - " token_id = row_values.iloc[0]\n", - " label_pandas = list(row_values.iloc[1:]) # we exclude the token_id from the colum\n", - " label_dict = list(labelled_token_ids_dict[token_id].values())[:]\n", - " assert label_pandas == label_dict, f\"The dataframes are not equal for row {token_id}\\n{label_pandas}\\n{label_dict}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAJMCAYAAAAbs7k3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAACsAUlEQVR4nOzdd1RU1/c28GcA6U1EQBQQxUaxoFGJJRYUlRhr7IpKNBq7RtFv1FgSNSYmamI0xp7EFo3doNiNglEUsSIgCIpgQUBQ+nn/4OX+GMHCFBgyz2etWYu597LnzMDc2XPuOfvIhBACRERERFpMp7wbQERERFTemBARERGR1mNCRERERFqPCRERERFpPSZEREREpPWYEBEREZHWY0JEREREWk+vvBtQEeTn5yMhIQFmZmaQyWTl3RwiIiJ6B0IIPH/+HPb29tDReXMfEBOid5CQkAAHB4fybgYREREpID4+HjVq1HjjMUyI3oGZmRmAghfU3Ny8nFtDRERE7yItLQ0ODg7S5/ibMCF6B4WXyczNzZkQERERVTDvMtyFg6qJiIhI67GHiIg00sWLF3Hp0iWkpKQAAGxsbNC2bVvUqVNHOiY+Ph4nTpzAgwcPIJPJYGdnhyFDhqBSpUpysXJzc7Fu3TokJSXh008/hZ2dHQAgNjYWISEhePDgAbKysmBlZYX3338fDRs2LLPnSUSagQkREWkkc3NzeHt7w8rKCgAQFhaG7du349NPP4WNjQ3i4+Pxxx9/oHXr1ujatSt0dHSQlJRUYtd4UFAQzMzMkJSUJLc9Pj4eNjY2aNWqFUxMTHDnzh3s3bsXhoaGqFu3bpk8TyLSDEyIiEgj1atXT+5+x44dcenSJdy/fx82NjY4cuQImjdvjtatW0vHWFtbF4sTGRmJu3fvol+/foiKipLb16ZNG7n7LVu2xN27d3Hr1i0mRERahgkREWm8/Px83Lx5Ezk5OXBwcEBGRgYePHgADw8PrF+/Hs+ePYO1tTU6dOgAR0dH6ffS09Nx4MABDBgwoNhltNfJzMwsMbEiov82JkREpLGSkpKwfv165ObmQl9fH/3790fVqlVx//59AMDp06fRqVMn2NnZ4erVq9iyZQvGjh2LKlWqQAiBffv2oVmzZrC3t5fGIr3JjRs3kJCQgA8//FDNz4yINA0TIiLSWNbW1hgzZgwyMzNx8+ZN7N27F8OHD4cQAgDQtGlTNGnSBABQrVo1xMTE4MqVK/D29sa///6LrKwsuUtqbxITE4N9+/ahe/fusLGxUdtzIiLNxISIiDSWrq6uNKja3t4eCQkJCAkJkZKcqlWryh1ftWpVpKWlAShIcO7fv4+vvvpK7pi1a9eiYcOG6Nmzp7QtNjYW27Ztg4+PDxo1aqTGZ0REmooJERFVGEII5OXlwdLSEmZmZnjy5Inc/qdPn8LFxQUA0LVrV3To0EHa9/z5c/z+++/o27evXAn/2NhYbN26Fd7e3mjatGnZPBEi0jhMiIhIIx07dgx16tSBhYUFsrKycO3aNcTGxmLIkCGQyWR4//33cerUKdjZ2cHOzg5hYWF48uQJPv74YwCAhYWFXDx9fX0AgJWVlVRxPiYmBtu2bUOLFi3g6uqK9PR0AAU9U0ZGRmX4bImovDEhIiKNlJGRgT179iA9PR0GBgawtbXFkCFDULt2bQAFU+Rzc3Nx5MgRvHz5Era2thg6dKh0ie1dXL16FTk5Ofjnn3/wzz//SNudnJwwfPhwVT8lItJgMlE4OpFeKy0tDRYWFkhNTeVaZkRERBVEaT6/uZYZERERaT0mRERERKT1mBARERGR1uOgaiKqcGrOPKSSOLFLfFUSh4gqPvYQERERkdZjQkRERERajwkRERERaT0mRERERKT1mBARERGR1mNCRERERFqPCRERERFpPSZEREREpPWYEBEREZHWK9eEaPXq1WjYsCHMzc1hbm4OLy8v/P3339L+du3aQSaTyd3GjBkjFyMuLg6+vr4wNjaGjY0Npk+fjtzcXLljTp06BU9PTxgYGMDFxQWbNm0qi6dHREREFUS5Lt1Ro0YNLFmyBHXq1IEQAps3b0aPHj1w5coVuLm5AQBGjRqFBQsWSL9jbGws/ZyXlwdfX1/Y2dnh/PnzePjwIYYNG4ZKlSph0aJFAICYmBj4+vpizJgx+OOPP3D8+HF88sknqFatGnx8fMr2CRMREZFGkgkhRHk3oigrKyt8++238Pf3R7t27dC4cWMsX768xGP//vtvfPjhh0hISICtrS0AYM2aNQgICMDjx4+hr6+PgIAAHDp0CNevX5d+b8CAAUhJSUFgYOA7tSktLQ0WFhZITU2Fubm50s+RiJTDtcyI6F2U5vNbY8YQ5eXlYfv27cjIyICXl5e0/Y8//oC1tTXc3d0xa9YsvHjxQtoXHBwMDw8PKRkCAB8fH6SlpeHGjRvSMd7e3nKP5ePjg+Dg4Ne2JSsrC2lpaXI3IiIi+u8q99Xur127Bi8vL2RmZsLU1BR79uyBq6srAGDQoEFwcnKCvb09wsPDERAQgIiICPz1118AgMTERLlkCIB0PzEx8Y3HpKWl4eXLlzAyMirWpsWLF2P+/Pkqf65ERESkmco9IapXrx7CwsKQmpqKXbt2wc/PD6dPn4arqytGjx4tHefh4YFq1aqhY8eOiI6ORu3atdXWplmzZmHq1KnS/bS0NDg4OKjt8YiIiKh8lfslM319fbi4uKBp06ZYvHgxGjVqhBUrVpR4bIsWLQAAUVFRAAA7OzskJSXJHVN4387O7o3HmJubl9g7BAAGBgbSzLfCGxEREf13lXtC9Kr8/HxkZWWVuC8sLAwAUK1aNQCAl5cXrl27hkePHknHBAUFwdzcXLrs5uXlhePHj8vFCQoKkhunRERERNqtXC+ZzZo1C127doWjoyOeP3+OrVu34tSpUzhy5Aiio6OxdetWdOvWDVWqVEF4eDimTJmCtm3bomHDhgCAzp07w9XVFUOHDsXSpUuRmJiI2bNnY9y4cTAwMAAAjBkzBj/99BNmzJiBkSNH4sSJE9i5cycOHVLNLBUiIiKq+Mo1IXr06BGGDRuGhw8fwsLCAg0bNsSRI0fQqVMnxMfH49ixY1i+fDkyMjLg4OCAPn36YPbs2dLv6+rq4uDBgxg7diy8vLxgYmICPz8/ubpFzs7OOHToEKZMmYIVK1agRo0aWLduHWsQERERkUTj6hBpItYhItIsrENERO+iQtYhIiIiIiovTIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ65ZoQrV69Gg0bNoS5uTnMzc3h5eWFv//+W9qfmZmJcePGoUqVKjA1NUWfPn2QlJQkFyMuLg6+vr4wNjaGjY0Npk+fjtzcXLljTp06BU9PTxgYGMDFxQWbNm0qi6dHREREFUS5JkQ1atTAkiVLEBoaikuXLqFDhw7o0aMHbty4AQCYMmUKDhw4gD///BOnT59GQkICevfuLf1+Xl4efH19kZ2djfPnz2Pz5s3YtGkT5s6dKx0TExMDX19ftG/fHmFhYZg8eTI++eQTHDlypMyfLxEREWkmmRBClHcjirKyssK3336Lvn37omrVqti6dSv69u0LALh9+zYaNGiA4OBgtGzZEn///Tc+/PBDJCQkwNbWFgCwZs0aBAQE4PHjx9DX10dAQAAOHTqE69evS48xYMAApKSkIDAw8J3alJaWBgsLC6SmpsLc3Fz1T5qISqXmzEMqiRO7xFclcYhIM5Xm81tjxhDl5eVh+/btyMjIgJeXF0JDQ5GTkwNvb2/pmPr168PR0RHBwcEAgODgYHh4eEjJEAD4+PggLS1N6mUKDg6Wi1F4TGGMkmRlZSEtLU3uRkRERP9d5Z4QXbt2DaampjAwMMCYMWOwZ88euLq6IjExEfr6+rC0tJQ73tbWFomJiQCAxMREuWSocH/hvjcdk5aWhpcvX5bYpsWLF8PCwkK6OTg4qOKpEhERkYYq94SoXr16CAsLw4ULFzB27Fj4+fnh5s2b5dqmWbNmITU1VbrFx8eXa3uIiIhIvfTKuwH6+vpwcXEBADRt2hQXL17EihUr0L9/f2RnZyMlJUWulygpKQl2dnYAADs7O/z7779y8QpnoRU95tWZaUlJSTA3N4eRkVGJbTIwMICBgYFKnh8RERFpvnLvIXpVfn4+srKy0LRpU1SqVAnHjx+X9kVERCAuLg5eXl4AAC8vL1y7dg2PHj2SjgkKCoK5uTlcXV2lY4rGKDymMAYRERFRufYQzZo1C127doWjoyOeP3+OrVu34tSpUzhy5AgsLCzg7++PqVOnwsrKCubm5pgwYQK8vLzQsmVLAEDnzp3h6uqKoUOHYunSpUhMTMTs2bMxbtw4qYdnzJgx+OmnnzBjxgyMHDkSJ06cwM6dO3HokGpmqRAREVHFV64J0aNHjzBs2DA8fPgQFhYWaNiwIY4cOYJOnToBAH744Qfo6OigT58+yMrKgo+PD37++Wfp93V1dXHw4EGMHTsWXl5eMDExgZ+fHxYsWCAd4+zsjEOHDmHKlClYsWIFatSogXXr1sHHx6fMny8RERFpJo2rQ6SJWIeISLOwDhERvYsKWYeIiIiIqLwwISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOuVa0K0ePFivPfeezAzM4ONjQ169uyJiIgIuWPatWsHmUwmdxszZozcMXFxcfD19YWxsTFsbGwwffp05Obmyh1z6tQpeHp6wsDAAC4uLti0aZO6nx4RERFVEOWaEJ0+fRrjxo1DSEgIgoKCkJOTg86dOyMjI0PuuFGjRuHhw4fSbenSpdK+vLw8+Pr6Ijs7G+fPn8fmzZuxadMmzJ07VzomJiYGvr6+aN++PcLCwjB58mR88sknOHLkSJk9VyIiItJceuX54IGBgXL3N23aBBsbG4SGhqJt27bSdmNjY9jZ2ZUY4+jRo7h58yaOHTsGW1tbNG7cGAsXLkRAQADmzZsHfX19rFmzBs7Ozli2bBkAoEGDBvjnn3/www8/wMfHR31PkIiIiCoEjRpDlJqaCgCwsrKS2/7HH3/A2toa7u7umDVrFl68eCHtCw4OhoeHB2xtbaVtPj4+SEtLw40bN6RjvL295WL6+PggODi4xHZkZWUhLS1N7kZERET/XeXaQ1RUfn4+Jk+ejFatWsHd3V3aPmjQIDg5OcHe3h7h4eEICAhAREQE/vrrLwBAYmKiXDIEQLqfmJj4xmPS0tLw8uVLGBkZye1bvHgx5s+fr/LnSERERJpJYxKicePG4fr16/jnn3/kto8ePVr62cPDA9WqVUPHjh0RHR2N2rVrq6Uts2bNwtSpU6X7aWlpcHBwUMtjERERUfnTiEtm48ePx8GDB3Hy5EnUqFHjjce2aNECABAVFQUAsLOzQ1JSktwxhfcLxx297hhzc/NivUMAYGBgAHNzc7kbERER/XeVa0IkhMD48eOxZ88enDhxAs7Ozm/9nbCwMABAtWrVAABeXl64du0aHj16JB0TFBQEc3NzuLq6SsccP35cLk5QUBC8vLxU9EyIiIioIivXhGjcuHH4/fffsXXrVpiZmSExMRGJiYl4+fIlACA6OhoLFy5EaGgoYmNjsX//fgwbNgxt27ZFw4YNAQCdO3eGq6srhg4diqtXr+LIkSOYPXs2xo0bBwMDAwDAmDFjcPfuXcyYMQO3b9/Gzz//jJ07d2LKlCnl9tyJiIhIc5RrQrR69WqkpqaiXbt2qFatmnTbsWMHAEBfXx/Hjh1D586dUb9+fUybNg19+vTBgQMHpBi6uro4ePAgdHV14eXlhSFDhmDYsGFYsGCBdIyzszMOHTqEoKAgNGrUCMuWLcO6des45Z6IiIgAADIhhCjvRmi6tLQ0WFhYIDU1leOJiDRAzZmHVBIndomvSuIQkWYqzee3RgyqJiIiIipPTIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItJ5KEqK8vDyEhYXh2bNnqghHREREVKYUSogmT56M9evXAyhIhj744AN4enrCwcEBp06dUmX7iIiIiNROoYRo165daNSoEQDgwIEDiImJwe3btzFlyhR88cUXKm0gERERkboplBA9efIEdnZ2AIDDhw/j448/Rt26dTFy5Ehcu3ZNpQ0kIiIiUjeFEiJbW1vcvHkTeXl5CAwMRKdOnQAAL168gK6urkobSERERKRueor80ogRI9CvXz9Uq1YNMpkM3t7eAIALFy6gfv36Km0gERERkboplBDNmzcP7u7uiI+Px8cffwwDAwMAgK6uLmbOnKnSBhIRERGpm0IJEQD07dsXAJCZmSlt8/PzU75FRERERGVMoTFEeXl5WLhwIapXrw5TU1PcvXsXADBnzhxpOj4RERFRRaFQQvT1119j06ZNWLp0KfT19aXt7u7uWLduncoaR0RERFQWFEqItmzZgrVr12Lw4MFys8oaNWqE27dvq6xxRERERGVBoYTowYMHcHFxKbY9Pz8fOTk5SjeKiIiIqCwplBC5urri7Nmzxbbv2rULTZo0UbpRRERERGVJoVlmc+fOhZ+fHx48eID8/Hz89ddfiIiIwJYtW3Dw4EFVt5GIiIhIrRTqIerRowcOHDiAY8eOwcTEBHPnzsWtW7dw4MABqWo1ERERUUWhcB2iNm3aICgoSJVtISIiIioXCvUQXbx4ERcuXCi2/cKFC7h06ZLSjSIiIiIqSwolROPGjUN8fHyx7Q8ePMC4ceOUbhQRERFRWVIoIbp58yY8PT2LbW/SpAlu3rypdKOIiIiIypJCCZGBgQGSkpKKbX/48CH09BQelkRERERULhRKiDp37oxZs2YhNTVV2paSkoL//e9/nGVGREREFY5C3Tnfffcd2rZtCycnJ6kQY1hYGGxtbfHbb7+ptIFERERE6qZQQlS9enWEh4fjjz/+wNWrV2FkZIQRI0Zg4MCBqFSpkqrbSERERKRWCg/4MTExwejRo1XZFiIiIqJyoXBCFBkZiZMnT+LRo0fIz8+X2zd37lylG0ZERERUVhRKiH799VeMHTsW1tbWsLOzg0wmk/bJZDImRERERFShKDTL7KuvvsLXX3+NxMREhIWF4cqVK9Lt8uXL7xxn8eLFeO+992BmZgYbGxv07NkTERERcsdkZmZi3LhxqFKlCkxNTdGnT59iU/7j4uLg6+sLY2Nj2NjYYPr06cjNzZU75tSpU/D09ISBgQFcXFywadMmRZ46ERER/QcplBA9e/YMH3/8sdIPfvr0aYwbNw4hISEICgpCTk4OOnfujIyMDOmYKVOm4MCBA/jzzz9x+vRpJCQkoHfv3tL+vLw8+Pr6Ijs7G+fPn8fmzZuxadMmuV6qmJgY+Pr6on379ggLC8PkyZPxySef4MiRI0o/ByIiIqr4ZEIIUdpf8vf3x3vvvYcxY8aotDGPHz+GjY0NTp8+jbZt2yI1NRVVq1bF1q1b0bdvXwDA7du30aBBAwQHB6Nly5b4+++/8eGHHyIhIQG2trYAgDVr1iAgIACPHz+Gvr4+AgICcOjQIVy/fl16rAEDBiAlJQWBgYFvbVdaWhosLCyQmpoKc3NzlT5nIiq9mjMPqSRO7BJflcQhIs1Ums9vhcYQubi4YM6cOQgJCYGHh0exqfYTJ05UJKxU6NHKygoAEBoaipycHHh7e0vH1K9fH46OjlJCFBwcDA8PDykZAgAfHx+MHTsWN27cQJMmTRAcHCwXo/CYyZMnl9iOrKwsZGVlSffT0tIUej5ERERUMSiUEK1duxampqY4ffo0Tp8+LbdPJpMplBDl5+dj8uTJaNWqFdzd3QEAiYmJ0NfXh6Wlpdyxtra2SExMlI4pmgwV7i/c96Zj0tLS8PLlSxgZGcntW7x4MebPn1/q50BEREQVk0IJUUxMjKrbgXHjxuH69ev4559/VB67tGbNmoWpU6dK99PS0uDg4FCOLSIiIiJ1UmhQdaHs7GxEREQUm9FVWuPHj8fBgwdx8uRJ1KhRQ9puZ2eH7OxspKSkyB2flJQEOzs76ZhXZ50V3n/bMebm5sV6h4CCxWvNzc3lbkRERPTfpVBC9OLFC/j7+8PY2Bhubm6Ii4sDAEyYMAFLlix55zhCCIwfPx579uzBiRMn4OzsLLe/adOmqFSpEo4fPy5ti4iIQFxcHLy8vAAAXl5euHbtGh49eiQdExQUBHNzc7i6ukrHFI1ReExhDCIiItJuCiVEs2bNwtWrV3Hq1CkYGhpK2729vbFjx453jjNu3Dj8/vvv2Lp1K8zMzJCYmIjExES8fPkSAGBhYQF/f39MnToVJ0+eRGhoKEaMGAEvLy+0bNkSANC5c2e4urpi6NChuHr1Ko4cOYLZs2dj3LhxMDAwAACMGTMGd+/exYwZM3D79m38/PPP2LlzJ6ZMmaLI0yciIqL/GIXGEO3duxc7duxAy5Yt5apUu7m5ITo6+p3jrF69GgDQrl07ue0bN27E8OHDAQA//PADdHR00KdPH2RlZcHHxwc///yzdKyuri4OHjyIsWPHwsvLCyYmJvDz88OCBQukY5ydnXHo0CFMmTIFK1asQI0aNbBu3Tr4+Pgo8OyJiIjov0ahhKiwXtCrMjIy5BKkt3mXEkiGhoZYtWoVVq1a9dpjnJyccPjw4TfGadeuHa5cufLObSMiIiLtodAls2bNmuHQof8rjFaYBK1bt47jcoiIiKjCUaiHaNGiRejatStu3ryJ3NxcrFixAjdv3sT58+eL1SUiIiIi0nQK9RC1bt0aV69eRW5uLjw8PHD06FHY2NggODgYTZs2VXUbiYiIiNSq1D1EOTk5+PTTTzFnzhz8+uuv6mgTERERUZkqdQ9RpUqVsHv3bnW0hYiIiKhcKHTJrGfPnti7d6+Km0JERERUPhQaVF2nTh0sWLAA586dQ9OmTWFiYiK3X9HV7omIiIjKg0IJ0fr162FpaYnQ0FCEhobK7VN0tXsiIiKi8qIxq90TERERlRelVrsnIiIi+i9QqIdo5MiRb9y/YcMGhRpDREREVB4USoiePXsmdz8nJwfXr19HSkoKOnTooJKGEREREZUVhRKiPXv2FNuWn5+PsWPHonbt2ko3ioiIiKgsqWwMkY6ODqZOnYoffvhBVSGJiIiIyoRKB1VHR0cjNzdXlSGJiIiI1E6hS2ZTp06Vuy+EwMOHD3Ho0CH4+fmppGFEREREZUWhhOjKlSty93V0dFC1alUsW7bsrTPQiIiIiDSNQgnRyZMnVd0OIiIionKj0BiimJgYREZGFtseGRmJ2NhYZdtEREREVKYUSoiGDx+O8+fPF9t+4cIFDB8+XNk2EREREZUphRKiK1euoFWrVsW2t2zZEmFhYcq2iYiIiKhMKZQQyWQyPH/+vNj21NRU5OXlKd0oIiIiorKkUELUtm1bLF68WC75ycvLw+LFi9G6dWuVNY6IiIioLCg0y+ybb75B27ZtUa9ePbRp0wYAcPbsWaSlpeHEiRMqbSARERGRuinUQ+Tq6orw8HD069cPjx49wvPnzzFs2DDcvn0b7u7uqm4jERERkVop1EMEAPb29li0aJEq20JERERULhTqIdq4cSP+/PPPYtv//PNPbN68WelGEREREZUlhRKixYsXw9rauth2Gxsb9hoRERFRhaNQQhQXFwdnZ+di252cnBAXF6d0o4iIiIjKkkIJkY2NDcLDw4ttv3r1KqpUqaJ0o4iIiIjKkkIJ0cCBAzFx4kScPHkSeXl5yMvLw4kTJzBp0iQMGDBA1W0kIiIiUiuFZpktXLgQsbGx6NixI/T0CkLk5eXBz8+PY4iIiIiowlEoIdLX18eOHTvw+eefIzY2FkZGRvDw8ICTk5Oq20dERESkdqVOiFJSUvDFF19gx44dePbsGQCgcuXKGDBgAL766itYWlqquo1EREREalWqhCg5ORleXl548OABBg8ejAYNGgAAbt68iU2bNuH48eM4f/48KleurJbGEhEREalDqRKiBQsWQF9fH9HR0bC1tS22r3PnzliwYAF++OEHlTaSiIiISJ1KNcts7969+O6774olQwBgZ2eHpUuXYs+ePSprHBEREVFZKFVC9PDhQ7i5ub12v7u7OxITE9853pkzZ9C9e3fY29tDJpNh7969cvuHDx8OmUwmd+vSpYvcMcnJyRg8eDDMzc1haWkJf39/pKenyx0THh6ONm3awNDQEA4ODli6dOk7t5GIiIj++0qVEFlbWyM2Nva1+2NiYmBlZfXO8TIyMtCoUSOsWrXqtcd06dIFDx8+lG7btm2T2z948GDcuHEDQUFBOHjwIM6cOYPRo0dL+9PS0tC5c2c4OTkhNDQU3377LebNm4e1a9e+czuJiIjov61UY4h8fHzwxRdfICgoCPr6+nL7srKyMGfOnGI9OG/StWtXdO3a9Y3HGBgYwM7OrsR9t27dQmBgIC5evIhmzZoBAH788Ud069YN3333Hezt7fHHH38gOzsbGzZsgL6+Ptzc3BAWFobvv/9eLnEiIiIi7VWqHqIFCxYgIiICderUwdKlS7F//37s27cPS5YsQZ06dXDr1i3Mnz9fpQ08deoUbGxsUK9ePYwdOxZPnz6V9gUHB8PS0lJKhgDA29sbOjo6uHDhgnRM27Zt5RI4Hx8fRERESGUDXpWVlYW0tDS5GxEREf13laqHqEaNGggODsZnn32GWbNmQQgBAJDJZOjUqRN++uknODg4qKxxXbp0Qe/eveHs7Izo6Gj873//Q9euXREcHAxdXV0kJibCxsZG/gnp6cHKykoay5SYmFhsIdrCQeGJiYkllghYvHixyhM7IiIi0lylLszo7OyMv//+G8+ePUNkZCQAwMXFpVRjh95V0XXRPDw80LBhQ9SuXRunTp1Cx44dVf54hWbNmoWpU6dK99PS0lSa6BEREZFmUWjpDqCgOnXz5s1V2Za3qlWrFqytrREVFYWOHTvCzs4Ojx49kjsmNzcXycnJ0rgjOzs7JCUlyR1TeP91Y5MMDAxgYGCghmdAREREmkih1e7Ly/379/H06VNUq1YNAODl5YWUlBSEhoZKx5w4cQL5+flo0aKFdMyZM2eQk5MjHRMUFIR69eqxojYREREBKOeEKD09HWFhYQgLCwNQMG0/LCwMcXFxSE9Px/Tp0xESEoLY2FgcP34cPXr0gIuLC3x8fAAADRo0QJcuXTBq1Cj8+++/OHfuHMaPH48BAwbA3t4eADBo0CDo6+vD398fN27cwI4dO7BixQq5S2JERESk3co1Ibp06RKaNGmCJk2aAACmTp2KJk2aYO7cudDV1UV4eDg++ugj1K1bF/7+/mjatCnOnj0rdznrjz/+QP369dGxY0d069YNrVu3lqsxZGFhgaNHjyImJgZNmzbFtGnTMHfuXE65JyIiIolMFE4Vo9dKS0uDhYUFUlNTYW5uXt7NIdJ6NWceUkmc2CW+KolDRJqpNJ/fFWoMEREREZE6MCEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrMSEiIiIirceEiIiIiLQeEyIiIiLSekyIiIiISOsxISIiIiKtx4SIiIiItB4TIiIiItJ6TIiIiIhI6zEhIiIiIq3HhIiIiIi0HhMiIiIi0npMiIiIiEjrlWtCdObMGXTv3h329vaQyWTYu3ev3H4hBObOnYtq1arByMgI3t7eiIyMlDsmOTkZgwcPhrm5OSwtLeHv74/09HS5Y8LDw9GmTRsYGhrCwcEBS5cuVfdTIyIiogqkXBOijIwMNGrUCKtWrSpx/9KlS7Fy5UqsWbMGFy5cgImJCXx8fJCZmSkdM3jwYNy4cQNBQUE4ePAgzpw5g9GjR0v709LS0LlzZzg5OSE0NBTffvst5s2bh7Vr16r9+REREVHFIBNCiPJuBADIZDLs2bMHPXv2BFDQO2Rvb49p06bh888/BwCkpqbC1tYWmzZtwoABA3Dr1i24urri4sWLaNasGQAgMDAQ3bp1w/3792Fvb4/Vq1fjiy++QGJiIvT19QEAM2fOxN69e3H79u13altaWhosLCyQmpoKc3Nz1T95IiqVmjMPqSRO7BJflcQhIs1Ums9vjR1DFBMTg8TERHh7e0vbLCws0KJFCwQHBwMAgoODYWlpKSVDAODt7Q0dHR1cuHBBOqZt27ZSMgQAPj4+iIiIwLNnz0p87KysLKSlpcndiIiI6L9LYxOixMREAICtra3cdltbW2lfYmIibGxs5Pbr6enByspK7piSYhR9jFctXrwYFhYW0s3BwUH5J0REREQaS2MTovI0a9YspKamSrf4+PjybhIRERGpkcYmRHZ2dgCApKQkue1JSUnSPjs7Ozx69Ehuf25uLpKTk+WOKSlG0cd4lYGBAczNzeVuRERE9N+lsQmRs7Mz7OzscPz4cWlbWloaLly4AC8vLwCAl5cXUlJSEBoaKh1z4sQJ5Ofno0WLFtIxZ86cQU5OjnRMUFAQ6tWrh8qVK5fRsyEiIiJNVq4JUXp6OsLCwhAWFgagYCB1WFgY4uLiIJPJMHnyZHz11VfYv38/rl27hmHDhsHe3l6aidagQQN06dIFo0aNwr///otz585h/PjxGDBgAOzt7QEAgwYNgr6+Pvz9/XHjxg3s2LEDK1aswNSpU8vpWRMREZGm0SvPB7906RLat28v3S9MUvz8/LBp0ybMmDEDGRkZGD16NFJSUtC6dWsEBgbC0NBQ+p0//vgD48ePR8eOHaGjo4M+ffpg5cqV0n4LCwscPXoU48aNQ9OmTWFtbY25c+fK1SoiIiIi7aYxdYg0GesQEWkW1iEionfxn6hDRERERFRWmBARERGR1mNCRERERFqPCRERERFpPSZEREREpPWYEBEREZHWY0JEREREWo8JEREREWk9JkRERESk9ZgQERERkdZjQkRERERajwkRERERaT0mRERERKT1mBARERGR1mNCRERERFqPCRERERFpPSZEREREpPWYEBEREZHWY0JEREREWo8JEREREWk9JkRERESk9ZgQERERkdZjQkRERERajwkRERERaT0mRERERKT1mBARERGR1mNCRERERFqPCRERERFpPSZEREREpPWYEBEREZHWY0JEREREWo8JEREREWk9JkRERESk9ZgQERERkdZjQkRERERajwkRERERaT0mRERERKT1mBARERGR1tPohGjevHmQyWRyt/r160v7MzMzMW7cOFSpUgWmpqbo06cPkpKS5GLExcXB19cXxsbGsLGxwfTp05Gbm1vWT4WIiIg0mF55N+Bt3NzccOzYMem+nt7/NXnKlCk4dOgQ/vzzT1hYWGD8+PHo3bs3zp07BwDIy8uDr68v7OzscP78eTx8+BDDhg1DpUqVsGjRojJ/LkRERKSZND4h0tPTg52dXbHtqampWL9+PbZu3YoOHToAADZu3IgGDRogJCQELVu2xNGjR3Hz5k0cO3YMtra2aNy4MRYuXIiAgADMmzcP+vr6Zf10iIiISANp9CUzAIiMjIS9vT1q1aqFwYMHIy4uDgAQGhqKnJwceHt7S8fWr18fjo6OCA4OBgAEBwfDw8MDtra20jE+Pj5IS0vDjRs3XvuYWVlZSEtLk7sRERHRf5dG9xC1aNECmzZtQr169fDw4UPMnz8fbdq0wfXr15GYmAh9fX1YWlrK/Y6trS0SExMBAImJiXLJUOH+wn2vs3jxYsyfP1+1T6YMXbx4EZcuXUJKSgoAwMbGBm3btkWdOnXkjhNCYOvWrYiKikL//v3lxmcBQFhYGIKDg/H06VMYGBjA1dUVvr6+ZfU0iIiIyoxGJ0Rdu3aVfm7YsCFatGgBJycn7Ny5E0ZGRmp73FmzZmHq1KnS/bS0NDg4OKjt8VTN3Nwc3t7esLKyAlCQ2Gzfvh2ffvopbGxspONCQkJeGyM4OBjBwcHo1KkTqlevjpycHCnBIiIi+q/R+EtmRVlaWqJu3bqIioqCnZ0dsrOzi31IJyUlSWOO7Ozsis06K7xf0rikQgYGBjA3N5e7VST16tVDnTp1UKVKFVSpUgUdO3aEvr4+7t+/Lx2TmJiI4OBg9OjRo9jvv3z5EidOnEDPnj3h4eEBKysr2Nraol69emX5NIiIiMpMhUqI0tPTER0djWrVqqFp06aoVKkSjh8/Lu2PiIhAXFwcvLy8AABeXl64du0aHj16JB0TFBQEc3NzuLq6lnn7y0N+fj6uX7+OnJwcqZcrJycHu3fvRrdu3WBqalrsd+7evQshBJ4/f45Vq1bh+++/x59//onU1NSybj4REVGZ0OhLZp9//jm6d+8OJycnJCQk4Msvv4Suri4GDhwICwsL+Pv7Y+rUqbCysoK5uTkmTJgALy8vtGzZEgDQuXNnuLq6YujQoVi6dCkSExMxe/ZsjBs3DgYGBuX87NQrKSkJ69evR25uLvT19dG/f39UrVoVABAYGAgHB4diY4YKPXv2DEIInD17Fl26dIGhoSFOnDiB3377DWPHjoWurm5ZPhUiIiK10+geovv372PgwIGoV68e+vXrhypVqiAkJET6YP/hhx/w4Ycfok+fPmjbti3s7Ozw119/Sb+vq6uLgwcPQldXF15eXhgyZAiGDRuGBQsWlNdTKjPW1tYYM2YMPvnkEzRr1gx79+7F48ePERERgdjYWHTp0uW1vyuEQH5+Prp27QoXFxfUqFEDffr0QXJyMmJiYsrwWRAREZUNje4h2r59+xv3GxoaYtWqVVi1atVrj3FycsLhw4dV3TSNp6urKw2qtre3R0JCAkJCQlCpUiUkJydjyZIlcsfv3LkTjo6OGD58uHQZrTDxBAATExMYGxvzshkREf0naXRCRKojhEBeXh7at28PT09PuX2rV6+Gj48P6tatCwBwdHQEADx58kQaUP7y5Uu8ePGiWJkDIiKi/wImRP9Bx44dQ506dWBhYYGsrCxcu3YNsbGxGDJkCExNTUscSG1hYYHKlSsDAKpUqYJ69eohMDAQ3bt3h4GBAY4fPw5ra2vUrFmzjJ8NERGR+jEh+g/KyMjAnj17kJ6eDgMDA9ja2mLIkCGoXbv2O8fo1asXAgMDsXXrVshkMjg5OWHw4MEcUE1ERP9JMiGEKO9GaLq0tDRYWFggNTW1wtUkIvovqjnzkErixC5h5XWi/7LSfH5r9CwzIiIiorLAhIiIiIi0HhMiIiIi0npMiIiIiEjrcZbZfxwHnxKRJrl37x7Onz+PhIQEpKeno3///nLLCKWnp+PYsWOIjo5GZmYmnJyc0LVrV1SpUkU6Jjk5GUFBQYiLi0Nubi5cXFzQtWvXEkuKEL0r9hAREVGZyc7Ohq2tLbp161ZsnxACO3bswLNnzzBgwAB8+umnsLCwwG+//Ybs7Gzp93///XcAwLBhwzBy5Ejk5eVh27Zt4KRpUgYTIiIiKjN16tRBhw4d0KBBg2L7kpOTcf/+ffj6+qJ69eqwtrbGhx9+iJycHFy/fh0AEB8fj5SUFPTs2RO2trawtbVFz549kZCQwLUWSSlMiIiISCPk5uYCAPT0/m80h0wmg56eHuLi4uSOKVokVk9PDzKZTDqGSBFMiIiISCNYW1vDwsICx48fx8uXL5GXl4d//vkHaWlpSE9PBwDUqFED+vr6OHbsGHJycpCdnY2jR49CCIHnz5+X8zOgioyDqomISCPo6uqiX79+2L9/P5YuXQqZTIZatWrBxcVFOsbExAQff/wxDh06hAsXLkAmk8HDwwPVqlWDTCYrx9ZTRceEiIiINIa9vT3GjBmDzMxM5OXlwcTEBOvWrUO1atWkY2rXro2JEyfixYsX0NHRgaGhIb777ju4ubmVY8upouMlMyIi0jiGhoYwMTHB06dPkZCQIDc1v5CxsTEMDQ0RExODjIwM1KtXrxxaSv8V7CEiIqIyk52djeTkZOn+s2fPkJiYCCMjI1hYWODGjRswMTGBhYUFkpKSEBgYiPr166N27drS71y5cgVVq1aFsbEx7t+/j8DAQLRs2RLW1tbl8ZToP4IJERERlZmEhARs3rxZun/06FEAQKNGjdCzZ0+kp6fj6NGjSE9Ph5mZGRo2bIgPPvhALsbTp0+lgdeWlpZo06YNWrZsWabPg/57mBARkdrk5+fj1KlTuHbtmvQB16hRI7Rt21YaADt//vwSf9fb2xutWrUqy+ZSGahZsya+/PLL1+5v0aIFWrRo8cYY3t7e8Pb2VnXTSMsxISIitTl37hwuXbqEnj17wsbGBgkJCdi3bx8MDQ2lD71p06bJ/U5kZCT2798PV1fX8mhyhfYuCagQAqdOncLly5eRmZkJBwcH+Pr6yi2NQaSNmBARkdrEx8ejXr16qFu3LgDA0tIS169fx4MHD6RjXl1/KiIiAs7OzqhcuXKZtvW/4F0S0HPnzuHChQvo2bMnKleujJMnT+L333/HuHHj5AoiEmkbzjIjIrVxcHBATEwMnj59CgBITExEXFycXF2ZotLT0xEZGYkmTZqUZTP/M4omoJaWlnB1dUXt2rWlBFQIgQsXLqBt27aoX7++tOzF8+fPcfv27XJuPVH54tcBIlKb1q1bIysrCz/99BN0dHSQn5+PDh06oGHDhiUef/XqVejr65e4zhW9nYODA0JDQ/H06VNUqVJFSkA7d+4MAEhJSUF6ejpq1aol/Y6hoSFq1KiB+Ph4uLu7l2l7a848pJI4sUt8VRKHtBsTIiJSmxs3buDatWvo06cPqlatisTERBw5cgRmZmZo3LhxseOvXLkCDw8PXrpR0NsS0MLlL0xMTOR+z8TEBBkZGWXeXiJNwrMOEalNUFAQWrVqJfU82NraIjU1Ff/880+xhOjevXt4+vQp+vbtWw4t/W8obQJK2uXs2bO4ffs2njx5Aj09PTg4OMDb21uuftOmTZtw7949ud9r2rQpPvzww7JubpljQkREuHfvHs6fP4+EhASkp6ejf//+xSoDP378GMeOHcO9e/eQn5+PqlWrol+/frCwsHht3JycnGLrS8lkMgghih175coVVKtWDXZ2dqp5Ukp42+uRnZ2NY8eO4fbt21ItnBYtWqBZs2bl2Oq3J6CFA9gzMjJgZmYm/V5GRgZsbW3Lpc1Udu7du4f33nsP9vb2yM/Px4kTJ/D777/js88+g76+vnScp6cn2rdvL92vVKnSO8WuiO+ZopgQERGys7Nha2uLxo0bY+fOncX2JycnY+PGjWjSpAnatWsHAwMDPH78+K2XturWrYuzZ8/CwsICNjY2ePjwIUJCQor1VmRlZeHmzZvSWJfy9rbX48iRI4iJiUHv3r1haWmJ6OhoHDp0CGZmZuW6fMTbElBLS0uYmpri7t27UuKZlZWF+/fvv/WD6U0feHl5eThx4gSioqLw7NkzGBgYoFatWvD29pZLvEhxaWlpOHbsGKKiopCTkwMrKyv06NED9vb27xxjyJAhcvd79OiB7777Dg8fPoSTk5O0vVKlSsVmf75NRX3PFMWEqAL4999/cf78eaSnp8POzg5du3ZF9erVy7tZb6TONlfE2OqK+y49O++iTp06qFOnzmv3nzhxAnXq1EGnTp2kbVZWVm+N27VrV5w8eRKHDx+WeiWaNm1arPLw9evXIYQo80G9r/O21yM+Ph6NGjVCzZo1ARRcUggNDcWDBw8UOrmr6u/4tgRUJpOhRYsWOHv2LKpUqQJLS0ucPHkSZmZmb328N33g5eTkIDExEW3btoWtrS0yMzMRGBiIbdu2YfTo0aV+HupW0c4hL1++xIYNG+Ds7IzBgwfD2NgYycnJMDQ0VCpuVlYWAMDIyEhu+7Vr1xAeHg5TU1PUrVsXH3zwwVt7icr6PaMOTIg03PXr13H06FH4+vqiRo0aCAkJwe+//47x48cXGxipKdTZ5ooYW51tftu3MlUQQiAyMhLvv/8+fv/9dzx8+BCVK1dG69at3/ohamBggC5duqBLly5vPK5p06Zo2rSpKputVg4ODrhz5w6aNGkCMzMzxMbG4unTp/Dx8VEonqr+ju+SgLZq1Qo5OTk4cOAAMjMz4ejoiCFDhry1t+9NH3iGhoYYOnRosbasW7cOqampb7ysWtYq4jnk3LlzsLCwQI8ePaRtytbpEkIgMDAQDg4OsLGxkbZ7eHjAwsICZmZmSEpKwrFjx/D06VP0799fqcdT9XtGHZgQabiQkBB4enpKdVk+/PBDREZG4sqVK2jdunU5t65k6mxzRYytzja/7VuZKmRkZCA7Oxvnzp1D+/bt4e3tjaioKOzYsQN+fn7SNz5t0rVrVxw8eBA//PADdHR0IJPJ0L17d7nLDqWhqr/juySgMpkM7du3lxsjog6FvQ/K9mKoWkU8h0RERKB27dr4888/ERsbC3NzczRr1kypLxGHDh3Co0ePMHLkSLntRWPa2trCzMwMW7ZsQXJy8jv1Cr+Oqt8z6sCESIPl5eUhISFB7o0kk8lQq1Yt3L9/vxxb9nrqbHNFjF0R/4avKhx/Uq9ePXh5eQEA7OzsEB8fj9DQUK1MiP7991/cv38fAwYMgKWlJe7du4fDhw/DzMxMrsaPtsrNzcWxY8fg4eEBAwOD8m6OpCKeQwDg2bNnuHTpEry8vNC6dWskJCQgMDAQurq6Cs0ePHz4MCIjIzF8+HCYm5u/8djCy33KJkQV4T3DhEiDvXjxAkKIEmuGPHnypJxa9WbqbHNFjF0R/4avMjY2ho6ODqpWrSq33draGvHx8eXUqvKTk5OD48ePo3///tKSJLa2tkhMTMT58+c15uReXvLy8vDnn39CCAFfX80qmFgRzyFAwZcSe3t7dOzYEQBQrVo1PHr0CKGhoaVKiIQQ+Pvvv3H79m34+fm902W3xMREAFBqcHxFec8wISKiN9LV1YW9vb20/Eah5ORkjRobUlby8/ORn5//zuUEtEleXh527dqF1NRUDBs2TKN6hyoyMzOzEr+Q3Lp1q1RxDh8+jGvXrmHAgAEwMDCQCnUaGBigUqVKSE5OxrVr11CnTh0YGxsjKSkJR44cgZOTk1JlGSrKe4YJkQYzNjaGTCYrVkE2IyOj1FMiy4o621wRY1eUv2F2djaSk5Ol+8+ePUNiYiKMjIxgYWGB999/H7t27YKjoyOcnZ0RFRWFiIgIDB8+/LUxK/KyDG97PZycnBAUFIRKlSrBwsIC9+7dQ3h4eLmVDdCE17owGXr69Cn8/PxgbGyskjapUkU8hwAFA5Jf/ULy9OnTUn8huXTpEgBg8+bNctt79OiBxo0bQ1dXFzExMbhw4QKys7NhYWGBBg0aoG3btm+NXdHeMyVhQqTBCr+Z3717V5rNI4TA3bt30bx583JuXcnU2eaKGLui/A0TEhLkTpJHjx4FADRq1Ag9e/ZEgwYN8OGHH+Kff/5BYGAgqlSpgn79+sHR0bG8mqxWb3s9+vbti+PHj+Ovv/7Cy5cvYWFhgQ4dOmhUkTlVe9MHnqmpKf788088fPgQAwcOhBBC6n0wMjKCrq5ueTVbTkU8hwBAy5YtsWHDBpw9exZubm548OABLl++XOrq0V9++eUb91tYWLzxS86b/BfeM0yINFzLli2xd+9e2Nvbo3r16ggJCUFOTo5Gl+FXZ5srYmx1tvlt38reVc2aNd96smzSpInWrEL/ttfD1NRUbgq0slT1d1SnN33gtWvXDhEREQCAX375Re73NG0mYkU8h1SvXh39+/fH8ePHcfr0aVSuXBk+Pj6vXSS5PJT1e0YdmBBpOHd3d7x48QKnTp2SCn0NHjxYoy63vEqdba6IsdXZ5rd9K6OKoSL8Hd/2gfe2hFpTVMRzCFBQdLNwQDKph0xo0ogmNVu1ahW+/fZbJCYmolGjRvjxxx/fqSszLS0NFhYWSE1NfesURU2jCWMLSDup83+P/9fyKurrUVHbTRVHaT6/dcqoTeVux44dmDp1Kr788ktcvnwZjRo1go+PDx49elTeTSMiIqJypjWXzL7//nuMGjUKI0aMAACsWbMGhw4dwoYNGzBz5sxybh1R+VDVN3Tgv/Mtnb0W9F/EHtu304qEKDs7G6GhoZg1a5a0TUdHB97e3ggODi52fFZWllR2HgBSU1MBFHS9VTT5WS9UEqciPnd6O1X9fwDF/0fU+b9XUWOrizrb7P7lEZXEvj6/+JpVFfG1VqeK+lpr8t+xMOY7jQ4SWuDBgwcCgDh//rzc9unTp4vmzZsXO/7LL78UAHjjjTfeeOONt//ALT4+/q25glb0EJXWrFmzMHXqVOl+fn4+kpOTUaVKlWKVNstCWloaHBwcEB8fr/JB3RUxdkVsc0WNXRHbzNhlF5exyy4uYytGCIHnz5/D3t7+rcdqRUJkbW0NXV1dJCUlyW1PSkqCnZ1dseMNDAyKlZy3tLRUZxPfibm5udr+mSpi7IrY5ooauyK2mbHLLi5jl11cxi69d63lpRWzzPT19dG0aVMcP35c2pafn4/jx49Lq3cTERGR9tKKHiIAmDp1Kvz8/NCsWTM0b94cy5cvR0ZGhjTrjIiIiLSX1iRE/fv3x+PHjzF37lwkJiaicePGCAwMVGoF37JiYGCAL7/8Ui0rR1fE2BWxzRU1dkVsM2OXXVzGLru4jK1+WlWpmoiIiKgkWjGGiIiIiOhNmBARERGR1mNCRERERFqPCRERERFpPSZEpHLZ2dmIiIhAbm5ueTelVOLj4xEfH1/ezSg3OTk5GDlyJGJiYsq7KVonKioKR44cwcuXLwHg3dZdIq2iyvNTTk4OateujVu3bqkk3n+F1ky7r2hSUlKwa9cuREdHY/r06bCyssLly5dha2uL6tWrl3fzSvTixQtMmDABmzdvBgDcuXMHtWrVwoQJE1C9enXMnDlT4diRkZE4efIkHj16hPz8fLl9c+fOVThubm4u5s+fj5UrVyI9PR0AYGpqigkTJuDLL79EpUqVFI7t5+cHf39/tG3bVuEYr5OUlITPP/8cx48fx6NHj4p9gObl5ZU6ZqVKlbB7927MmTNHVc0s5vjx41KbX/07btiwQeG4eXl52LRp02tjnzhxolTxii7d8zbff/99qWIX9fTpU/Tv3x8nTpyATCZDZGQkatWqBX9/f1SuXBnLli1TOHZ+fj6ioqJKfD2U/Z9MSUnBv//+W2LsYcOGKRUbKPhSVVJsR0dHjY6tauo6P1WqVAmZmZmqbKqcDz74AP7+/vj4449hZGSktsdRNSZEGig8PBze3t6wsLBAbGwsRo0aBSsrK/z111+Ii4vDli1bSh1z5cqV73zsxIkTSx0fKFgD7urVqzh16hS6dOkibff29sa8efMUToh+/fVXjB07FtbW1rCzs5NbT04mkymVEE2YMAF//fUXli5dKlUtDw4Oxrx58/D06VOsXr1a4dipqanw9vaGk5MTRowYAT8/P5Uls8OHD0dcXBzmzJmDatWqqWyNvZ49e2Lv3r2YMmWKSuIVNX/+fCxYsADNmjVTaZsBYNKkSdi0aRN8fX3h7u6udOwrV66803HKPs6UKVOgp6eHuLg4NGjQQNrev39/TJ06VeGEKCQkBIMGDcK9e/eKJcsymUyhhLnQgQMHMHjwYKSnp8Pc3LzY+1GZhCgyMhIjR47E+fPn5bYLIZRutypjV65c+Z3/9snJyaVqZ1HqPD+NGzcO33zzDdatWwc9PdWmAk2aNMHnn3+OCRMmoF+/fvD390fLli1V+hjqwDpEGsjb2xuenp5YunQpzMzMcPXqVdSqVQvnz5/HoEGDEBsbW+qYzs7OcvcfP36MFy9eSGu0paSkwNjYGDY2Nrh7965C7XZycsKOHTvQsmVLuXZHRUXB09MTaWlpCsf97LPPEBAQoNDvv4mFhQW2b9+Orl27ym0/fPgwBg4ciNTUVKXiP378GL/99hs2b96MmzdvwtvbG/7+/ujRo4dSvU9mZmY4e/YsGjdurFT7XvXVV19h2bJl6NixI5o2bQoTExO5/YomywBQrVo1LF26FEOHDlW2mcVYW1tjy5Yt6Natm8pjq5OdnR2OHDmCRo0ayb1n7t69i4YNG0q9AqXVuHFj1K1bF/Pnzy8x+XzXtZ1KUrduXXTr1g2LFi2CsbGxwnFK0qpVK+jp6WHmzJkltrtRo0YaEbuwF/xd+Pn5vfOxr1Ln+alXr144fvw4TE1N4eHhUey9/tdffykcGyjo3dq/fz82b96Mv//+Gy4uLhg5ciSGDh2quQWRBWkcc3NzERUVJYQQwtTUVERHRwshhIiNjRUGBgZKx//jjz9Eq1atxO3bt6Vtt2/fFm3atBG///67wnGNjIykthZtd1hYmDA3N1c4rpmZmRRL1apWrSpu3rxZbPvNmzeFtbW1Sh8rNDRUjB8/XhgaGgpra2sxefJkcefOHYViNWjQQFy+fFml7RNCiJo1a7725uzsrFRsKysr6f9a1apVqyYiIiLUEludTE1Npf+Bou+ZixcvCisrK4XjGhsbi8jISJW0saTY6no/Ghsbi1u3blW42OqizvPT8OHD33hTpaSkJLFw4UJhaGgoKlWqJHr06CGOHz+u0sdQBSZEGqhq1arSh13Rk+TRo0dFjRo1lI5fq1atEj9ML126JGrWrKlw3DZt2oiVK1cKIQrafffuXSGEEOPHjxc+Pj4Kxx05cqRYvXq1wr//JvPnzxcDBw4UmZmZ0rbMzEwxePBgMW/ePJU9TkJCgliyZImoV6+eMDExEcOGDRMdO3YUenp64vvvvy91vCNHjojOnTuLmJgYlbVR3WbMmCEWLFigltjfffed+Oyzz0R+fr5a4l+8eFFMnz5d9O/fX/Tq1UvupoyuXbuK2bNnCyH+7z2Tl5cnPv74Y9GnTx+F47Zv3178/fffSrXtdXr16iV27NihltjNmjUTZ8+erXCxc3Nzxa5du8TChQvFwoULxV9//SVyc3OVjltW5yd1unDhghgzZoywtLQUjo6OYu7cucLf318YGRmJadOmlXfz5HAMkQb66KOPsGDBAuzcuRNAwXX5uLg4BAQEoE+fPkrHf/jwYYkzwPLy8pCUlKRw3EWLFqFr1664efMmcnNzsWLFCty8eRPnz5/H6dOnFY7r4uKCOXPmICQkBB4eHsUuNZX2Mk7v3r3l7h87dgw1atSQusyvXr2K7OxsdOzYUeE2AwUzOfbv34+NGzfi6NGjaNiwISZPnoxBgwbB3NwcALBnzx6MHDmy1GN2+vfvjxcvXqB27dowNjYu9pooM24BKBh4GhMTg9q1a6tsfEFmZibWrl2LY8eOoWHDhsXarMzg5H/++QcnT57E33//DTc3t2Kxlen+3759O4YNGwYfHx8cPXoUnTt3xp07d5CUlIRevXopHBcAli5dio4dO+LSpUvIzs7GjBkzcOPGDSQnJ+PcuXMKx50wYQKmTZuGxMTEEt8zDRs2VDi2r68vpk+fjps3b5YY+6OPPlI49jfffIMZM2Zg0aJFJcYufN9oUuyoqCh069YNDx48QL169QAAixcvhoODAw4dOoTatWuXKl5ZnZ+Agstap06dQnR0NAYNGgQzMzMkJCTA3NwcpqamCsd99OgRfvvtN2zcuBGRkZHo3r07tm3bBh8fH+lS5fDhw9GlSxd89913Sj8PVeEYIg2UmpqKvn374tKlS3j+/Dns7e2RmJgILy8vHD58uNi13tLq3r07Hjx4gHXr1sHT0xMAEBoaitGjR6N69erYv3+/wrGjo6OxZMkSXL16Fenp6fD09ERAQAA8PDwUjvnq+KeiZDJZqcc8jRgx4p2P3bhxY6liF2VtbY38/HwMHDgQo0aNKnG8T0pKCpo0aVLqqe5vG8Og6LgFdc4UbN++/Wv3yWSyUs8EK+ptf1Nl/o4NGzbEp59+inHjxknjfJydnfHpp5+iWrVqmD9/vsKxgYL3+08//ST3nhk3bhyqVaumcEwdneIVVWQymUoGJ5cUu+hjqCL2q+N7VNluVcfu1q0bhBD4448/YGVlBaBg9uCQIUOgo6ODQ4cOlSpeWZ2f7t27hy5duiAuLg5ZWVnSe33SpEnIysrCmjVrFI6tr6+P2rVrY+TIkRg+fDiqVq1a7Ji0tDT06NEDJ0+eVPhxVI0JkQY7d+6c3EnS29tbJXEfP34MPz8/BAYGSt+ScnNz4ePjg02bNsHGxkYlj6PphBCIj49H1apV1TI19LfffsPHH38MQ0NDlcdWl0mTJuHcuXNYvnw5unTpgvDwcNSqVQv79u3DvHnz3nn21X+JiYkJbty4gZo1a6JKlSo4deoUPDw8cOvWLXTo0AEPHz4s7yYWc+/evTfud3JyKqOWlM7bepI/+OADjYttYmIi9V4XdfXqVbRq1UrhgfHqPj/17NkTZmZmWL9+PapUqSIN6D916hRGjRqFyMhIhWOfPXsWbdq0UWFrywYvmWmwVq1aoVWrViqPW7VqVRw+fBh37tzB7du3AQD169dH3bp1lYrr7e2NIUOGoHfv3kp1bb9JYf6uiinbQgi4uLjgxo0bqFOnjtLxisrJycGIESPQpEkTuLu7qzR2oby8POzdu1cqrubm5oaPPvoIurq6Csfcu3evNFOw6Gvs5uaG6Ohopdtc6P79+wCAGjVqqCymulSuXBnPnz8HAFSvXh3Xr1+Hh4cHUlJS8OLFi1LHCw8Pf+djFb20pakJz9sok/CUV2wDAwPp/6Oo9PR06OvrKxxXnecnoCBpOX/+fLE21qxZEw8ePFAq9pdffom//vpLmsVcKC0tDT179lSqN1idmBBpoIkTJ8LFxaXY2JiffvoJUVFRWL58uUoep2bNmhBCqGyciJubG2bNmoXPPvsMvr6+GDJkCLp166bU9PJCW7Zswbfffit9a6lbty6mT5+u1BRuHR0d1KlTB0+fPlX5CadSpUpwdHRUqov/TVQ9bqHQ48ePS+whzMjIUDoJzc/Pl6b1F35rNjMzw7Rp0/DFF1+88VLM2zg7O7+xfYqWkgAKihgGBQXBw8MDH3/8MSZNmoQTJ04gKChIoXEcjRs3li5fvYkyl3HeVqtMmVpBCxYseON+ZeqCAQWXkdevXy+X6I8cOVKpUgHqjP3hhx9i9OjRWL9+PZo3bw4AuHDhAsaMGaPUeCp1np+AgvdjSf9f9+/fh5mZmVKxT58+jezs7GLbMzMzcfbsWaViq1U5DOSmt7C3txeXLl0qtj00NFRUr15d6fgZGRli5MiRQldXV+jq6kqz2MaPHy8WL16sVOy8vDxx5MgR4efnJ8zNzUXlypXFqFGjxKlTpxSOuWzZMmFsbCxmzJgh9u3bJ/bt2yemT58ujI2NFZqhVdT+/ftF69atxbVr15SKU5J169aJbt26iadPn6o8dteuXUWXLl3kYj958kR06dJFdOvWTeG46popKIQQM2fOFFWrVhU///yzuHr1qrh69apYtWqVqFq1qvjf//6nVOzly5fL3b799lsxaNAgYWVlpfT/9NOnT8WDBw+EEAX/34sXLxbdu3cXU6dOFcnJyaWOFxsb+843RVlaWsrdTExMhEwmEwYGBqJy5coKxxVCiMaNG8vd3NzchLGxsTA3NxdNmjRRKnZhuYHq1atLs/hq1KghqlSpIkJDQzUy9rNnz8RHH30kZDKZ0NfXF/r6+kJHR0f07NlTpKSkKNVmdZ6f+vXrJ0aNGiWE+L/3+vPnz0WHDh0UnnZf+L6WyWTi5MmT0v2rV6+Ky5cvi0WLFgknJycVPgvVYkKkgQwMDEqsIRIZGamSOkQTJ04UTZs2FWfPnhUmJiZSQrR3717RuHFjpeMXevnypdi5c6do1KiR0NHRUThOzZo1xebNm4tt37Rpk1JlAoQo+OAoPIEZGhqKypUry92U0bhxY2FqaioMDAxE3bp1RZMmTeRuyjA2Nhbh4eHFtoeFhQkTExOF4549e1aYmpqKMWPGCENDQzFp0iTRqVMnYWJiUmKSXhrVqlUT+/btK7Z97969wt7eXqnYr/PTTz8pVVMlJydHbN68WSQmJqqwVeXjzp07omPHjiIwMFDlsVNTU0WvXr3Eli1blIrTunVrMXz4cJGTkyNty8nJEX5+fqJNmzYaG1uIgtd33759Yv/+/SqrAaXO81N8fLxwdXUVDRo0EHp6eqJly5aiSpUqol69eiIpKUmhmDKZTOjo6AgdHR0hk8mK3YyNjcX69euVarc6cVC1BnJ3d8eYMWMwfvx4ue0//vgjVq9ejZs3byoVX10VpYtKTEzE9u3b8fvvv+Py5cto3rw5QkJCFIplaGiI69evw8XFRW57ZGQkPDw8lFqTR12ztQC8dfbRl19+qXBsKysrHDx4EO+//77c9nPnzqF79+5KTbtXx0xBoODvGB4eXmysWkREBBo3biwtbKpKd+/eRePGjZX6nzY2NsatW7fUMi5n8eLFsLW1xciRI+W2b9iwAY8fP1Z5dfZLly5hyJAh0thBVbp27Rq6d++uUCX9QkZGRrhy5Qrq168vt/3mzZto1qyZQmO2yiJ2IaHCMY6Aes9PQMFkmu3btyM8PFx6rw8ePFjhQdyFS8XUqlUL//77r9zsMn19fdjY2Cg1xlHtyjcfo5KsX79eGBkZiblz54pTp06JU6dOiTlz5ghjY2Oxdu1apeOrq6J0amqq2LBhg/D29hZ6enqibt26Yv78+UpXJ3ZzcxNff/11se0LFy4U7u7uSsWuqIYOHSrc3NxESEiIyM/PF/n5+SI4OFi4u7sLPz+/8m5eiZo3by4mTJhQbPv48eNFixYt1PKY33zzjdJd9B988IHYu3evahr0CicnJ3Hu3Lli20NCQpTu/SzJlStXhJmZmcrjClHQu2hpaalUDBsbG3HkyJFi2wMDA4WNjY3Gxl63bp1wc3OTLpm5ubmJX3/9VamYVPY4qFoDjRw5EllZWfj666+xcOFCAAUDoFevXq2SlaSbNWuGQ4cOYcKECQD+79vMunXrpAUEFWFra4vKlSujf//+WLx4MZo1a6Z0W4GCnpb+/fvjzJkz0qy7c+fO4fjx41LxSmVER0dj48aNiI6OxooVK2BjY4O///4bjo6OcHNzUzq+OqxcuRJ+fn7w8vKSK53w0UcfYcWKFUrFVtcq6UuXLoWvry+OHTsmt1BlfHw8Dh8+rFSbmzRpIvetXAiBxMREPH78GD///LNSsT/77DNMnToV8fHxJa7vpkyRw8TExBLrDVWtWlWp6fyv1hITQuDhw4f46aeflJ65+upC0YWxf/vtt2JrbpVW//794e/vj++++07q/Tx37hymT5+OgQMHamTsuXPn4vvvv8eECRPk/q+nTJmCuLi4tw5Cfxt1np8iIyNx8uTJEt/rpR0cv3//fnTt2hWVKlV6ay07ZQabq1U5J2T0Fo8ePRLPnz9XaUx1jRM5evSoyMvLU2FL/8+lS5fE4MGDhaenp/D09BSDBw9WyVpep06dEkZGRsLb21vo6+tLvWWLFy9WaukEIQrK+X/77bfivffeE7a2tiq9/l/ozp07Yv/+/SobtxAcHCycnZ1LHAOgzDiwQg8ePBD/+9//RO/evUXv3r3FF198IQ1YVsa8efPkbgsWLBCrV69WydpVJY2FKHx9lH1NXFxcxG+//VZs+5YtW5RaO66k9tra2oqBAweKhIQEZZpcbI27WrVqiRYtWohZs2aJtLQ0pWJnZWWJiRMnSuNmdHR0hIGBgZg8ebLc8hWaFNva2lps3bq12PatW7eKKlWqKNNktZ6f1q5dK3R1dYWtra1o1KiR3EB5RcY4ymQyaexRSe8ZVZ5H1IUJkZaKiooSn3zyiXjvvfdEgwYNxODBg0scpPtf17JlS7Fs2TIhhPzlwwsXLig9o2/OnDmiWrVq4rvvvhOGhoZi4cKFwt/fX1SpUkWsWLFC6barQ6NGjcTHH38sbt68KZ49eyZSUlLkbtpIXTPBhCi4pFelShWxYcMGKd769etFlSpVxKJFi1T0DCqejIwMER4eLsLDw0VGRoZGx7awsChxkeaIiAhhYWGhVGx1np8cHR3FkiVLlIrxX8NB1Rpq165d2LlzJ+Li4orVc7h8+XI5tao4T09PHD9+HJUrVy522eJVpWl3WlqaVNzxbQNilSkCaWpqimvXrsHZ2VlugHlsbCzq16+v1IDt2rVrY+XKlfD19YWZmRnCwsKkbSEhIdi6dWup4k2dOhULFy6EiYkJpk6d+sZjFV0XzMTEBFevXi02gF1R4eHhcHd3h46OzlsLEipz6QlQT6FKdRNCYObMmVi5cqX0Pjc0NERAQIDS9XyKPgaguoG+RVWkApvqMmHCBFSqVKnYe+7zzz/Hy5cvsWrVKoVjq/P8ZG5ujrCwMNSqVUvhGK+zZcsW9O/fHwYGBnLbs7OzpbUBNRHHEGmglStX4osvvsDw4cOxb98+jBgxAtHR0bh48SLGjRundPwOHTrggw8+KDbL6dmzZ+jTp0+pqoj26NFD+qfv0aOHyk66lStXxsOHD2FjYwNLS8sS4woVrG9kaWmJhw8fFlsv7cqVK6hevbrCcQFIC2sCBSe21NRUAAWF3ObMmVPqeFeuXEFOTo70szq0aNECUVFRKkuIGjdujMTERNjY2LyxIKGyf0d1Faos9Ntvv2HNmjWIiYlBcHAwnJycsHz5cjg7O6NHjx4Kx5XJZPjmm28wZ84c3Lp1C0ZGRqhTp06xDxJFqKOYKaD6Apu9e/fGpk2bYG5uXmxh01eVdpFedcUu+oVEJpNh3bp1OHr0KFq2bAmgoDBjXFyc0h/86jw/ffzxxzh69CjGjBmjVJySjBgxAl26dClW5PX58+cYMWIEEyJ6dz///DPWrl2LgQMHYtOmTZgxYwZq1aqFuXPnKr2KOQCcOnUK165dw5UrV/DHH39Ig0Szs7NLvSp90aRq3rx5Sret0IkTJ6SFEtW5+N+AAQMQEBCAP//8EzKZDPn5+Th37hw+//xzpd+0NWrUwMOHD+Ho6IjatWvj6NGj8PT0xMWLFxX6wCv6OqjrNVH1KukxMTHS1NvSLmBbGhMnTkTt2rUREhJSbIHNiRMnlnqBzaJWr16NuXPnYvLkyfj666+lxM3S0hLLly9XKiEqZGpqivfee0/pOIW+//57zJkzB+PHj5cGUf/zzz8YM2YMnjx5gilTpigc+4svvsD69euxZMkSudjz5s1DZmYmvv7661LFs7CwkL7wmJubq7QnS12xX/1C0rRpUwCQlrextraGtbU1bty4odTjqPr8VHRAvIuLC+bMmSOtw/bqe/3VlRJKo/DL6qvu37+vkorjalOOl+voNYyMjKSxCVWrVhVhYWFCiIIBtFZWVkrHl8lkIiwsTLRo0UK4u7uLmJgYIYQQiYmJSg14c3Z2Fk+ePCm2/dmzZ0oNEL13757Iz88vtj0/P1/cu3dP4bhCFAy0/OSTT4Senp6QyWSiUqVKQkdHRwwZMkTk5uYqFTsgIEAqF7B9+3ahp6cnXFxchL6+vggICFAq9ogRI0ocwJqeni5GjBihcFx1DiA+ffq0XFG8Qjk5OeL06dNKxVZXoUohhGjQoIHYs2ePEEJ+HMe1a9cUGjTbq1cvkZqaKv38ppui1FnMtDwKbGorVZ+fXh0Q/7qboufrwgHZOjo6wsPDQ64QbcOGDYWZmZn4+OOPFYpdFthDpIHs7OyQnJwMJycnODo6IiQkBI0aNUJMTMxb1z96V9WqVcPp06cxYsQIvPfee/jzzz/RoEEDpWLGxsaWeNkjKytLGmugCGdnZ+nyWVHJyclwdnZW6lKLvr4+fv31V8yZMwfXr19Heno6mjRpopK1g5YsWSL93L9/fzg6OiI4OBh16tRB9+7dlYq9efNmLFmypNiaQy9fvsSWLVuwYcMGheKqsxenffv2Jf4dU1NT0b59e6X+jupaYBMoeE2aNGlS4mNmZGSUOp46e0QKPXz4sFjRTgB4//33lZrODxS8714tbggULBCtbA92hw4d1LYoqDpjq1rfvn3xySefwMfHB7/++ivmzp2La9euKX1+Uuf7GwB69uwJAAgLC4OPjw9MTU2lffr6+qhZsyb69Omj1jYopbwzMirO399fzJs3TwhRsPRA4bRLS0tLMXLkSKXj6+joyJVmX7hwoTAwMBBz585VqBegcH0xmUwmtmzZIt3ft2+f+Ouvv8S4ceNE3bp1FW6vTCYTjx49KrY9NjZWGBsbKxxXiIISBBVJamqqSElJETKZTERFRYnU1FTplpycLDZv3iyqVatW3s0s0ev+jhEREUoXC1RnocoGDRpIhRmL9hCtXLlS6SVY1EWdxUzVWWCz6NTtopKSkoSenp5Gxn758qVYunSp6Nq1q2jatKlKlujp0KGD0NHRETVq1BBz5syR1hRUpfnz55c4y+7Fixdi/vz5SsXetGmTePnypVIxygNnmWmg/Px85OfnSyvQb9++HefPn0edOnXw6aefKv2NV0dHRxroWmj37t3w8/PDy5cvS/1NvXAQZUkDZitVqoSaNWti2bJl+PDDD0sVt3Dg4ooVKzBq1CgYGxtL+/Ly8nDhwgXo6uri3LlzpYpblL6+PqpXr46BAwdiyJAhcHV1VTgWoP7iZDo6Om/sUZDJZJg/fz6++OKLd46p7jYXDmbdt28funTpIjd+Ki8vD+Hh4ahXrx4CAwNLHbtQSkoK/Pz8cODAgWKFKjdt2qTUuIV169Zh3rx5WLZsGfz9/bFu3TpER0dj8eLFWLduHQYMGKBwbHX1WuzevRv9+/eHt7d3icVMe/XqpXCbT58+DV9fXzg6OpZYYLNNmzaljlk4A7Fx48Zy4weBgv+RwMBA/PLLLwotC6LO2AAwePBgHD16FH379oWtrW2x96eiS/Tcu3cPGzduxJYtW3Dv3j188MEH+OSTT9CnTx+VDLrX1dUtscf26dOnsLGxUarHtlB2dnaJRR8dHR2Vjq0OTIi00L179+Dg4FBsNsj169cRGhqq8Po4zs7OuHjxIqytrVXRTLRv3x5AwQnYy8tLLhEs7H79/PPPlbq89eTJE2zfvh3btm1DcHAwGjZsiMGDB2PgwIEKTSUummy+abaNorOqTp8+DSEEOnTogN27d8ud3PX19eHk5AR7e3uNavOIESMAFFzm69evn9w6SYV/x1GjRqnk/yYyMlJap6tBgwYqmy33xx9/YN68edKgWXt7e8yfPx/+/v5KxS3pywkAPHr0CNWrV5dmFSoiNDQUP/zwg1SGoEGDBpg2bVqJl/9KKyEhAatWrZJ7rT/77LNS/+8VKprol/SRZGRkhB9//LHYmm/lHRsouAR6+PBhpSuAv8mJEyewYcMG7NmzBwYGBhg4cCBGjhwpDeZWhI6ODpKSkuTWGyt8rP79++Px48cKx46MjMTIkSNx/vx5ue1CBTOD1YkJkYZ69uwZ1q9fL53MXF1dMWLECLkPQG0xYsQIrFixQql6Q+8iJiYGW7duxbZt23D79m20bdtWo8YVFHXv3j04OjqqZfyJusyfPx/Tp0+X6+mraF68eIH09PRiCUxpqbvXoqJR56Kg6l5w1NXVFdu3b1e6jta7eP78ObZu3Yr//e9/SE1NRW5ubqljVK5cGTKZDKmpqcXGsOXl5SE9PR1jxoxRqn5Sq1atoKenh5kzZ6JatWrFzlONGjVSOLY6MSHSQGfOnMFHH30Ec3NzaT2w0NBQpKSk4MCBAwqtJ6XOWh+FJk6cCBcXl2LTNX/66SdERUVh+fLlCsUtS3l5efj7778xZ84chIeHa+w3mY0bN8LU1BQff/yx3PY///wTL168UHoVbHWIiYlBbm5usR69yMhI6dJqab3rOlGqKnKoKurutQBUvyZdXFzcOx2nqZdD1OXvv//GypUrsWbNGjg5OantcWJiYrBp0yZs2rQJDx48gLe3t0KXmTdv3gwhBEaOHInly5fLXU4u7LFVZk1LoKDAa2hoaImD7zUZZ5lpoHHjxqFfv35YvXq19M0lLy8Pn332GcaNG4dr166VOmbRmS3qqgOxe/fuEsegvP/++1iyZEmpEqKySOCKOnfuHP744w/s2rULmZmZ6NGjBxYvXlzqOK8ufPkmytT5WLx4MX755Zdi221sbDB69GilEqKLFy++dsFHRStgA8Dw4cMxcuTIYgnRhQsXsG7dOpw6darUMffs2fPafTKZDBEREcjMzFQqIXr69Cnmzp372tdEkZlVhTNG1dVrERISgkGDBkm9I0UpesmiZs2abyyQWhhbkV6LV928ebPEKv3KLAq6ePFi2NraFksyN2zYgMePHyMgIEChuM2aNUNmZiZq1aoFY2PjYvV8lJl5l5mZiV27dmHDhg04c+YMHBwc4O/vjxEjRsDBwUGhmIXnBmdnZ6knR9VcXV3x5MkTlcdVNyZEGigqKgq7du2SOyHq6upi6tSp2LJli0IxN27cWOLPqvT06dMSky1zc/NSvznKIoEDgFmzZmH79u1ISEhAp06dsGLFCvTo0UPhyzo//PDDOx0nk8mUSoji4uKKVa8FACcnp3f+Jl+SRYsWYfbs2ahXr16xAaLKXp67cuVKieMsWrZsifHjxyscsyRhYWGYOXMmrl+/jlGjRikUu9DQoUMRFRUFf3//EgfNKsLJyQk5OTnw8/NDlSpVVN6zMGbMGDRr1gyHDh0q8ZKFIl73WgshsH37dqxcuVJumrUi7t69i169euHatWtykzQK269Mj+0vv/xS4nI5bm5uUgFERQwcOBAPHjzAokWLVPb/8e+//2LDhg3YsWMHMjMz0atXLwQGBqJjx44qu0zu4uKCn3/+GXfu3AEA1KtXD71791a6AjYAfPPNN5gxYwYWLVpUYtFHdQ9/UFhZTmmjd/P+++9LheCK2rNnj9LTWtXJzc1N/Pjjj8W2r1y5UjRo0KAcWvR277//vli1apV4/PhxeTelVBwcHF5bHE+ZRR9tbGzExo0blWjZ65mbm4vLly8X237p0iVhamqqkse4e/euGDx4sNDT0xP9+vUrcdHN0jI1NZWKo6qahYWFWqZUGxsbi8jISJXHfVVQUJBo2rSpMDMzE19++aXSq91/+OGHokePHuLx48fC1NRU3Lx5U5w9e1Y0b95cnDlzRqnYBgYGJb7W0dHRwsDAQOG4RkZGKv//kMlkonHjxuLHH38UycnJKo0thBCrVq0SBgYGQiaTCQsLC2FhYSFkMpkwMDAQq1atUjp+0aKuRW+avto9e4g00MSJEzFp0iRERUVJa+OEhIRg1apVWLJkidwime86kO9tC68WpejisVOnTsX48ePx+PFjdOjQAQBw/PhxLFu2TKnxQy9fvoQQQuq1uXfvHvbs2QNXV1d07txZ4bgAlJqyX54GDhyIiRMnwszMTBoPcvr0aUyaNEmpaeA6Ojpqmy3Ttm1bLF68GNu2bZO7FLx48WK0bt1aqdhPnjzB/PnzsXbtWrRu3Rrnz59X2VIY9evXx8uXL1US61U9evTA3r17lVpKoySqXpPuVZcvX0ZAQADOnj2LTz75BIcPH1Z6oDlQMH3/xIkTsLa2ho6ODnR0dNC6dWssXrwYEydOVGoNPwcHB5w7d65Yz+q5c+cUnh0HqOf/49KlS/D09FRpzEKHDh3CxIkTMXnyZEybNg3VqlUDUFDM89tvv8WkSZNQs2ZNdOvWTeHHUOdyS+rEQdUa6G2LIxZ2JZdmLMD8+fPf+fEVrZsBFKz79PXXXyMhIQFAwbiDefPmKbUuWOfOndG7d2+MGTMGKSkpqFevHvT19fHkyRN8//33GDt2bKniva3WTlGlHbNQFivSAwX1PYYOHYo///xTGgOQn5+PYcOGYc2aNQrXqlq6dCkSEhLUMgD+5s2baNu2LSwtLaVaNWfPnkVaWhpOnDgBd3f3UsfMyMjAd999h++//x4uLi5YvHix0knyqy5evIiZM2di7ty5cHd3V2n3f+EiqR07dkTTpk2ldQULKXpZdc+ePZg9ezamT5+ukjXpCkVHR+N///sfdu/ejX79+uGrr75S6WrplStXxuXLl+Hs7IzatWtj3bp1aN++PaKjo+Hh4YEXL14oHHvp0qVYunQpvv32W7kvbDNmzMC0adMwa9YsheIePXoU8+fPx9dff10hLg+1a9cOrVu3xldffVXi/tmzZ+Off/5RaExfRceESAPdu3fvnY9V56wGZTx+/BhGRkZKjykAChZKPH36NNzc3LBu3Tr8+OOPuHLlCnbv3o25c+dKpQne1asJ56sFJV+dhloa7du3x549e2BpaSnVUSqJTCZTyZT+O3fu4OrVqzAyMoKHh4fS/w/5+fnw9fXFnTt34OrqWuzkruwA9oSEBPz0009Smxs2bIjx48crXE7Czs4Oz58/x4QJEzBw4MDX9oIqMyU6MjISgwYNKtZzWtovJSUpaRxYIZlMhrt37yoUt6QvVYp8kSrqs88+w/r169G+fXssWbIEjRs3Vqhtb9KmTRtMmzYNPXv2xKBBg/Ds2TPMnj0ba9euRWhoKK5fv65wbCEEZs6ciZUrV0qDtQ0NDREQEKDUoPuihWlffTxNrLljbm6Oixcvol69eiXuj4iIwHvvvYe0tLRSx166dCkmTJgg1Ro7d+4cmjVrJhWSfP78OQICAvDzzz8r/gTUiAkRqVRubi5OnTqF6OhoDBo0CGZmZkhISIC5ubnCyZGxsTFu374NR0dH9OvXD25ubvjyyy8RHx+PevXqKfWt8dixYwgICMCiRYvkqu7Onj0bixYtQqdOnRSOXRGNHz9e+lZe0gBRdQ3IV1TRD/6SEltVfCg1b94cenp6mDRpUomvyQcffKBwbHV525cqRRJnHR0dGBoavnUqtaKX3AHgyJEjyMjIQO/evREVFYUPP/wQd+7cQZUqVbBjxw6pZ0cZ6enpuHXrFoyMjFCnTh2lqz6fOnXqjcMRNO3/w8TEBNeuXXttz97du3fh4eGh0Dp9r1a/Njc3R1hYmPRYSUlJsLe317gksRDHEGmgzZs3w9raGr6+vgCAGTNmYO3atXB1dcW2bduU7gXIy8vDDz/8gJ07d5Y4tVXRaaL37t1Dly5dEBcXh6ysLHTq1AlmZmb45ptvkJWVhTVr1igU18XFBXv37kWvXr1w5MgRabzFo0ePlO6Onjx5MtasWSM3hsXHxwfGxsYYPXp0qXufinr8+HGxKrCFrl27Bg8PD4Vjv60+jaKLu27evBm7d++W/vdULSUlBf/++2+J09cVuayq7sUqgYIK7leuXHntN2pNpI6eY2Uupb8rHx8f6WcXFxfcvn0bycnJUjFBVTA1NVXZ+DKg4BJUReLm5oZ9+/a9dtza3r174ebmplDsV/tXKlp/CxMiDbRo0SKsXr0aQEFvxU8//YTly5fj4MGDmDJlitKXLebPn49169Zh2rRpmD17Nr744gvExsZi7969SnUdT5o0Cc2aNcPVq1dRpUoVaXuvXr2Umvo8d+5cDBo0CFOmTEGHDh2knpyjR48qvQxBdHR0sXWkgIKp/spWCfbw8MD69euLJRffffcd5syZo9RAzGfPnsndz8nJwfXr15GSkqLUt2grKyvUrl1b4d9/kwMHDmDw4MFIT08vViFXJpMplBCVxSXjZs2aSb2R6nD//n3s37+/xC8nyowzi46OxvLly+Wq3U+aNEnhv29ZJEQlUVV1/vbt278xqVL0ErazszNGjBiB4cOHq7wopTomlIwbNw5jx46FgYEBRo8eLY1BzM3NxS+//ILZs2dr7CUttSvraW30dkZGRuLevXtCCCFmzJghhg4dKoQQ4vr168La2lrp+LVq1RIHDx4UQhRMKY6KihJCCLFixQoxcOBAheNaWVmJ27dvS3ELVwWPiYkRRkZGSrX54cOH4vLlyyIvL0/aduHCBXHr1i2l4rZp00Z06tRJJCYmStsSExNF586dRdu2bZWK/c033wgDAwMxZswY8eLFC3H//n3RoUMHUbVqVfHXX38pFbskeXl5YvTo0eKbb75ROMaGDRtEv379SlwFW1l16tQRkyZNUktsddq5c6dwdXUVGzduFJcuXRJXr16Vuynj2LFjwtjYWLi7uws9PT3RuHFjYWlpKSwsLET79u0VjhsYGCj09fVF8+bNxZQpU8SUKVNE8+bNhYGBgTh69KhSbVandu3aifbt27/2pozJkyfL3caNGydatWolLCwsxMSJExWO+8MPP4hGjRoJXV1d4e3tLbZt2yYyMzOVamuhTp06idWrVwshhHj27JmwtbUVNWrUEIaGhuLnn39WOO60adOETCYT5ubmokmTJqJx48bC3Nxc6OjoiMmTJyscVyaTiaSkJOl+0c8BIQrOrZo87Z4JkQaqWrWqVK+lcePGYsuWLUIIIaKiooSJiYnS8Y2NjaWEy87OToSGhgohCupxmJubKxzX0tJS3LhxQwgh/0Y4e/assLGxUbLVBeLj40V8fLxKYgkhRGRkpHB3dxf6+vqidu3aonbt2kJfX1+4ubmppIbN5cuXhZubm3BxcRFWVlaia9eu4uHDhypoeclu374t7OzsFP79xo0bCzMzM2Fqairc3d1FkyZN5G7KMDY2ljs5VhSFNVWK3lRVU+W9994Tc+fOFUL833vm+fPn4qOPPlLqA69x48YiICCg2PaAgACl/47qpK6k5U2+/PJLMW3aNKXjhIaGigkTJghra2tRuXJlMW7cOOncqqgqVaqI69evCyGE+PXXX0XDhg1FXl6e2Llzp6hfv75SsYODg8XEiRNF165dRdeuXcWkSZNEcHCwUjFlMpn4+uuvxYoVK8SKFSuEoaGhmDNnjnT/q6++0uiEiJfMNFCnTp3wySefoEmTJrhz545UD+LGjRsKrff0qho1auDhw4dwdHRE7dq1cfToUXh6euLixYtKDTDs3Lkzli9fjrVr1wIouAySnp6OL7/8UqmaFvn5+dL05PT0dACAmZkZpk2bhi+++OKtZQrexMXFBeHh4QgKCpJbudvb21slYxZcXFzg7u6O3bt3AwD69+8POzs7peO+TnR0tFJLJ/Ts2VN1jXmFj48PLl26pNJp2mVBneOUbt26hW3btgEA9PT08PLlS5iammLBggXo0aNHqUtKFI27c+fOYtsL16/SVK+r9D5v3jzpva9qQ4YMQfPmzfHdd98pFcfT0xOenp5YtmwZfv75ZwQEBGD16tXw8PDAxIkTMWLEiFKfU168eAEzMzMABUMEevfuDR0dHbRs2bJUs5FL0rJlS6nOnao4Ojri119/le7b2dnht99+K3aMpmJCpIFWrVqF2bNnIz4+Hrt375bG44SGhmLgwIFKx+/VqxeOHz+OFi1aYMKECRgyZAjWr1+PuLg4pQrELVu2DD4+PnB1dUVmZiYGDRqEyMhIWFtbSyd9RXzxxRdYv349lixZIhUN/OeffzBv3jxkZmbi66+/Vjg2UJC4de7cWbomL4RAYGAg1q9fj127dikc99y5cxgyZAisrKwQHh6Oc+fOYcKECTh8+DDWrFmDypUrKxz71RpHQgg8fPgQhw4dUngds9zcXMhkMowcORI1atRQuG2v4+vri+nTp+PmzZsl1mtRZp0qdVLnOCUTExNp3FC1atUQHR0tDWhVZi2oqlWrIiwsrNi6cWFhYSopoFjWVJW0lCQ4OBiGhoZKx8nJycGePXuwceNGBAUFoWXLlvD398f9+/fxv//9D8eOHStx6ZA3UeeEEnVQdtxleeO0e0JISAjOnz+POnXqoHv37krFys3Nxfbt2xEeHo709HR4enpi8ODBUl0KRdjb22PNmjXFPjD37duHzz77DA8ePFCqzYViYmKwYcMGbNq0CY8fP4a3tzcOHjyocDwDAwNMmTIFCxculD78o6OjMWTIEMTHx+P+/fsKx361xpGOjg6qVq2KDh06YOTIkQov2GhmZoZr166ppCfyVW/qyVN2arw6q5m/bf1AZYqO9uzZE76+vhg1ahQ+//xz7Nu3D8OHD8dff/2FypUr49ixYwrFXbBgAX744QfMnDkT77//PoCCBP2bb77B1KlTMWfOHIXi5uTkoEuXLlizZk2xZEudfvvtNwQEBEgFXxXx6iLRhV8iLl26hDlz5ig8aPzy5cvYuHEjtm3bBh0dHQwbNgyffPKJXHmC69ev47333iv1RIpdu3Zh0KBByMvLQ8eOHXH06FEABQvVnjlzBn///bdCbabXKM/rdVQ+Tp8+LXJycoptz8nJEadPny6HFr2ZgYGBiIiIKLb99u3bwtDQUKnYmZmZ4vfffxft27cXlSpVEjo6OuL7778XqampSsUVQohTp06VuD0vL08sWLBA6fjq8NFHH4lNmzaVdzNKTV2DT4UoGBtX9GZiYiKt+1S5cmWlYkdHR0sDs9PT08Wnn34qPDw8RO/evUVsbKzCcfPz88X3338vqlevLo17ql69uli+fLnIz89Xqs3W1tYqGV9Xkl69esndevbsKVq0aCF0dXXFvHnzlIo9fPhwudvIkSNFQECAOHLkiFJxdXR0hI+Pj9i5c6fIzs4u8Zj09HQxfPhwheKra0IJFcceIi30avGsQk+fPoWNjU2pvqnv378fXbt2RaVKld66JIapqSnq169f6nWDWrRogRYtWmDlypVy2ydMmICLFy8iJCSkVPGAgsuP69evx7Zt2+Di4oKhQ4eif//+qFGjBq5evQpXV9dSx/wvWLNmDebPn4/BgweXuJSEpl7WUnU187eJjIzE2LFjMX36dLnaOaWRlpaGCxcuIDs7G82bN39tzarSys3NxdatW+Hj4wNbW1s8f/4cAKSxKMqaMmUKDAwMsGTJEpXEK2r48OFy42yK9nyqekkWVbl3757GrhhApcOESAvp6OggKSmp2An4zp07aNasWalKtuvo6CAxMRE2NjbvNLhZV1cXS5cuLdVYpdOnT8PX1xeOjo5y1aTj4+Nx+PBhaV2s0tDT08OECRMwZswYufoylSpVUllCtGDBgjfuL23NJ09PTxw/fhyVK1d+62K9pqamcHNzw//+9z84ODi882Oo87KWql+PotRZzfx1Ll26hCFDhkiD8UsjLCwM3bp1Q1JSEoQQMDMzw86dOxVOrl5lbGyMW7duqeWDesKECdiyZQvq1KlTYtKsTO2kii4zMxM7duxARkYGOnXqpPBlxVcv772OMjXp1HmZuaLioGotUvgmk8lkGD58uNyMsry8PISHh0vjDd5V0WrDr1YeflV2dja2bt2KWbNmlSoh+uCDD3Dnzh2sWrVK+vDp3bs3PvvsM4VXqe7YsSPWr1+PR48eYejQofDx8VFZJdxCe/bskbufk5ODmJgY6OnpoXbt2qVOAHr06CH9zd42GywrKwvHjx/HkCFDcPr06Xd+jLf9DZWh6tejqPIYfKqnp6fwmJaAgAA4Oztj9+7dMDQ0xMKFCzF+/HhERkaqpG3NmzfHlStX1JIQXb9+XVqJ/c6dO3L7lH0P1apVCxcvXpQr7AoUVDj39PQs9dpuVlZWuHPnDqytrd9a7brwS8Q333zzTmvfTZ06FTk5Ofjxxx8BFJzfvLy8cOPGDRgbG2PGjBkICgqSvsSVhoWFRal/p7R69Oght2h2ixYtUKlSJYUXzf4vYEKkRQrfZIXfSIsOdNbX10fLli2Vqij9Nvr6+ujTpw/Cw8NL/bv29vZKzyYr6siRI4iPj8fGjRsxduxYvHz5Ev379weg/Em90JUrV4ptS0tLw/Dhw9GrV69Sxys66PNdBoAWnbGkiMzMTJXMvimk6tejqKLVzDt27KjSauavXgoW/38w7k8//STNeiyt0NBQqdwFULDUipWVFdLS0lSSwH322WeYNm0a7t+/X2IvjjKL3Z48eVLZ5r1WbGxsib2QWVlZCk2e+OGHH6RLhW8rN5CVlYXDhw9jxIgRCA0NfWvso0ePYtGiRdL9P/74A/fu3UNkZCQcHR0xcuRIfPXVVzh06FCp210WawZevnxZKnOwa9cu2Nrayl1mLm1CVJorC5o4Qw7gJTONlJSUhM8//xzHjx/Ho0ePiq0Ho+zCePPnz8fnn39e7CSpiLeNGypKmfEnz549w/r16+WWIRgxYoTKyvoHBQVh48aN2LNnDxwcHNC3b1/07dtX+sBSpWvXrqF79+5lMkU1NTW1VN828/LysGjRIqxZswZJSUm4c+cOatWqhTlz5qBmzZrw9/dXeRtV9XokJibi4cOHaNSokXTp799//4W5uflbFyR9k1cvI8pkMmlcy7Jly1CtWjWFYhZeai5kZmaG8PBwODs7K9zW17UZUN1it4WioqIQHR2Ntm3bwsjISIqtiMLzSM+ePbF582a5/9m8vDwcP34cQUFBiIiIULrdbxIfH4+mTZvi0aNHbz3W3Nwcly9fhouLCwBg4MCBMDMzk+qwFV4WVWZmnDqp+jKzjo7OO//9ubgrvbPhw4cjLi4Oc+bMQbVq1VR+KUeVaxK9eummpBXHCyn6Jjhz5gy6d+8OCwsLNGvWDACwcuVKLFiwAAcOHEDbtm0ViltUp06d0KlTJzx79gy///47NmzYgG+++UYtb9zU1FSkpqYqFeNdk+bSdr1//fXX2Lx5M5YuXSrXW+ju7o7ly5erJSFSxesBFBSBe7XoZfPmzZWOq67LiDdv3kRiYqJ0XwiBW7duSYOgAcV7ctRZTPLp06fo168fTp48CZlMhsjISNSqVQv+/v6oXLkyli1bVuqYhecRmUxWrI5WpUqVULNmTYXiliQ7O7vExYUdHR3h4ODwTskQUJAAFH3fhYSEyJUzsLS0LLbmoCZR9WXmoj2HsbGxmDlzJoYPHy437nPz5s1YvHixap6AGrCHSAOZmZnh7NmzaNy4scpilmZA7uXLlxV6jGPHjiEgIACLFi2SexPMnj0bixYtQqdOnRSK6+HhAS8vL6xevRq6uroACj7wP/vsM5w/fx7Xrl1TKO7bXL58WakeoldnxRVeavntt9/wwQcflLpIW1Fdu3ZFXFwcxo8fX2LS3KNHD4Xiuri44JdffkHHjh1hZmaGq1evolatWrh9+za8vLyUOsGr4/Uoi8Gn6lD4bbqk06+yPTnqmr1WaNiwYXj06BHWrVuHBg0aSP8jR44cwdSpU3Hjxg2FYzs7O+PixYuwtrZWYYsL3LlzB/7+/jh//rzcdkVfay8vL3z88cfSc27YsCGioqKkHr7Tp0/Dz89PY4sVqrPGUceOHfHJJ58UKyS8detWrF27FqdOnVKm6WrDHiIN5ODgUOKJUhmlGZCrqMmTJ2PNmjVo3bq1tM3HxwfGxsYYPXq0wlOfo6KisGvXLikZAgpmq02dOvWtRfOUoezlsleXISicQuzn54dZs2YpFfuff/5RedIMAA8ePJAuARSVn5+PnJwcpWKr4/VQ5+DTt82KK6TIYHB19eCoe/YaUDB25siRI8WqmdepU0fp5STU2bM1YsQI6Onp4eDBgyrpeZ8xYwYGDBiAQ4cO4caNG+jWrZvc5c7Dhw+rpIdSXfr27YvWrVtLl5kLdezYUekxfcHBwVizZk2x7c2aNcMnn3yiVGx1YkKkgZYvX46ZM2fil19+UVnF4NIOyFVEdHQ0LC0ti223sLBQ6luSp6cnbt26JTc9HihYr6noG1nTqPPkro6kGSgYm3X27Nlis5N27dql9OBkdbwe6hx8+uqsuKJkMhkiIiKQmZmpUEKkrro16p69BgAZGRnSVO2ikpOTlVoLEQAmTpwIFxcXTJw4UW77Tz/9hKioKKXWYQsLC0NoaKhS48mK6tWrFw4fPoyDBw+ic+fOmDBhgtx+Y2NjfPbZZyp5LHVR12VmBwcH/Prrr1i6dKnc9nXr1pWqDEhZ4yUzDfHqlNCMjAzk5ubC2Ni42JpPycnJKnnMS5cuyQ1Sbtq0qVLx2rZtC0NDQ/z222+wtbUFUDDWZdiwYcjMzCzV9O+iduzYgRkzZmDChAnSYoQhISFYtWoVlixZggYNGkjHKjN7piwIFa2TdvToUSxbtkylSTNQsBxKYY/NggULMH/+fERERGDLli04ePCgwpc9/0vCwsIwc+ZMnDhxAiNHjizxm3B5sba2lpu9lpKSAisrK6SkpKhsZk+3bt3QtGlTLFy4UBoI7uTkhAEDBiA/P1+p/+vq1atj//79xc5Fly9fxkcffaTUcjfvvfcefvjhB7kebG1UFpeZDx8+jD59+sDFxQUtWrQAUDDBITIyErt371ZqsW91YkKkITZv3vzOxyq6eGeh+/fvY+DAgTh37pzUo5OSkoL3338f27dvV3hhz6ioKPTq1Qt37tyRvgXEx8ejTp062LNnj8JFyt5W8FHVs2fUQdXrpFWuXBkvXrxQS9J89uxZLFiwAFevXpXWo5s7d65Cxdp69+6NTZs2wdzc/K0n4sI6MGPGjCmTOiylFRMTgzlz5mDHjh3o3bs3vvrqqzJdz+tdqHv2GlBQh6hjx47w9PTEiRMn8NFHH+HGjRtITk7GuXPnULt2bYVjGxoa4vr168Uu20ZFRcHd3R2ZmZkKxz5x4oQ0nrGkxYU1dSq4qo0YMeKdjlO29zU+Ph6rV6+Wasc1aNAAY8aMYQ8RaZYuXbogJSUFmzdvli5DRUREYMSIETA3N0dgYKDCsYUQCAoKknsTeHt7K3W9vjTjEkp7KUKd1VqzsrKwa9curF+/Hv/88w/y8vLw3Xffwd/fX+mT79sSaGWTZlUZMWIEVq5cCTMzs2LLMrwqKysLwcHB8PDwKFU5B3V78uQJ5s+fj7Vr16J169ZYsmQJ3nvvvfJuVol0dHRw4sQJuXIU77//Pnbu3Cn3RUfZntTU1FT89NNPcknzuHHjFCpBUJS7uzvGjBmD8ePHy23/8ccfsXr1aty8eVPh2IVfrF79H9T0L1NUhtS5UBopRkdHRyQlJRXb/uTJE6Gjo6N0fENDQ3H58uVi2y9duiSMjIyUjl9Ufn6+OHz4sOjTp49K46qKOhYFvXTpkhg7dqywtLQUzZo1EytWrBCJiYlCT09P3LhxQ5XNVzlnZ2fx5MmTYtufPXsmnJ2d1f74N27cEMbGxmp/nHeRnp4u5s2bJ8zNzYWnp6fSi4CWBZlMJnR0dKQFXYveCrer4hyiLuvXrxdGRkZi7ty54tSpU+LUqVNizpw5wtjYWKxdu1ap2IXxXncj1Tpz5owYPHiw8PLyEvfv3xdCCLFlyxZx9uzZcm7Z63FQtQYSr+m0y8rKgr6+vtLxHRwcSpwxlJeXp/BSGK8q6RJRaZRVwUdVV2sFChajnTBhAkJCQooNBFelvLw87N27VxoH5ubmho8++khuNl5pqbpScKGcnBwYGRkhLCwM7u7urz2uXr16xaZFl5fatWvj+fPnmDBhAgYOHAiZTFZilXVleltU3UOpzoH8RamrUOrIkSORlZWFr7/+GgsXLgQA1KxZE6tXr8awYcOUiv3BBx8o9fv07nbv3o2hQ4di8ODBuHz5MrKysgAU9CwuWrQIhw8fLucWloyXzDRIYZ2WKVOmYOHChTA1NZX25eXl4cyZM4iNjS1xCYTS2LdvHxYtWoRVq1ZJhQ4vXbqECRMmICAgQOFp+aq8RFRSdWCh4oKPgHoWBfXx8UFwcDC6d+8ut06aKheOjYqKQrdu3fDgwQO5y54ODg44dOhQqcdxlEWl4Fq1amHPnj0aPTOwqKL/gyX9/wkVXGrp3Lmz3HpS9evX1/j1pEoqlBoaGoqUlBSVFUoFgMePH8PIyEjuPKislJQUuUTOzc0NI0eOVGrMGhdJLa5JkyaYMmUKhg0bJlfP7MqVK+jatatcQVKNUn6dU/SqmjVripo1awqZTCYcHByk+zVr1hR169YVnTt3FiEhIUo/jqWlpdDX1xc6OjpCX19f7ufKlSvL3d6Fui8RBQUFCU9PTxEYGChSU1NFamqqCAwMFM2aNRNHjx5VKraHh4dYsWKFiIuLE+bm5uL8+fPSc7K1tVU4blxcnJg/f76oWbOmsLW1FRMnThR6enri5s2bSrW3UNeuXUWXLl3E06dPpW1PnjwRXbp0Ed26dSt1vFcvqxS96evri7p164oDBw4o1eZ169aJbt26ybVZk8XGxr7TTRlVqlQR169fF0II8euvv4qGDRuKvLw8sXPnTlG/fn1VPA2Vc3d3F6NGjRK5ubnSttzcXDF69Gjh7u5eji17s4sXLworKytRvXp10atXL9GrVy9Ro0YNUaVKFREaGqpwXHVcdq/ojIyMRExMjBBCCFNTUxEdHS2EECI6OloYGBiUY8vejAmRBmrXrp1ITk5WW/xNmza98+1d6OrqismTJ4vbt2/LbVdVQuTm5lbideczZ84o/aHx559/ikqVKgkdHR3RqVMnafuiRYtEly5dlIpd6OjRo2LgwIHC0NBQ1KlTR8yaNUupE7AQQhgbG4vw8PBi28PCwoSJiYnCcWvWrCkeP36sTNNeq3HjxsLU1FQYGBiIunXriiZNmsjdtJGRkZG4d++eEEKIjz/+WMybN08IUZBQq3o8n6oYGhoWe68LIcTt27eFoaGhUrETExPFkCFDRLVq1YSurq7Q0dGRuymjdevWYvjw4SInJ0falpOTI/z8/ESbNm0UjlsRk1p1c3Z2FkFBQUII+YRo8+bNokGDBuXZtDfiGCINpM7VpAHVz0Dq2LEj1q9fj0ePHsldIlIVdRV8BNRbrbWQOtZJMzAwkFvzqlB6erpS48yKjkFR9Wr36qqQXpGpej2psqDOQqnqXMfx0qVL+PXXX6Gn938fe3p6epgxY4Z06U8RL168gJmZGYCC+mC9e/eGjo4OWrZsqXTl7opq1KhRmDRpEjZs2ACZTIaEhAQEBwfj888/l1vvTeOUd0ZGBaZMmSLS09Oln990U6WXL19Kl6EKb4pQ5yWiNm3aiE6dOonExERpW2JioujcubNo27at0vHLg7I9REOHDhVubm4iJCRE5Ofni/z8fBEcHCzc3d2Fn5+fwnHz8vLEggULhL29vdDV1ZW+2c2ePVusW7dOqTZTcWXRQ6lq27dvF46OjuLbb78VZ8+eFWfPnhXffvutqFmzpti+fbu4evWqdCstU1NTceXKFdU3WghhY2NT4kzBwMBAYWNjo3BcdV12r8jy8/PFV199JUxMTKRL74aGhmL27Nnl3bQ34qBqDdG+fXvs2bMHlpaWaN++/WuPk8lkOHHihFKPlZGRgYCAAOzcuRNPnz4ttl/ZehxBQUHYuHEj9uzZAwcHB/Tt2xd9+/ZVeG0wdRR8rKiLghZKSUmBn58fDhw4IBWYy83NxUcffYRNmzYpPEh0wYIF2Lx5MxYsWIBRo0bh+vXrqFWrFnbs2IHly5cjODhY6Xbv2rUL0dHRmD59OqysrHD58mXY2tqievXqSsWuqBITE6UeysKB3P/++y/Mzc0VXmZCnQN91Vko1dXVFX/88YfSy8SUZOLEidizZw++++47vP/++wCAc+fOYfr06ejTp4/Cy4Koc5HUii47OxtRUVFIT0+Hq6urSgfIqwMTIi00btw4nDx5EgsXLsTQoUOxatUqPHjwAL/88guWLFmCwYMHq+Rxil4iCg8PVyrREiou+FhW1VrVLSoqSpox06BBgxIXZi0Nda52Hx4eDm9vb+lSZ0REBGrVqoXZs2cjLi5OrQv1aht1zl5TZ6FUdS1JAxR8OE+fPh1r1qxBbm4uAKBSpUoYO3YslixZotQ6bOpIaqnsMSHSQCdOnECrVq2UXijxdRwdHbFlyxa0a9cO5ubmuHz5MlxcXPDbb79h27ZtaqkRcfnyZaVXjy9KqGhNMJJnZGSE27dvw8nJSS4hunnzJpo3b4709HSFY3t7e8PT0xNLly6Vi33+/HkMGjRI6fFg6qKO3hZ191BaW1vj9OnTcHNzw7p16/Djjz/K1dcqTKI1jTqXpCn04sULREdHAyioNVXSQrWknMzMTPz44484efIkHj16hPz8fLn9ly9fLqeWvRkHVWugjz76CLm5uXjvvffQrl07fPDBB2jVqhWMjIxUEj85ORm1atUCULB+T+FJpnXr1mqre6KqZEjZgo//FX369EHz5s0REBAgt33p0qW4ePEi/vzzT4XiqnO1+4sXL+KXX34ptr169eqaW5cEQI8ePeR6W1q0aKF0b4u612pT90Df6OhoLF++XK4w46RJk5RaxwyAUqvZv01qairy8vJgZWUFDw8PaXtycjL09PRKPYi9ol92Vyd/f38cPXoUffv2RfPmzVU6OF6dmBBpoGfPnuHff//F6dOncfr0aSxfvhzZ2dlo1qwZ2rdvj6+++kqp+LVq1UJMTAwcHR1Rv3597Ny5E82bN8eBAwdKnM1V3tS5Jpg6qXMcx5kzZzBv3rxi27t27Yply5YpHHfu3Lnw8/PDgwcPkJ+fj7/++ktutXtlGBgYIC0trdj2O3fuoGrVqkrFVid1VDNX96VYdc5eO3LkCD766CM0btwYrVq1AlAwFsfNzQ0HDhxAp06dFI6tzjX4BgwYgO7du+Ozzz6T275z507s37+/1D3jmrgAsaY4ePAgDh8+LP1/VBjlM5abSuP69evCz89P6OnpqWQdou+//16sWLFCCFFQ9NDQ0FAYGBgImUwmli9frnR8VanIa4IJod6Cba+rBXPr1i2la8GcOXNGeHt7i6pVqwojIyPRqlUrlazj5e/vL3r27Cmys7OFqampuHv3rrh3755o0qSJmDRpktLx1aUi1gpS5+y1xo0bi4CAgGLbAwIClK4nde/evTfelFG5cuUSZ73eunVLWFlZKRWb5DVo0EChWYbljQmRBoqIiBC//PKLGDhwoLC3txdVqlQRPXv2FMuXLxdhYWEqf7zY2Fixe/fuEgv9lSd1F3xUN3UWbHvvvffE/Pnzi23/8ssvhaenp1Kx1SUlJUV4e3sLS0tLoaurKxwcHESlSpVE27ZtpZITmqiiTqt++PChuHz5ssjLy5O2XbhwQdy6dUupuAYGBuLOnTvFtkdERChdhbiwWvrrbsp4XTHT8PBwjU1sK6rDhw+LLl26KF3JvazxkpkGql+/PqpWrYpJkyZh5syZ8PDwUMk12BMnTmD8+PEICQmR6zZ3cnKCpaUl3n//faxZswZt2rRRKL6qLxGpu+CjuqlzHMecOXPQu3dvREdHo0OHDgCA48ePY9u2bQqPHwIKBquHhoYiNjYWMpkMtWrVQuPGjVXyultYWCAoKAjnzp3D1atXkZ6eDk9PT40fBzZ37lwMGjQIU6ZMQceOHeHl5QWg4G+qjunhqmJnZwc7Ozu5bc2bN1c6btWqVREWFlas3EVYWBhsbGyUiv3qOo05OTm4cuUKvv/+e3z99ddKxW7evDnWrl2LH3/8UW77mjVr0LRpU6Vik7xmzZohMzMTtWrVUtvgeHVgQqSBJk6ciDNnzmDBggU4ePAg2rVrh3bt2qF169ZKzYhYvnw5Ro0aVeIYAgsLC3z66af4/vvvFU6IVD349MiRI4iPj8fGjRsxduxYvHz5Ev379weACpEYqXMcR/fu3bF3714sWrQIu3btgpGRERo2bIhjx44pvKr3yZMn4e/vj3v37kkLmcpkMjg7O2PDhg1KL9q5ZcsW9O/fH61atZIbW5CdnY3t27crvZq5upRFNXNVKYuBvqNGjcLo0aNx9+5duXo+33zzDaZOnapwXAAlVrpu1qwZ7O3t8e23377z8yvJV199BW9vb1y9ehUdO3YEUPAl4uLFi1LtIFKNgQMH4sGDB1i0aBFsbW0rxPka4LR7jZaSkoKzZ89Kg6tv3LiBJk2a4Ny5cwrFc3JyQmBgIBo0aFDi/tu3b6Nz586Ii4tTKL66p/qquuCjulWkgm1RUVFo1KgRWrRogUmTJqF+/foQQuDmzZtYuXIlLl26hPDwcGl2oiJ0dXXx8OHDYr0IT58+hY2NjdIFQals6msJIbB8+XIsW7YMCQkJAAB7e3tMnz4dEydOVMuHX+H/Z0ZGhlJxwsLC8O3/a+/uw3q+9z+AP7/flFaiSMkmpZLI/U2ZbWeJsuaSOs5u2DTb3HSOOGHDNsLOxjYyN7txDruwsXEcc2Y2miRyEyvJXUhUqKZIuvFTX+/fH119j69upPr2eX/q+bgu17Xv59P18dwmvT7vm9f7s8+QlJSkf4mYO3dunZq7UvUsLCxw5MiReh/l0thYEEksLy8PsbGxiImJwf79+3H27FnY2NggNze3Ts8zNzfH6dOnq23el5qaip49e6KkpKROz7ewsEBKSgocHR3x0ksvoUePHoiIiEBmZibc3d1RXFxcp+c+rCEbPhqbWhq2TZ06FefOnUN0dHSle0IIDBs2DN27d6803fA4tFotcnJyKu0oO3nyJHx8fKQbRue26srKysqwefNm+Pv7w97eXn+eXsXUcH09vAtRCIGsrCwsWLAAKSkpSEpKapDfh4yrX79++PLLL+Ht7a10lMfCKTMJTZs2zaAAeu655zBx4kQ8//zzBv0zHteTTz5ZY0GUnJwMBweHOj+/sQ6qtLGxQVhYGMLCwqRt8FXBWOs4Gtr+/fuxePHiKu9pNBr8/e9/x9y5c+v07L59+0Kj0UCj0cDX19fgcE2dTofLly9jxIgRdXq2MXFbdWUtWrTAlClT9KO9DVUIVbC2tq40wiSEQKdOnfDDDz806O9FxrNkyRLMnDkTH330EXr27FlpDZGs7VJYEEkoKysLkyZNwvPPPw9PT88Ge25AQADmzZuHESNGVDrFvKSkBBERERg5cmSdn6/E4lMZp8vUOLKQkZFRY7Ht6elZ54XgFafcJyUlwd/f3+A8IzMzMzg5OeHPf/5znZ5tTLIf26KUQYMG4cSJE499LEdtxMTEGHzWarVo3749XF1dDQppklvFC07FWq0Koo5n3DUWTpk1Izk5OejXrx9MTEwwdepUuLu7AyhfO/TFF19Ap9PpD9qsK7VMERmTGs9J02q1yM7OrnaXUE5ODjp27Fivv8g2bNiAl19+uVIxTuqydetWzJ07F+Hh4ejfvz8sLS0N7vfq1atOzy0oKEB8fDzu3buHQYMGSd2sk2oWGxtb4/26bvwwNhZEzUx6ejpCQ0OxZ88eg51E/v7++OKLL+Ds7KxwQlKCVqvFvn370LZt2yrv5+bmYvjw4Q3yZnfv3r0qzzdydHSs97PJ+Ko67b4+J9wD5aOHAQEByMnJgRACVlZW2Lp1K/z9/RsiMlGtsCBqpm7duoXU1FQIIeDm5gYbG5s6P0uNU0RqV932Zo1GA3Nzc7i6uiIwMLDaAudhWq1W/0Otqmc2xFD3xYsX8eabb+Lw4cMG12UfRidDj5o6rctUmr+/PwoLC7F06VKYm5vjww8/xKlTp3Dx4sW6xiQJFBcXIyMjA/fu3TO4XtdRRGNjQUT1ZuwpImOeCaZWPj4+SExMhE6n0099XrhwASYmJujWrRvOnz8PjUaDuLg4dO/e/ZHPq+36oPqsGxkyZAhatGiBOXPmwMHBodLiWbVt0W2OjDWtZWtri6ioKP2awPz8fLRt2xb5+fkNtgA3KCioypYAD75EjB07Vv/9RHV348YNTJgwodrWItK+/DRKP2yiejDmmWBqtXz5chEcHCxu376tv5afny/GjBkjPv/8c1FUVCQCAwOFn5+fgikNWVhY1PvYCFLOiRMnhIODg9BqtUKj0YjWrVuL3bt3N8izNRqNyMnJMbhWcd5dQwkJCRFt2rQRnTt3FsHBwSI4OFg4OTkJa2tr8dJLLwl3d3fRsmVLERcX12C/Z3M1duxYMWTIEHH8+HFhaWkpoqKixLfffivc3d3Fzz//rHS8arEgklBGRobIzMzUf46PjxfTp08Xa9asUTCVcox5JphadezYscrz3E6fPi06duwohBAiISFBtGvXrrGjVWvAgAHi4MGDSsegOvLz8xNPP/20OHz4sEhMTBRBQUHC1dW1QZ6t0WhETEyMOHnypP6XpaWl2LVrl8G1+pg9e7YIDQ01ONtNp9OJqVOnirlz54r79++LSZMmiSFDhtT3X6fZ69Chg4iPjxdCCGFlZSXOnz8vhBDiv//9r9T/fVkQSeiZZ54RGzduFEKUH9DYunVrMXjwYGFra1vlgZ5NnRpPGzc2S0tLERMTU+l6TEyMaNWqlRBCiEuXLgkrK6tGTla96OhoMXjwYBETEyNyc3PF7du3DX6R3Nq1aycSEhL0n2/duiU0Gk2D/L+rONRVo9FU+lVxvb6Hu9ra2up/MD/o/Pnz+heH5ORk0aZNm3r9PlReBF2+fFkIIYSjo6N+1C0tLU3qv7PZ2EFCp0+f1jfv27p1Kzw9PXHo0CFERUVhypQpmD9/vsIJG1djNXxUk8DAQLz55ptYtmwZBg4cCAA4fvw4Zs2ape/7c+zYMXTt2lXBlIYqDnFVW28SKnfz5k089dRT+s/W1tawtLREXl5evb8PL1++XN94j1RWVoaUlJRK3xMpKSn6P3vm5uaqOXdLZu7u7jh//jycnJzQu3dvrFmzBk5OTvj666/r1fzX2FgQSai0tBQtW7YEAOzduxejRo0CAHTr1g1ZWVlKRlOEWk8bN6Y1a9YgPDwcr7zyCsrKygCUdxEOCQnB8uXLAZT/eVm7dq2SMQ083HSP1Ofs2bPIzs7WfxZC4Ny5c/ojPIC67SAyRpPHh73++ut466238N577xm8RHz88cf6g4UrzmKk+pk+fbr+Z1VERARGjBiBTZs2wczMDOvXr1c2XA24y0xCXl5e8PHxwYsvvgg/Pz8cPXoUvXv3xtGjRzFmzBhcvXpV6YiNjg0fq1ZYWIi0tDQAQJcuXQy6QNcFd/RRdRqjNYMx6XQ6LFmyBKtXr0ZOTg4AwN7eHmFhYZg9ezZMTEyQkZEBrVZrMBJG9VdcXKw/59LW1lbpONViQSSh/fv3IygoCAUFBQgJCcE333wDAHjvvfeQkpLCfj5kNH5+fggODsaUKVOQn5+Pbt26wdTUFLm5uYiMjERoaOhjPzM5OblWXydrbxIq1xitGRpLxSGyzXXKnarGgkhSOp0OBQUFBg0Tr1y5AktLy2bT0p4NH6tXVFSEJUuWIDo6usquzxWjRo/L1tZWP22wdu1arFq1CidOnMB//vMfzJ8/X3+o5+OoaWShgswjC0RUO4sWLarV18m6DpZriCQ0dOhQbN++vVL36LZt22L06NHYt2+fQskaF08br97bb7+N2NhYvP7661U2Oayr4uJi/QnmUVFRCA4Ohlarhbe3d50Pd22MBbNEj5KTk4NZs2bpXyIeLtBZkNffjz/+WOP9Cxcu4O7duyyIqPb2799fqdU5ANy9excHDx5UIJEyZDr8VDa//vordu3ahSFDhjToc42xo08NUygkB2OuYXvjjTeQkZGBefPmNehLBP3PiRMnqryelJSEOXPm4MyZM5g4cWIjp6o9FkQSeXCtxcO7OXQ6HXbv3o0nn3xSiWgkGRsbm1qfU/Y4uKOPlBQYGGiwhs3Ly6vea9gqxMXF4eDBg+jTp0/DBaYaXb58GfPmzcOWLVsQHByMM2fOwM3NTelY1VOi+RFVraL5WHUNyiwsLMS6deuUjkkS+Pbbb8WYMWNEUVFRgz87KytLJCYmGnT0jY+P57EbZHTG7Erv4eEhEhMTGyImPcKNGzfE1KlThZmZmRg6dKg4duyY0pFqhYuqJZKeng4hBLp06YJjx44ZLJ42MzODnZ0dTExMFExIsujbty8uXboEIQScnJxgampqcD8xMVGhZNTUGXNay8LCQr89+6WXXkKPHj0QERGBzMxMuLu7o7i4uM7PjoqKwrJly/RNAqnhFRUVYenSpYiMjISrqysWL16sqnYdnDKTSOfOnVFaWoqQkBC0a9eOay+oWhXdqBsKd/RRbRlzWsuYXelffvllFBcXw8XFBRYWFpVeIm7evFmv5xPg4uKCO3fuICwsDK+++io0Gk2VbTdkbbHBESIJWVtb48SJE3B2dlY6CjUTEyZMqNXX1WehO5s+Ng3GaM1QYdu2bRg7dix0Oh18fX0RFRUFAFi8eDEOHDiAX3/9tc7P3rBhQ433Q0JC6vxsKlfRNBdApVYbamjeyYJIQiEhIejTp4/+7YioKTBG00dqfMac1gLYlV7N1N68k1NmEnJzc8OiRYtw6NAh9O/fH5aWlgb3p02bplAyUlLbtm1x4cIF2NrawsbGpsZtwzIO/ycmJurPWdu2bRvs7e0NRhZYEKmDsQ9b7tChAzp06GBwreKw68dVUFCgz1TRnbo67Fpdf7IWOrXFgkhC69atg7W1NRISEpCQkGBwT6PRsCBqppYvX65vmrh8+XLV9VExRtNHanzGaM1grDVsNjY2yMrKgp2dHaytrav8npF9GocaDwsiCbGzL1XlwTUOb7zxhnJB6sjYIwvUOMaMGYNnnnlGP61VwdfXF0FBQXV6prG60u/bt0/frysmJsYovwc1HVxDRKRCJiYm+jffB+Xl5cHOzk7Kt11jLpglepSMjAx06tSp0iiREAKZmZlwdHRUKBnJggWRpK5evYqffvoJGRkZlY7xiIyMVCgVyUKr1SI7O7tSQXT9+nW4uLigpKREoWQ144JZ9VJ7awY1vkRQ4+KUmYSio6MxatQodOnSBSkpKfD09MSVK1cghEC/fv2UjkcKWrlyJYDytWRr165Fq1at9Pd0Oh0OHDggdWHRkAtmqXGp/bDlirVCDyssLIS5ubkCiZoutbbY4AiRhAYNGoQXXngBCxcuhJWVFU6ePAk7OzuMGzcOI0aM4G6cZqyiN1V6ejqeeuopg87lZmZmcHJywqJFi+Dl5aVUxErUPrJA6jZjxgwAwIoVKzBx4kT9D2mg/CUiPj4eJiYmOHTokFIRmxy1ttjgCJGEzp07h++//x4A0KJFC5SUlKBVq1ZYtGgRAgMDpf3DRMZXseDex8cH27dvh42NjcKJHk3tIwukbhUnsAshcOrUKZiZmenvmZmZoXfv3pg1a5ZS8ZoktbbYYEEkIUtLS/26IQcHB1y6dAk9evQAAOTm5ioZjSShph0z9eluTVRfFd8rEyZMwIoVK7ijsRGotcUGCyIJeXt7Iy4uDh4eHggICMDMmTNx6tQpbN++Hd7e3krHI0lw4T1R7bEwbzxqbbHBgkhCkZGRKCwsBAAsXLgQhYWF2LJlC9zc3PiDjgBw4T1RXfz+++/YunVrlS8RXMPWcIzRvLMxcFE1kQpx4T3R4/nhhx8wfvx4+Pv7IyoqCn5+frhw4QJycnIQFBTEEaQGpsYWGyyIJNSlSxccP34c7dq1M7ien5+Pfv36IS0tTaFkJAsrKyskJSXBxcUFNjY2iIuLQ48ePXDy5EkEBgbiypUrSkckkkqvXr0wefJk/O1vf9O/RDg7O2Py5MlwcHDAwoULlY5ICuOUmYSuXLlSZZOw//u//8O1a9cUSESy4cJ7osdz6dIlvPjiiwDKd5cVFRVBo9EgPDwcQ4cOZUHUANTeYoMFkUR++ukn/T/v2bPHYLuyTqdDdHQ0nJycFEhGsuHCe6LHY2Njgzt37gAAnnzySZw+fRo9e/ZEfn4+iouLFU7XNKi9xQanzCRSMc+q0Wjw8P8WU1NTODk5YdmyZRg5cqQS8UgiaWlpKCwsRK9evVBUVISZM2fi8OHD+oX3nTt3VjoikVTGjh2LAQMGYMaMGfjwww+xatUqBAYG4rfffkO/fv2kHbWgxsOCSELOzs44fvw4bG1tlY5CRNQk3Lx5E3fv3kXHjh1x//59fPrpp/qXiA8++EAVTU7JuFgQEanYvXv38Mcff+D+/fsG13lyNxHR4+EaIokcOXIEeXl5BlNiGzduREREBIqKijB69GisWrUKLVu2VDAlyeDChQt46623cPjwYYPrFQdY8uRuosru37+P1NTUKl8innvuOYVSkSxYEElk0aJFeP755/UF0alTp/DWW2/hjTfegIeHBz777DN07NgRCxYsUDYoKW7ChAlo0aIFfv75Zzg4OFR5ijcR/c/Ro0cxduxYpKenV1qjyZcIAjhlJhUHBwfs3LkTAwYMAAC8//77iI2NRVxcHADg3//+NyIiInD27FklY5IELC0tkZCQIG2DMyLZ9OnTB127dsXChQurfIlQ+w4pqj+OEEnk1q1bsLe313+OjY3FCy+8oP88cOBAZGZmKhGNJNO9e3f2GyJ6DBcvXsS2bdvg6uqqdBSSlFbpAPQ/9vb2uHz5MoDyxbKJiYkGPWXu3LkDU1NTpeKRRD755BO8++672L9/P/Ly8lBQUGDwi4gMeXl5ITU1VekYJDGOEEkkICAAc+bMwSeffIIdO3bAwsICzz77rP5+cnIyXFxcFExIshg2bBgAwNfX1+A6F1UTVS0sLAwzZ85EdnY2evbsWenlslevXgolI1lwDZFEcnNzERwcjLi4OLRq1QobNmxAUFCQ/r6vry+8vb3x0UcfKZiSZBAbG1vj/T/96U+NlIRIHSoa3z6oogkuXyIIYEEkpdu3b6NVq1YwMTExuH7z5k20atUKZmZmCiUjIlKn9PT0Gu+zuzuxICJSieTkZHh6ekKr1SI5ObnGr+XwPxHR42FBRKQSWq0W2dnZsLOzg1arrfLMO4A9VYgq/PTTT3jhhRdgampqcHh2VUaNGtVIqUhWLIiIVCI9PR2Ojo7QaDQc/ieqhYdfIqrDlwgCWBARERERcds9kVo8asj/QRz+JyJ6PBwhIlKJh4f8H15D9OBRBBz+JwJWrlxZ66+dNm2aEZOQGrAgIlKhvXv3Yvbs2fj4448xePBgAMCRI0fwwQcf4OOPP8bw4cMVTkikPGdnZ4PPN27cQHFxMaytrQEA+fn5sLCwgJ2dHdLS0hRISDJhQUSkQp6envj666/xzDPPGFw/ePAgJk2ahHPnzimUjEhOmzdvxpdffol169bB3d0dAHD+/HlMnDgRkydPxrhx4xROSEpjQUSkQk888QSOHz8OT09Pg+vJycnw8vJCSUmJQsmI5OTi4oJt27ahb9++BtcTEhIwZswY/TmS1HzxcFciFRo4cCBmzJiBnJwc/bWcnBy88847GDRokILJiOSUlZWFsrKyStd1Op3B9xE1XyyIiFRo3bp1yMrKgqOjI1xdXeHq6gpHR0dcu3YN69atUzoekXR8fX0xefJkJCYm6q8lJCQgNDRUf1gyNW+cMiNSKSEEfvvtN6SkpAAAPDw8MGzYMIPdZkRU7saNGwgJCcHu3bv1J92XlZXB398f69evh52dncIJSWksiIhUprS0FE888QSSkpIqrSEiosqEEMjMzET79u1x9epV/aaDbt26oWvXrgqnI1mwMSORypiamsLR0ZG9hohqSQgBV1dXnDlzBm5ubnBzc1M6EkmIa4iIVOj999/He++9h5s3byodhUh6Wq0Wbm5uyMvLUzoKSYxTZkQq1LdvX6SmpqK0tBSdO3eGpaWlwf0HF44SEbBz5058+umn+OqrrzjVTFXilBmRCo0ePVrpCESqMn78eBQXF6N3794wMzPDE088YXCfo63EESIiImryNmzYUOP9kJCQRkpCsmJBRKRiCQkJ+h0zPXr0qNSFl4iIaodTZkQq9Mcff+CVV17B/v37DQ6q9PHxwQ8//ID27dsrG5BIQjqdDjt27DB4iRg1ahRMTEwUTkYy4AgRkQq9/PLLSEtLw8aNG+Hh4QEAOHv2LEJCQuDq6orvv/9e4YREcklNTUVAQACuXbtmcLhrp06dsGvXLri4uCickJTGgohIhdq0aYO9e/di4MCBBtePHTsGPz8/5OfnKxOMSFIBAQEQQmDTpk1o27YtACAvLw+vvfYatFotdu3apXBCUhqnzIhU6P79+/rjBx5kamqK+/fvK5CISG6xsbE4evSovhgCgHbt2mHJkiUYMmSIgslIFmzMSKRCQ4cOxfTp03H9+nX9tWvXriE8PBy+vr4KJiOSU8uWLXHnzp1K1wsLC2FmZqZAIpINCyIiFVq9ejUKCgrg5OQEFxcXuLi4wNnZGQUFBVi1apXS8YikM3LkSEyaNAnx8fEQQkAIgaNHj2LKlCkYNWqU0vFIAlxDRKRSQgjs3bu30mn3RFRZfn4+QkJCsHPnToPT7keNGoX169ejTZs2CickpbEgIiKiZuPixYs4d+4cNBoNPDw84OrqqnQkkgQLIiKVio2NxdKlS/U9Vbp374533nkHzz77rMLJiORW8WNPo9EonIRkwjVERCr03XffYdiwYbCwsMC0adMwbdo0mJubw9fXF5s3b1Y6HpGU1q1bB09PT5ibm8Pc3Byenp5Yu3at0rFIEhwhIlIhDw8PTJo0CeHh4QbXIyMj8a9//Us/akRE5ebPn4/IyEiEhYVh8ODBAIAjR45g9erVCA8Px6JFixROSEpjQUSkQi1btsSZM2cqrX9ITU2Fp6cn7t69q1AyIjm1b98eK1euxKuvvmpw/fvvv0dYWBhyc3MVSkay4JQZkQp16tQJ0dHRla7v3bsXnTp1UiARkdxKS0sxYMCAStf79++PsrIyBRKRbNipmkiFZs6ciWnTpiEpKQlPP/00AODQoUNYv349VqxYoXA6Ivm8/vrr+OqrrxAZGWlw/Z///CfGjRunUCqSCQsiIhUKDQ1Fhw4dsGzZMmzduhVA+bqiLVu2IDAwUOF0RHKYMWOG/p81Gg3Wrl2LqKgoeHt7AwDi4+ORkZGB8ePHKxWRJMI1RERE1CT5+PjU6us0Gg327dtn5DQkOxZERCpy69YtfPfddwgJCUHr1q0N7t2+fRsbN26s8h4REdWMi6qJVGT16tU4cOBAlQVPmzZtcPDgQZ5lRkRUBxwhIlKRPn36YNmyZdWeaB8dHY1Zs2bhxIkTjZyMSG4+Pj41dqbmlBlxUTWRily6dAlubm7V3ndzc8OlS5caMRGROvTp08fgc2lpKZKSknD69GmEhIQoE4qkwoKISEVMTExw/fp1ODo6Vnn/+vXr0Go5E070sOXLl1d5fcGCBSgsLGzkNCQj/s1JpCJ9+/bFjh07qr3/448/om/fvo0XiEjlXnvtNXzzzTdKxyAJcISISEWmTp2KV155BU899RRCQ0NhYmICANDpdPjyyy+xfPlyHu5K9BiOHDkCc3NzpWOQBLiomkhl3n//fSxevBhWVlbo0qULACAtLQ2FhYV45513sGTJEoUTEsknODjY4LMQAllZWfj9998xb948REREKJSMZMGCiEiFjh07hk2bNiE1NRVCCHTt2hVjx47FoEGDlI5GJKUJEyYYfNZqtWjfvj2GDh0KPz8/hVKRTFgQERERUbPHNURERNSs3L17F1u2bEFRURGGDx9eYysLaj44QkRERE3WjBkzUFpaqu/gfu/ePQwaNAhnz56FhYUFysrK8Ntvv2Hw4MEKJyWlcds9ERE1WVFRURg+fLj+86ZNm5CRkYGLFy/i1q1b+Mtf/oJ//OMfCiYkWbAgIiKiJisjIwPdu3fXf46KisKYMWPQuXNnaDQaTJ8+nUfdEAAWRESqVFJSguLiYv3n9PR0fP7554iKilIwFZF8tFotHlwZcvToUXh7e+s/W1tb49atW0pEI8mwICJSocDAQGzcuBEAkJ+fDy8vLyxbtgyBgYH46quvFE5HJA8PDw/s3LkTAHDmzBlkZGTAx8dHfz89PR329vZKxSOJsCAiUqHExEQ8++yzAIBt27bB3t4e6enp2LhxI1auXKlwOiJ5vPvuu5g7dy58fX3h6+uLgIAAODs76+//8ssv7N9FAFgQEalScXExrKysAJSviQgODoZWq4W3tzfS09MVTkckj6CgIPzyyy/o1asXwsPDsWXLFoP7FhYW+Otf/6pQOpIJt90TqVCvXr3w9ttvIygoCJ6enti9ezcGDx6MhIQEvPjii8jOzlY6IhGRqnCEiEiF5s+fj1mzZsHJyQleXl76HipRUVE87Z6IqA44QkSkUtnZ2cjKykLv3r2h1Za/2xw7dgytW7dGt27dFE5HRKQuLIiIiIio2eNZZkQqEhwcXKuv2759u5GTEBE1LSyIiFSkTZs2SkcgUqWSkhIIIWBhYQGgvP/Qjz/+iO7du8PPz0/hdCQDTpkREVGT5+fnh+DgYEyZMgX5+fno1q0bTE1NkZubi8jISISGhiodkRTGXWZERNTksZkpPQoLIiIiavLYzJQehQURERE1ea6urtixYwcyMzOxZ88e/bqhP/74A61bt1Y4HcmABRERETV5bGZKj8JF1URE1CywmSnVhAURERERNXvsQ0RERE0Wm5lSbbEgIiKiJovNTKm2OGVGREREzR53mREREVGzx4KIiIiImj0WRERERNTssSAiIiKiZo8FERERETV7LIiIiIio2WNBRERERM0eCyIiIiJq9v4f9/YAuL+XbYUAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# visualize the occurences of TRUE per column\n", - "count = df[df==1].count()\n", - "index, values = count.index[1:], count.values[1:]\n", - "\n", - "plt.figure()\n", - "#_ = plt.pie(values, autopct='%1.1f%%')\n", - "plt.bar(index, values)\n", - "# the values are plotted above the bars\n", - "for i, value in enumerate(values):\n", - " plt.text(i, value+50, str(value), ha=\"center\", color=\"grey\")\n", - "plt.ylabel(\"Occurences\")\n", - "# rotate x labels\n", - "_ = plt.xticks(rotation=90)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv_tinyevals", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/training_demo.ipynb b/notebooks/training_demo.ipynb deleted file mode 100644 index 18c7e533..00000000 --- a/notebooks/training_demo.ipynb +++ /dev/null @@ -1,45 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from delphi.train.config.utils import load_preset\n", - "from delphi.train.training import run_training\n", - "from delphi.train.utils import ModelTrainingState\n", - "from delphi.train.run_context import RunContext\n", - "\n", - "\n", - "def train() -> tuple[ModelTrainingState, RunContext]:\n", - " config = load_preset(\"v0-llama2-100k\")\n", - " config.wandb_config.entity = \"jaiwithani\"\n", - " return run_training(config)\n", - "\n", - "model_train_result = train()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "tinyevals", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/vis_demo.ipynb b/notebooks/vis_demo.ipynb deleted file mode 100644 index 842804d0..00000000 --- a/notebooks/vis_demo.ipynb +++ /dev/null @@ -1,148 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import torch; torch.set_grad_enabled(False)\n", - "from transformers import AutoTokenizer, AutoModelForCausalLM\n", - "\n", - "from delphi.eval.utils import tokenize, get_next_and_top_k_probs, load_validation_dataset\n", - "from delphi.eval.vis import vis_sample_prediction_probs\n", - "\n", - "model_name = \"roneneldan/TinyStories-1M\"\n", - "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", - "model = AutoModelForCausalLM.from_pretrained(model_name)\n", - "ds = load_validation_dataset(\"tinystories-v2-clean\")\n", - "ds_txt = ds[\"story\"][:100]\n", - "ds_tok = [tokenize(tokenizer, txt) for txt in ds_txt]\n", - "sample_tok = ds_tok[0]\n", - "\n", - "correct_probs, top_3_probs = get_next_and_top_k_probs(model, sample_tok, k=3)\n", - "_, top_5_probs = get_next_and_top_k_probs(model, sample_tok, k=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### collect top k predictions" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "
<|endoftext|>
Once
 upon
 a
 time
,
 there
 was
 a
 kind
 girl
 named
 Lily
.
 Lily
 loved
 to
 mix
 things
.
 One
 day
,
 she
 found
 a
 big
 box
 full
 of
 colors
.
 Lily
 was
 very
 happy
.
\\n

L
ily
 took
 out
 a
 strip
 of
 red
 and
 a
 strip
 of
 blue
.
 She
 mixed
 them
 together
 and
 made
 a
 new
 color
,
 purple
!
 Lily
 was
 so
 excited
.
 She
 wanted
 to
 mix
 more
 colors
.
\\n

Next
,
 Lily
 took
 a
 strip
 of
 yellow
 and
 a
 strip
 of
 green
.
 She
 mixed
 them
 together
 and
 made
 a
 new
 color
,
 orange
!
 Lily
 was
 very
 proud
 of
 herself
.
 She
 showed
 her
 new
 colors
 to
 her
 mom
 and
 dad
,
 and
 they
 were
 proud
 of
 her
 too
.
 They
 all
 lived
 happily
 ever
 after
.
\n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "_ = vis_sample_prediction_probs(sample_tok, correct_probs, top_3_probs, tokenizer)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "
<|endoftext|>
Once
 upon
 a
 time
,
 there
 was
 a
 kind
 girl
 named
 Lily
.
 Lily
 loved
 to
 mix
 things
.
 One
 day
,
 she
 found
 a
 big
 box
 full
 of
 colors
.
 Lily
 was
 very
 happy
.
\\n

L
ily
 took
 out
 a
 strip
 of
 red
 and
 a
 strip
 of
 blue
.
 She
 mixed
 them
 together
 and
 made
 a
 new
 color
,
 purple
!
 Lily
 was
 so
 excited
.
 She
 wanted
 to
 mix
 more
 colors
.
\\n

Next
,
 Lily
 took
 a
 strip
 of
 yellow
 and
 a
 strip
 of
 green
.
 She
 mixed
 them
 together
 and
 made
 a
 new
 color
,
 orange
!
 Lily
 was
 very
 proud
 of
 herself
.
 She
 showed
 her
 new
 colors
 to
 her
 mom
 and
 dad
,
 and
 they
 were
 proud
 of
 her
 too
.
 They
 all
 lived
 happily
 ever
 after
.
\n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "_ = vis_sample_prediction_probs(sample_tok, correct_probs, top_5_probs, tokenizer)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/pyproject.toml b/pyproject.toml index c7742381..dec159e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,25 +5,14 @@ dependencies = [ "torch==2.1.2", "datasets==2.16.1", "tqdm==4.66.1", - "ipywidgets==8.1.1", - "nbformat==5.9.2", - "pytest==7.4.4", - "black==23.12.1", "jaxtyping==0.2.25", "beartype==0.18.2", - "pre-commit==3.6.0", - "isort==5.13.2", "chardet==5.2.0", - "sentencepiece==0.1.99", - "protobuf==4.25.2", "plotly==5.18.0", "wandb==0.16.3", - "spacy==3.7.2", - "pandas==1.3.4", "dacite==1.8.1", - "panel==1.4.0", - "jupyter_bokeh==4.0.1", "transformers==4.40.0", + "platformdirs==4.2.2" ] [project.optional-dependencies] @@ -31,6 +20,19 @@ mamba_cuda = [ "mamba_ssm==1.2.0.post1", "causal-conv1d==1.2.0.post2", ] +notebooks = [ + "ipykernel==6.29.4", + "panel==1.4.0", + "jupyter_bokeh==4.0.1", + "ipywidgets==8.1.1", + "nbformat==5.9.2", +] +dev = [ + "pytest==7.4.4", + "black==23.12.1", + "isort==5.13.2", + "pre-commit==3.6.0", +] [build-system] requires = ["setuptools", "wheel"] diff --git a/requirements-nocuda.txt b/requirements-nocuda.txt deleted file mode 100644 index d3052815..00000000 --- a/requirements-nocuda.txt +++ /dev/null @@ -1,2 +0,0 @@ -# this references packages specified by pyproject.toml -. diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 10c94e31..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# this references dependencies in pyproject.toml (including optional cuda dependencies) -. -.[mamba_cuda] \ No newline at end of file diff --git a/scripts/get_next_logprobs.py b/scripts/get_next_logprobs.py index 5cf0d26e..d44b7015 100755 --- a/scripts/get_next_logprobs.py +++ b/scripts/get_next_logprobs.py @@ -9,7 +9,6 @@ from transformers import AutoModelForCausalLM from delphi import utils -from delphi.eval.utils import get_all_and_next_logprobs torch.set_grad_enabled(False) @@ -61,7 +60,7 @@ def get_logprobs_single_model( for i in trange(0, n_seq, batch_size): batch_tokens = dataset[i : i + batch_size][feature] logprobs[i : i + batch_size, 1:] = ( - get_all_and_next_logprobs(model, batch_tokens)[1].cpu().numpy() # type: ignore + utils.get_all_and_next_logprobs(model, batch_tokens)[1].cpu().numpy() # type: ignore ) return Dataset.from_dict({"logprobs": [row for row in logprobs]}) diff --git a/scripts/map_tokens.py b/scripts/map_tokens.py deleted file mode 100755 index b0393fa5..00000000 --- a/scripts/map_tokens.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -import argparse - -import pandas as pd -from datasets import Dataset - -from delphi.eval.token_map import token_map -from delphi.eval.utils import load_validation_dataset - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="") - - parser.add_argument( - "dataset_name", - type=str, - help="Dataset from huggingface to run token_map on", - ) - parser.add_argument( - "--username", - type=str, - help="Hugging Face API username", - ) - parser.add_argument( - "--token", - type=str, - help="Hugging Face API token", - ) - parser.add_argument( - "--tokenizer-size", - type=int, - default=4096, - help="Size of the tokenizer", - ) - args = parser.parse_args() - - dataset = load_validation_dataset(args.dataset_name) - - hf_dataset = Dataset.from_dict( - {"prompt_pos_idx": token_map(dataset, args.tokenizer_size)} - ) - - repo_id = f"{args.username}/v0-token-map" # location in to hf - - hf_dataset.push_to_hub( - repo_id=repo_id, - split="validation", - private=False, - token=args.token, - ) diff --git a/scripts/spacy_label_all_tokens.py b/scripts/spacy_label_all_tokens.py deleted file mode 100644 index 3ff8ccda..00000000 --- a/scripts/spacy_label_all_tokens.py +++ /dev/null @@ -1,106 +0,0 @@ -import argparse -import pickle -from pathlib import Path - -import pandas as pd -from tqdm.auto import tqdm -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast - -from delphi.eval import spacy_token_labelling - - -def tokenize( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, sample_txt: str -) -> int: - # supposedly this can be different than prepending the bos token id - return tokenizer.encode(tokenizer.bos_token + sample_txt, return_tensors="pt")[0] - - -# Decode a sentence -def decode( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, token_ids: int | list[int] -) -> str: - return tokenizer.decode(token_ids, skip_special_tokens=True) - - -def main(): - # Setup argparse - parser = argparse.ArgumentParser(description="Tokenization and labeling utility.") - parser.add_argument( - "--model-name", - type=str, - help="Name of the model to use for tokenization and labeling.", - default="delphi-suite/delphi-llama2-100k", - required=False, - ) - parser.add_argument( - "--save-dir", type=str, help="Directory to save the results.", required=True - ) - args = parser.parse_args() - - # Access command-line arguments - # Directory to save the results - save_dir = Path(args.save_dir) - save_dir.mkdir(parents=True, exist_ok=True) # create directory if it does not exist - model_name = args.model_name - - print("\n", " LABEL ALL TOKENS ".center(50, "="), "\n") - print(f"You chose the model: {model_name}\n") - print( - f"The language model will be loaded from Huggingface and its tokenizer used to do two things:\n\t1) Create a list of all tokens in the tokenizer's vocabulary.\n\t2) Label each token with its part of speech, dependency, and named entity recognition tags.\nThe respective results will be saved to files located at: '{save_dir}'\n" - ) - - # ================ (1) ================= - print("(1) Create a list of all tokens in the tokenizer's vocabulary ...") - - # Load the tokenizer from Huggingface - tokenizer = AutoTokenizer.from_pretrained(model_name) - print("Loaded the tokenizer.\nThe vocab size is:", tokenizer.vocab_size) - - ( - tokens_str, - labelled_token_ids_dict, - ) = spacy_token_labelling.label_tokens_from_tokenizer(tokenizer) - - # Save the list of all tokens to a file - filename = "all_tokens_list.txt" - filepath = save_dir / filename # TODO: use the static files of python module - with open(filepath, "w", encoding="utf-8") as f: - f.write(tokens_str) - - print(f"Saved the list of all tokens to:\n\t{filepath}\n") - - # ================ (2) ================= - print("(2) Label each token ...") - - print("\nCreating the CSV ...") - - df = spacy_token_labelling.convert_label_dict_to_df(labelled_token_ids_dict) - - print("Sanity check pandas csv ...", end="") - # Perform sanity check, that the table was created correctly - for row_index, row_values in df.iterrows(): - token_id = row_values.iloc[0] - label_pandas = list( - row_values.iloc[1:] - ) # we exclude the token_id from the colum - label_dict = list(labelled_token_ids_dict[token_id].values())[:] - assert ( - label_pandas == label_dict - ), f"The dataframes are not equal for row {token_id}\n{label_pandas}\n{label_dict}" - print(" completed.") - - # TODO: Fix the issue with disappearing spaces when exporting DataFrame to CSV. - # There's a known problem where no token is classified as "starting with a space". - - # save the dataframe to a csv - filename = "spacy_labelled_token_ids.csv" - filepath = save_dir / filename - df.to_csv(filepath, index=False) - print(f"Saved the labelled tokens as CSV to:\n\t{filepath}\n") - - print(" END ".center(50, "=")) - - -if __name__ == "__main__": - main() diff --git a/scripts/tokenize_dataset.py b/scripts/tokenize_dataset.py index a74df312..627a5951 100755 --- a/scripts/tokenize_dataset.py +++ b/scripts/tokenize_dataset.py @@ -4,28 +4,32 @@ import os from pathlib import Path -from datasets import Dataset, Features, Value, load_dataset +from datasets import Dataset from huggingface_hub import HfApi from transformers import AutoTokenizer -from delphi.dataset.tokenization import get_tokenized_chunks +from delphi import utils +from delphi.tokenization import get_tokenized_chunks if __name__ == "__main__": - parser = argparse.ArgumentParser(description="", allow_abbrev=False) + parser = argparse.ArgumentParser( + description="Tokenize a text dataset using a specified tokenizer", + allow_abbrev=False, + ) parser.add_argument( - "--in-repo-id", + "--in-dataset", "-i", type=str, required=True, - help="Text dataset from huggingface to tokenize", + help="Dataset you want to tokenize. Local path or HF repo id", ) parser.add_argument( "--feature", "-f", type=str, required=True, - help="Name of the column containing text documents in the input dataset", + help="Name of the feature (column) containing text documents in the input dataset", ) parser.add_argument( "--split", @@ -34,18 +38,6 @@ required=True, help="Split of the dataset to be tokenized, supports slicing like 'train[:10%%]'", ) - parser.add_argument( - "--out-dir", - type=str, - required=False, - help="Local directory to save the resulting dataset", - ) - parser.add_argument( - "--out-repo-id", - type=str, - required=False, - help="HF repo id to upload the resulting dataset", - ) parser.add_argument( "--tokenizer", "-t", @@ -58,32 +50,39 @@ "-l", type=int, required=True, - help="Context size of the tokenized dataset as input of the model", + help="Length of the tokenized sequences", ) parser.add_argument( "--batch-size", "-b", type=int, default=50, - help="Size of input into batched tokenization", + help="How many text documents to tokenize at once (default: 50)", ) parser.add_argument( "--chunk-size", "-c", type=int, default=200_000, - help="Size of the parquet chunks uploaded to HuggingFace", + help="Maximum number of tokenized sequences in a single parquet file (default: 200_000)", + ) + parser.add_argument( + "--out-dir", + type=str, + required=False, + help="Local directory to save the resulting dataset", + ) + parser.add_argument( + "--out-repo", + type=str, + required=False, + help="HF repo id to upload the resulting dataset", ) args = parser.parse_args() - assert ( - args.out_repo_id or args.out_dir - ), "You need to provide --out-repo-id or --out-dir" + assert args.out_repo or args.out_dir, "You need to provide --out-repo or --out-dir" - print(f"Loading dataset '{args.in_repo_id}'...") - in_dataset_split = load_dataset( - args.in_repo_id, - split=args.split, - features=Features({args.feature: Value("string")}), + in_dataset_split = utils.load_dataset_split_string_feature( + args.in_dataset, args.split, args.feature ) assert isinstance(in_dataset_split, Dataset) print(f"Loading tokenizer from '{args.tokenizer}'...") @@ -92,9 +91,9 @@ assert tokenizer.eos_token_id is not None, "Tokenizer must have a eos_token_id" api = None - if args.out_repo_id: + if args.out_repo: api = HfApi() - api.create_repo(repo_id=args.out_repo_id, repo_type="dataset", exist_ok=True) + api.create_repo(repo_id=args.out_repo, repo_type="dataset", exist_ok=True) if args.out_dir: os.makedirs(args.out_dir, exist_ok=True) @@ -107,7 +106,7 @@ ) print(f"Tokenizing split='{args.split}'...") - split_name = args.split.split("[")[0] + split_name = utils.hf_split_to_split_name(args.split) for chunk_idx, ds_chunk in enumerate(ds_chunks_it): chunk_name = f"{split_name}-{chunk_idx:05}.parquet" if args.out_dir: @@ -117,11 +116,11 @@ ds_parquet_chunk = io.BytesIO() ds_chunk.to_parquet(ds_parquet_chunk) if api: - print(f"Uploading '{chunk_name}' to '{args.out_repo_id}'...") + print(f"Uploading '{chunk_name}' to '{args.out_repo}'...") api.upload_file( path_or_fileobj=ds_parquet_chunk, path_in_repo=f"data/{chunk_name}", - repo_id=args.out_repo_id, + repo_id=args.out_repo, repo_type="dataset", ) print(f"Done saving/uploading '{chunk_name}'") diff --git a/scripts/run_training.py b/scripts/train_model.py similarity index 73% rename from scripts/run_training.py rename to scripts/train_model.py index 98e126ed..ffe0af0c 100755 --- a/scripts/run_training.py +++ b/scripts/train_model.py @@ -3,14 +3,10 @@ import logging import sys from pathlib import Path -from typing import Any -from delphi.train.config import ( - build_config_from_files_and_overrides, - dot_notation_to_dict, -) +from delphi.train.config import build_config_from_files_and_overrides from delphi.train.training import run_training -from delphi.train.utils import save_results +from delphi.train.utils import overrides_to_dict, save_results def add_logging_args(parser: argparse.ArgumentParser): @@ -51,23 +47,21 @@ def set_logging(args: argparse.Namespace): def setup_parser() -> argparse.ArgumentParser: # Setup argparse - parser = argparse.ArgumentParser(description="Train a delphi model") + parser = argparse.ArgumentParser( + description="Train a delphi model", allow_abbrev=False + ) parser.add_argument( - "--config_files", - "--config_file", - "-c", + "config_files", help=( - "Path to json file(s) containing config values. Specific values can be overridden with --overrides. " - "e.g. `--config_files primary_config.json secondary_config.json" + "Path to json file(s) containing config values, e.g. 'primary_config.json secondary_config.json'." ), type=str, - required=False, nargs="*", ) parser.add_argument( "--overrides", help=( - "Override config values with comma-separated declarations. " + "Override config values with space-separated declarations. " "e.g. `--overrides model_config.hidden_size=42 run_name=foo`" ), type=str, @@ -79,12 +73,6 @@ def setup_parser() -> argparse.ArgumentParser: return parser -def overrides_to_dict(overrides: list[str]) -> dict[str, Any]: - # ["a.b.c=4", "foo=false"] to {"a": {"b": {"c": 4}}, "foo": False} - config_vars = {k: v for k, v in [x.split("=") for x in overrides if "=" in x]} - return dot_notation_to_dict(config_vars) - - def main(): parser = setup_parser() args = parser.parse_args() diff --git a/scripts/train_tokenizer.py b/scripts/train_tokenizer.py index 83e071ae..a82be7c3 100755 --- a/scripts/train_tokenizer.py +++ b/scripts/train_tokenizer.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import argparse -from datasets import Dataset, Features, Value, load_dataset +from datasets import Dataset, Features, Value from tokenizers import ByteLevelBPETokenizer # type: ignore from transformers import PreTrainedTokenizerFast +from delphi import utils + def train_byte_level_bpe( dataset: Dataset, feature: str, vocab_size: int @@ -27,14 +29,17 @@ def train_byte_level_bpe( if __name__ == "__main__": - parser = argparse.ArgumentParser(description="", allow_abbrev=False) + parser = argparse.ArgumentParser( + description="Train a custom, reversible, BPE tokenizer (GPT2-like). You need to provide --out-repo or --out-dir.", + allow_abbrev=False, + ) parser.add_argument( - "--in-repo-id", + "--in-dataset", "-i", type=str, required=True, - help="Input dataset", + help="Dataset you want to train the tokenizer on. Local path or HF repo id", ) parser.add_argument( "--feature", @@ -64,21 +69,16 @@ def train_byte_level_bpe( help="Local directory to save the resulting tokenizer", ) parser.add_argument( - "--out-repo-id", + "--out-repo", type=str, required=False, help="HF repo id to upload the resulting tokenizer", ) args = parser.parse_args() - assert ( - args.out_repo_id or args.out_dir - ), "You need to provide out_repo_id or out_dir" + assert args.out_repo or args.out_dir, "You need to provide --out-repo or --out-dir" - print(f"Loading dataset '{args.in_repo_id}'...") - in_dataset_split = load_dataset( - args.in_repo_id, - split=args.split, - features=Features({args.feature: Value("string")}), + in_dataset_split = utils.load_dataset_split_string_feature( + args.in_dataset, args.split, args.feature ) assert isinstance(in_dataset_split, Dataset) tokenizer = train_byte_level_bpe( @@ -90,9 +90,9 @@ def train_byte_level_bpe( print(f"Saving tokenizer to '{args.out_dir}' directory...") tokenizer.save_pretrained(args.out_dir) print("Done.") - if args.out_repo_id: - print(f"Pushing tokenizer to HF repo '{args.out_repo_id}'...") + if args.out_repo: + print(f"Pushing tokenizer to HF repo '{args.out_repo}'...") tokenizer.push_to_hub( - repo_id=args.out_repo_id, + repo_id=args.out_repo, ) print("Done.") diff --git a/scripts/training_run.sh b/scripts/training_run.sh deleted file mode 100644 index 7d1b2fe8..00000000 --- a/scripts/training_run.sh +++ /dev/null @@ -1,6 +0,0 @@ -counter=1 -for config in 4-76.json 6-112 6-204 -do - CUDA_VISIBLE_DEVICES=$counter CUBLAS_WORKSPACE_CONFIG=:4096:8 python scripts/run_training.py --config scripts/$config & > $config.log - counter=$((counter+1)) -done diff --git a/scripts/validate_configs.py b/scripts/validate_configs.py index 46d3f5d1..86b6b4f6 100755 --- a/scripts/validate_configs.py +++ b/scripts/validate_configs.py @@ -3,6 +3,7 @@ import pathlib from delphi.train.config import build_config_from_files_and_overrides +from delphi.train.utils import init_model, overrides_to_dict def get_config_path_with_base(config_path: pathlib.Path) -> list[pathlib.Path]: @@ -33,15 +34,32 @@ def main(): type=str, help="path to a training config json or directory of training config jsons", ) + parser.add_argument( + "--overrides", + help=( + "Override config values with space-separated declarations. " + "e.g. `--overrides model_config.hidden_size=42 run_name=foo`" + ), + type=str, + required=False, + nargs="*", + default=[], + ) + parser.add_argument("--init", help="initialize the model", action="store_true") args = parser.parse_args() config_paths = get_config_paths(args.config_path) print( f"validating configs: {' | '.join(str(config_path[-1]) for config_path in config_paths)}" ) + overrides = overrides_to_dict(args.overrides) errors = [] + sizes = [] for config_path in config_paths: try: - build_config_from_files_and_overrides(config_path, {}) + config = build_config_from_files_and_overrides(config_path, overrides) + if args.init: + model = init_model(config.model_config, seed=config.torch_seed) + sizes.append((config_path, model.num_parameters())) except Exception as e: errors.append((config_path, e)) continue @@ -51,6 +69,10 @@ def main(): print(f" {config_path[-1]}: {e}") else: print("all configs loaded successfully") + if sizes: + print("model sizes:") + for config_path, size in sizes: + print(f" {config_path[-1]}: {size}") if __name__ == "__main__": diff --git a/setup.py b/setup.py index 4a92f04d..5dd7948a 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,8 @@ setup( name="delphi", - packages=find_packages(where="src"), - package_dir={"": "src"}, + packages=find_packages(where="."), + package_dir={"": "."}, package_data={ "delphi": ["test_configs/**/*"], }, diff --git a/src/delphi/__init__.py b/src/delphi/__init__.py deleted file mode 100644 index a0ea3bb4..00000000 --- a/src/delphi/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from beartype.claw import beartype_this_package # <-- hype comes - -beartype_this_package() # <-- hype goes - -__version__ = "0.1.1" diff --git a/src/delphi/dummy.py b/src/delphi/dummy.py deleted file mode 100644 index 028ff372..00000000 --- a/src/delphi/dummy.py +++ /dev/null @@ -1,12 +0,0 @@ -import torch -from jaxtyping import Float, Int - -Type1 = Float[torch.Tensor, "dim"] -Type2 = Int[torch.Tensor, "batch dim"] - - -def dummy(arg: Type1 | Type2) -> Type1: - if isinstance(arg, Type1): - return arg + 1 - elif isinstance(arg, Type2): - return arg[0] - 0.1 diff --git a/src/delphi/eval/calc_model_group_stats.py b/src/delphi/eval/calc_model_group_stats.py deleted file mode 100644 index d9c5d4c1..00000000 --- a/src/delphi/eval/calc_model_group_stats.py +++ /dev/null @@ -1,54 +0,0 @@ -import numpy as np - - -def calc_model_group_stats( - tokenized_corpus_dataset: list, - logprobs_by_dataset: dict[str, list[list[float]]], - token_labels_by_token: dict[int, dict[str, bool]], - token_labels: list[str], -) -> dict[tuple[str, str], dict[str, float]]: - """ - For each (model, token group) pair, calculate useful stats (for visualization) - - args: - - tokenized_corpus_dataset: the tokenized corpus dataset, e.g. load_dataset(constants.tokenized_corpus_dataset))["validation"] - - logprob_datasets: a dict of lists of logprobs, e.g. {"llama2": load_dataset("transcendingvictor/llama2-validation-logprobs")["validation"]["logprobs"]} - - token_groups: a dict of token groups, e.g. {0: {"Is Noun": True, "Is Verb": False, ...}, 1: {...}, ...} - - models: a list of model names, e.g. constants.LLAMA2_MODELS - - token_labels: a list of token group descriptions, e.g. ["Is Noun", "Is Verb", ...] - - returns: a dict of (model, token group) pairs to a dict of stats, - e.g. {("llama2", "Is Noun"): {"mean": -0.5, "median": -0.4, "min": -0.1, "max": -0.9, "25th": -0.3, "75th": -0.7}, ...} - - Technically `models` and `token_labels` are redundant, as they are also keys in `logprob_datasets` and `token_groups`, - but it's better to be explicit - - stats calculated: mean, median, min, max, 25th percentile, 75th percentile - """ - model_group_stats = {} - for model in logprobs_by_dataset: - group_logprobs = {} - print(f"Processing model {model}") - dataset = logprobs_by_dataset[model] - for ix_doc_lp, document_lps in enumerate(dataset): - tokens = tokenized_corpus_dataset[ix_doc_lp]["tokens"] - for ix_token, token in enumerate(tokens): - if ix_token == 0: # skip the first token, which isn't predicted - continue - logprob = document_lps[ix_token] - for token_group_desc in token_labels: - if token_labels_by_token[token][token_group_desc]: - if token_group_desc not in group_logprobs: - group_logprobs[token_group_desc] = [] - group_logprobs[token_group_desc].append(logprob) - for token_group_desc in token_labels: - if token_group_desc in group_logprobs: - model_group_stats[(model, token_group_desc)] = { - "mean": np.mean(group_logprobs[token_group_desc]), - "median": np.median(group_logprobs[token_group_desc]), - "min": np.min(group_logprobs[token_group_desc]), - "max": np.max(group_logprobs[token_group_desc]), - "25th": np.percentile(group_logprobs[token_group_desc], 25), - "75th": np.percentile(group_logprobs[token_group_desc], 75), - } - return model_group_stats diff --git a/src/delphi/eval/compare_models.py b/src/delphi/eval/compare_models.py deleted file mode 100644 index e03b300c..00000000 --- a/src/delphi/eval/compare_models.py +++ /dev/null @@ -1,91 +0,0 @@ -from dataclasses import dataclass - -import torch -from jaxtyping import Int -from transformers import PreTrainedModel - -from delphi.eval.utils import get_all_and_next_logprobs_single - - -def identify_model(model: PreTrainedModel) -> str: - return model.config.name_or_path - - -@dataclass -class TokenPrediction: - token: int - base_model_prob: float - lift_model_prob: float - - -@dataclass -class NextTokenStats: - base_model: str - lift_model: str - next_prediction: TokenPrediction - topk: list[TokenPrediction] - - -def compare_models( - model_a: PreTrainedModel, - model_b: PreTrainedModel, - sample_tok: Int[torch.Tensor, "seq"], - top_k: int = 3, -) -> list[NextTokenStats | None]: - """ - Compare the probabilities of the next token for two models and get the top k token predictions according to model B. - Args: - - model_a: The first model (assumed to be the base model) - - model_b: The second model (assumed to be the improved model) - - sample_tok: The tokenized prompt - - top_k: The number of top token predictions to retrieve (default is 5) - Returns: - A list of NextTokenStats objects, one for each token in the prompt. - Tensors are aligned to the token they are predicting (by prepending a -1 to the start of the tensor) - """ - assert ( - model_a.device == model_b.device - ), "Both models must be on the same device for comparison." - - device = model_a.device - sample_tok = sample_tok.to(device) - - logprobs_a, next_probs_a = get_all_and_next_logprobs_single(model_a, sample_tok) - logprobs_b, next_probs_b = get_all_and_next_logprobs_single(model_b, sample_tok) - - probs_a = torch.exp(logprobs_a) - probs_b = torch.exp(logprobs_b) - - top_k_b = torch.topk(probs_b, top_k, dim=-1) - top_k_a_probs = torch.gather(probs_a, 1, top_k_b.indices) - - top_k_b_tokens = top_k_b.indices - top_k_b_probs = top_k_b.values - - comparisons = [] - # ignore first token when evaluating predictions - comparisons.append(None) - - for next_p_a, next_p_b, top_toks_b, top_probs_a, top_probs_b in zip( - next_probs_a, next_probs_b, top_k_b_tokens, top_k_a_probs, top_k_b_probs - ): - nts = NextTokenStats( - base_model=identify_model(model_a), - lift_model=identify_model(model_b), - next_prediction=TokenPrediction( - token=int(next_p_a.item()), - base_model_prob=next_p_a.item(), - lift_model_prob=next_p_b.item(), - ), - topk=[ - TokenPrediction( - token=int(top_toks_b[i].item()), - base_model_prob=top_probs_a[i].item(), - lift_model_prob=top_probs_b[i].item(), - ) - for i in range(top_k) - ], - ) - comparisons.append(nts) - - return comparisons diff --git a/src/delphi/eval/constants.py b/src/delphi/eval/constants.py deleted file mode 100644 index 5cd3daf1..00000000 --- a/src/delphi/eval/constants.py +++ /dev/null @@ -1,26 +0,0 @@ -corpus_dataset = "delphi-suite/tinystories-v2-clean" -tokenized_corpus_dataset = "delphi-suite/tinystories-v2-clean-tokenized-v0" - -LLAMA2_MODELS = [ - "delphi-llama2-100k", - "delphi-llama2-200k", - "delphi-llama2-400k", - "delphi-llama2-800k", - "delphi-llama2-1.6m", - "delphi-llama2-3.2m", - "delphi-llama2-6.4m", - "delphi-llama2-12.8m", - "delphi-llama2-25.6m", -] - -LLAMA2_NEXT_LOGPROBS_DATASETS_MAP = { - "llama2-100k": "delphi-suite/v0-next-logprobs-llama2-100k", - "llama2-200k": "delphi-suite/v0-next-logprobs-llama2-200k", - "llama2-400k": "delphi-suite/v0-next-logprobs-llama2-400k", - "llama2-800k": "delphi-suite/v0-next-logprobs-llama2-800k", - "llama2-1.6m": "delphi-suite/v0-next-logprobs-llama2-1.6m", - "llama2-3.2m": "delphi-suite/v0-next-logprobs-llama2-3.2m", - "llama2-6.4m": "delphi-suite/v0-next-logprobs-llama2-6.4m", - "llama2-12.8m": "delphi-suite/v0-next-logprobs-llama2-12.8m", - "llama2-25.6m": "delphi-suite/v0-next-logprobs-llama2-25.6m", -} diff --git a/src/delphi/eval/spacy_token_labelling.py b/src/delphi/eval/spacy_token_labelling.py deleted file mode 100644 index a9a82193..00000000 --- a/src/delphi/eval/spacy_token_labelling.py +++ /dev/null @@ -1,315 +0,0 @@ -from collections.abc import Callable -from pathlib import Path -from typing import Optional - -import pandas as pd -import spacy -from spacy.tokens import Doc, Token -from spacy.util import is_package -from tqdm.auto import tqdm -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast - -# make sure the english language model capabilities are installed by the equivalent of: -# python -m spacy download en_core_web_sm -# Should be run once, initially. Download only starts if not already installed. -SPACY_MODEL = "en_core_web_sm" # small: "en_core_web_sm", large: "en_core_web_trf" -NLP = None # global var to hold the language model -if not is_package(SPACY_MODEL): - spacy.cli.download(SPACY_MODEL, False, False) - - -TOKEN_LABELS: dict[str, Callable] = { - # --- custom categories --- - "Starts with space": (lambda token: token.text.startswith(" ")), # bool - "Capitalized": (lambda token: token.text[0].isupper()), # bool - # --- POS (part-of-speech) categories --- - # They include the Universal POS tags (https://universaldependencies.org/u/pos/) - # -> "POS Tag": (lambda token: token.pos_), # 'NOUN', 'VB', .. - "Is Adjective": (lambda token: token.pos_ == "ADJ"), - "Is Adposition": (lambda token: token.pos_ == "ADP"), - "Is Adverb": (lambda token: token.pos_ == "ADV"), - "Is Auxiliary": (lambda token: token.pos_ == "AUX"), - "Is Coordinating conjuction": (lambda token: token.pos_ == "CCONJ"), - "Is Determiner": (lambda token: token.pos_ == "DET"), - "Is Interjunction": (lambda token: token.pos_ == "INTJ"), - "Is Noun": (lambda token: token.pos_ == "NOUN"), - "Is Numeral": (lambda token: token.pos_ == "NUM"), - "Is Particle": (lambda token: token.pos_ == "PART"), - "Is Pronoun": (lambda token: token.pos_ == "PRON"), - "Is Proper Noun": (lambda token: token.pos_ == "PROPN"), - "Is Punctuation": (lambda token: token.pos_ == "PUNCT"), - "Is Subordinating conjuction": (lambda token: token.pos_ == "SCONJ"), - "Is Symbol": (lambda token: token.pos_ == "SYM"), - "Is Verb": (lambda token: token.pos_ == "VERB"), - "Is Other": (lambda token: token.pos_ == "X"), - # --- dependency categories --- - # -> "Dependency": (lambda token: token.dep_), # 'nsubj', 'ROOT', 'dobj', .. - # "Is Subject": (lambda token: token.dep_ == "nsubj"), - # "Is Object": (lambda token: token.dep_ == "dobj"), - # "Is Root": ( - # lambda token: token.dep_ == "ROOT" - # ), # root of the sentence (often a verb) - # "Is auxiliary": (lambda token: token.dep_ == "aux"), - # --- Named entity recognition (NER) categories --- - # "Named Entity Type": (lambda token: token.ent_type_), # '', 'PERSON', 'ORG', 'GPE', .. - "Is Named Entity": (lambda token: token.ent_type_ != ""), -} - - -def explain_token_labels(token: Optional[Token] = None) -> None: - """ - Prints the explanation of a specific token's labels or of ALL - possible labels (POS, dependency, NER, ...), if no token is provided. - - Parameters - ---------- - token : Optional[Token], optional - The token, whose labels should be explained. If None, all labels - possible labels are explained, by default None. - """ - if token is not None: - # get token labels - labels = label_single_token(token) - print(" Explanation of token labels ".center(45, "-")) - print("Token text:".ljust(20), token.text) - print("Token dependency:".ljust(20), spacy.glossary.explain(token.dep_)) - print("Token POS:".ljust(20), spacy.glossary.explain(token.pos_)) - print(" Token labels ".center(45, "-")) - for i, (label_name, value) in enumerate(labels.items()): - print(f" {i:2} ", label_name.ljust(20), value) - - else: - glossary = spacy.glossary.GLOSSARY - print( - f"Explanation of all {len(glossary.keys())} token labels (POS, dependency, NER, ...):" - ) - for label, key in glossary.items(): - print(" ", label.ljust(10), key) - - -def label_single_token(token: Token | None) -> dict[str, bool]: - """ - Labels a single token. A token, that has been analyzed by the spaCy - library. - - Parameters - ---------- - token : Token | None - The token to be labelled. - - Returns - ------- - dict[str, bool] - Returns a dictionary with the token's labels as keys and their - corresponding boolean values. - """ - labels = dict() # The dict holding labels of a single token - # if token is None, then it is a '' empty strong token or similar - if token is None: - for label_name, category_check in TOKEN_LABELS.items(): - labels[label_name] = False - labels["Is Other"] = True - return labels - # all other cases / normal tokens - for label_name, category_check in TOKEN_LABELS.items(): - labels[label_name] = category_check(token) - return labels - - -def label_sentence(tokens: Doc | list[Token]) -> list[dict[str, bool]]: - """ - Labels spaCy Tokens in a sentence. Takes the context of the token into account - for dependency labels (e.g. subject, object, ...), IF dependency labels are turned on. - - Parameters - ---------- - tokens : list[Token] - A list of tokens. - - Returns - ------- - list[dict[str, bool]] - Returns a list of the tokens' labels. - """ - labelled_tokens = list() # list holding labels for all tokens of sentence - # if the list is empty it is because token is '' empty string or similar - if len(tokens) == 0: - labels = label_single_token(None) - labelled_tokens.append(labels) - return labelled_tokens - # in all other cases - for token in tokens: - labels = label_single_token(token) - labelled_tokens.append(labels) - return labelled_tokens - - -def label_batch_sentences( - sentences: list[str] | list[list[str]], - tokenized: bool = True, - verbose: bool = False, -) -> list[list[dict[str, bool]]]: - """ - Labels tokens in a sentence batchwise. Takes the context of the token into - account for dependency labels (e.g. subject, object, ...). - - Parameters - ---------- - sentences : list - A batch/list of sentences, each being a list of tokens. - tokenized : bool, optional - Whether the sentences are already tokenized, by default True. If the sentences - are full strings and not lists of tokens, then set to False. If true then `sentences` must be list[list[str]]. - verbose : bool, optional - Whether to print the tokens and their labels to the console, by default False. - - Returns - ------- - list[list[dict[str, bool]] - Returns a list of sentences. Each sentence contains a list of its - corresponding token length where each entry provides the labels/categories - for the token. Sentence -> Token -> Labels - """ - global NLP, SPACY_MODEL - - if NLP is None: - # Load english language model - NLP = spacy.load(SPACY_MODEL) - # labelled tokens, list holding sentences holding tokens holding corresponding token labels - labelled_sentences: list[list[dict[str, bool]]] = list() - - # go through each sentence in the batch - for sentence in sentences: - if tokenized: - # sentence is a list of tokens - doc = Doc(NLP.vocab, words=sentence) # type: ignore - # Apply the spaCy pipeline, except for the tokenizer - for name, proc in NLP.pipeline: - if name != "tokenizer": - doc = proc(doc) - else: - # sentence is a single string - doc = NLP(sentence) # type: ignore - - labelled_tokens = list() # list holding labels for all tokens of sentence - labelled_tokens = label_sentence(doc) - - # print the token and its labels to console - if verbose is True: - # go through each token in the sentence - for token, labelled_token in zip(doc, labelled_tokens): - print(f"Token: {token}") - print(" | ".join(list(TOKEN_LABELS.keys()))) - printable = [ - str(l).ljust(len(name)) for name, l in labelled_token.items() - ] - printable = " | ".join(printable) - print(printable) - print("---") - # add current sentence's tokens' labels to the list - labelled_sentences.append(labelled_tokens) - - if verbose is True: - print("\n") - - return labelled_sentences - - -def label_tokens_from_tokenizer( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, -) -> tuple[str, dict[int, dict[str, bool]]]: - """ - Labels all tokens in a tokenizer's vocabulary with the corresponding token categories (POS, named entity, etc). Returns two things: 1) `tokens_str`, a string where each token comprises 'token_id,token_str\n' and 2) `labelled_token_ids_dict` a dict that contains for each token_id (key) the corresponding token labels, which is in turn a dict, whith the label categories as keys and their boolean values as the dict's values. - - Parameters - ---------- - tokenizer : The tokenizer with its tokens to be labelled. - - Returns - ------- - tokens_str, labelled_token_ids_dict - - """ - - def decode( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, - token_ids: int | list[int], - ) -> str: - return tokenizer.decode(token_ids, skip_special_tokens=True) - - vocab_size = tokenizer.vocab_size - - # 1) Create a list of all tokens in the tokenizer's vocabulary - tokens_str = "" # will hold all tokens and their ids - for i in range(vocab_size): - tokens_str += f"{i},{decode(tokenizer, i)}\n" - - # 2) let's label each token - labelled_token_ids_dict = {} # token_id: labels - max_token_id = vocab_size # stop at which token id, vocab size - # we iterate over all token_ids individually - for token_id in tqdm(range(0, max_token_id), desc="Labelling tokens"): - # decode the token_ids to get a list of tokens, a 'sentence' - token = decode(tokenizer, token_id) # list of tokens == sentence - # put the sentence into a list, to make it a batch of sentences - sentences = [token] - # label the batch of sentences - labels = label_batch_sentences(sentences, tokenized=True, verbose=False) - # create a dict with the token_ids and their labels - # update the labelled_token_ids_dict with the new dict - label = labels[0][0] # first sentence of batch, label of first token - labelled_token_ids_dict[token_id] = label - - return tokens_str, labelled_token_ids_dict - - -def import_token_labels(path: str | Path): - """ - Imports token labels from a *.csv file. - - Parameters - ---------- - path : str | Path - The path to the file. - - Returns - ------- - dict[int, dict[str, bool]] - Returns the labelled tokens dict. Each token_id has its own dict having the labels. - """ - if isinstance(path, str): - path = Path(path) - # make sure the file_type is compatible - file_type = path.suffix - assert ( - file_type == ".csv" - ), f"Invalid file type. Allowed: csv, pkl. Got: {file_type}" - # make sure file exists - if not path.exists(): - raise FileNotFoundError(f"There is no file under {path}") - - df = pd.read_csv(str(path)) - categories = list(df.columns[1:]) # excluding first column: token_id - loaded_label_dict: dict[int, dict[str, bool]] = {} - # go through each row and construct the dict - for _, row in df.iterrows(): - token_id = int(row["token_id"]) - labels = {cat: bool(row[cat] == 1) for cat in categories} - loaded_label_dict[token_id] = labels - - return loaded_label_dict - - -def convert_label_dict_to_df( - labelled_token_ids_dict: dict[int, dict[str, bool]] -) -> pd.DataFrame: - """ - Takes a `labelled_token_ids_dict` and converts it into a Pandas Dataframe. - """ - df = pd.DataFrame(labelled_token_ids_dict.items(), columns=["token_id", "label"]) - # split the label column into multiple columns - df = df.join(pd.DataFrame(df.pop("label").tolist())) - # Change datatype of columns to float - df = df.astype(int) - - return df diff --git a/src/delphi/eval/token_map.py b/src/delphi/eval/token_map.py deleted file mode 100644 index 4ac7b0df..00000000 --- a/src/delphi/eval/token_map.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -from typing import cast - -from datasets import Dataset - - -def token_map( - tokenized_dataset: Dataset, - tokenizer_size: int, -) -> list[list[tuple[int, int]]]: - """Return a mapping of tokens to their (prompt_idx, token_idx) locations in the tokenized_dataset.""" - - mapping = [[] for _ in range(tokenizer_size)] - for prompt_idx, prompt in enumerate(tokenized_dataset): - prompt = cast(dict, prompt) - for position_idx, token in enumerate(prompt["tokens"]): - mapping[token].append((prompt_idx, position_idx)) - return mapping diff --git a/src/delphi/eval/token_positions.py b/src/delphi/eval/token_positions.py deleted file mode 100644 index 5239a53f..00000000 --- a/src/delphi/eval/token_positions.py +++ /dev/null @@ -1,53 +0,0 @@ -from numbers import Number -from typing import Optional, cast - -import torch -from datasets import Dataset -from jaxtyping import Int - -from delphi.eval.utils import dict_filter_quantile - - -def get_all_tok_metrics_in_label( - token_ids: Int[torch.Tensor, "prompt pos"], - token_labels: dict[int, dict[str, bool]], - metrics: torch.Tensor, - label: str, - q_start: Optional[float] = None, - q_end: Optional[float] = None, -) -> dict[tuple[int, int], float]: - """ - From the token_map, get all the positions of the tokens that have a certain label. - We don't use the token_map because for sampling purposes, iterating through token_ids is more efficient. - Optionally, filter the tokens based on the quantile range of the metrics. - - Args: - - token_ids (Dataset): token_ids dataset e.g. token_ids[0] = {"tokens": [[1, 2, ...], [2, 5, ...], ...]} - - token_labels (dict[int, dict[str, bool]]): dictionary of token labels e.g. { 0: {"Is Noun": True, "Is Verb": False}, ...} - - metrics (torch.Tensor): tensor of metrics to search through e.g. torch.tensor([[0.1, 0.2, ...], [0.3, 0.4, ...], ...]) - - label (str): the label to search for - - q_start (float): the start of the quantile range to filter the metrics e.g. 0.1 - - q_end (float): the end of the quantile range to filter the metrics e.g. 0.9 - - Returns: - - tok_positions (dict[tuple[int, int], Number]): dictionary of token positions and their corresponding metrics - """ - - # check if metrics have the same dimensions as token_ids - if metrics.shape != token_ids.shape: - raise ValueError( - f"Expected metrics to have the same shape as token_ids, but got {metrics.shape} and {token_ids.shape} instead." - ) - - tok_positions = {} - for prompt_pos, prompt in enumerate(token_ids.numpy()): - for tok_pos, tok in enumerate(prompt): - if token_labels[tok][label]: - tok_positions[(prompt_pos, tok_pos)] = metrics[ - prompt_pos, tok_pos - ].item() - - if q_start is not None and q_end is not None: - tok_positions = dict_filter_quantile(tok_positions, q_start, q_end) - - return tok_positions diff --git a/src/delphi/eval/utils.py b/src/delphi/eval/utils.py deleted file mode 100644 index faf33757..00000000 --- a/src/delphi/eval/utils.py +++ /dev/null @@ -1,133 +0,0 @@ -import logging -from collections.abc import Callable -from typing import Any, cast - -import numpy as np -import torch -from datasets import Dataset, load_dataset -from jaxtyping import Float, Int -from transformers import PreTrainedModel, PreTrainedTokenizerBase - -from delphi.eval import constants - - -def get_all_logprobs( - model: Callable, input_ids: Int[torch.Tensor, "batch seq"] -) -> Float[torch.Tensor, "batch seq vocab"]: - # batch, seq, vocab - logits = model(input_ids).logits - return torch.log_softmax(logits, dim=-1) - - -# convenience wrapper for calling on a single sample -def get_single_logprobs( - model: Callable, input_ids: Int[torch.Tensor, "seq"] -) -> Float[torch.Tensor, "seq vocab"]: - return get_all_logprobs(model, input_ids.unsqueeze(0))[0] - - -def gather_logprobs( - logprobs: Float[torch.Tensor, "batch seq vocab"], - tokens: Int[torch.Tensor, "batch seq"], -) -> Float[torch.Tensor, "batch seq"]: - return torch.gather(logprobs, -1, tokens.unsqueeze(-1)).squeeze(-1) - - -def get_all_and_next_logprobs( - model: Callable, - input_ids: Int[torch.Tensor, "batch seq"], -) -> tuple[ - Float[torch.Tensor, "batch shorter_seq vocab"], - Float[torch.Tensor, "batch shorter_seq"], -]: - logprobs = get_all_logprobs(model, input_ids[:, :-1]) - next_tokens = input_ids[:, 1:] - return logprobs, gather_logprobs(logprobs, next_tokens) - - -def get_all_and_next_logprobs_single( - model: Callable, - input_ids: Int[torch.Tensor, "seq"], -) -> tuple[ - Float[torch.Tensor, "shorter_seq vocab"], - Float[torch.Tensor, "shorter_seq"], -]: - all_logprobs, next_logprobs = get_all_and_next_logprobs( - model, input_ids.unsqueeze(0) - ) - return all_logprobs[0], next_logprobs[0] - - -def get_next_and_top_k_probs( - model: PreTrainedModel, input_ids: Int[torch.Tensor, "seq"], k: int = 3 -) -> tuple[Float[torch.Tensor, "shorter_seq"], torch.return_types.topk,]: - all_logprobs, next_logprobs = get_all_and_next_logprobs_single(model, input_ids) - all_probs = torch.exp(all_logprobs) - next_probs = torch.exp(next_logprobs) - top_k = torch.topk(all_probs, k, dim=-1) - return next_probs, top_k - - -def load_delphi_dataset(dataset_name: str, split: str, slice: str = "") -> Dataset: - # check that split is either "train" or "validation" - if split not in ["train", "validation"]: - raise ValueError(f"Split must be either 'train' or 'validation', not {split}") - if "/" not in dataset_name: - dataset_name = f"delphi-suite/{dataset_name}" - data_files_str = f"data/{split}-*.parquet" - dataset = load_dataset( - dataset_name, - data_files=data_files_str, - verification_mode="no_checks", - # Currently, load_dataset returns a dataset dict *unless* a split is specified, - # EVEN IF NO SPLIT WITHIN THE DATA FILES SPECIFIED. If there's no split arg, - # huggingface just just says everything is in the "train" split and returns {"train": dataset}. - # In our case the data_files glob already specifies just the validation files, so we - # shouldn't need to specify a split. But we do need to specify a split to get a dataset object, - # or we'd get a Dataset dict. See https://github.com/huggingface/datasets/issues/5189 - split=f"train{slice}", - ) - dataset = cast(Dataset, dataset) - logging.info(f" Loaded {data_files_str} ({len(dataset)} entries)") - return dataset - - -def load_validation_dataset(dataset_name: str, slice: str = "") -> Dataset: - return load_delphi_dataset(dataset_name, "validation", slice) - - -def load_train_dataset(dataset_name: str, slice: str = "") -> Dataset: - return load_delphi_dataset(dataset_name, "train", slice) - - -def tokenize( - tokenizer: PreTrainedTokenizerBase, sample_txt: str -) -> Int[torch.Tensor, "seq"]: - # supposedly this can be different than prepending the bos token id - return cast( - Int[torch.Tensor, "seq"], - tokenizer.encode(tokenizer.bos_token + sample_txt, return_tensors="pt")[0], - ) - - -def load_logprob_dataset(model: str): - return load_dataset(f"transcendingvictor/{model}-validation-logprobs") - - -def load_logprob_datasets(split: str = "validation") -> dict[str, list[list[float]]]: - return { - model: cast(dict, load_logprob_dataset(model)[split])["logprobs"] # type: ignore - for model in constants.LLAMA2_MODELS - } - - -def dict_filter_quantile( - d: dict[Any, float], q_start: float, q_end: float -) -> dict[Any, float]: - if not (0 <= q_start < q_end <= 1): - raise ValueError("Invalid quantile range") - q_start_val = np.nanquantile(list(d.values()), q_start) - q_end_val = np.nanquantile(list(d.values()), q_end) - return { - k: v for k, v in d.items() if q_start_val <= v <= q_end_val and not np.isnan(v) - } diff --git a/src/delphi/eval/vis.py b/src/delphi/eval/vis.py deleted file mode 100644 index 1a69eae2..00000000 --- a/src/delphi/eval/vis.py +++ /dev/null @@ -1,257 +0,0 @@ -import math -import random -import uuid -from typing import cast - -import panel as pn -import torch -from IPython.core.display import HTML -from IPython.core.display_functions import display -from jaxtyping import Float, Int -from transformers import PreTrainedTokenizerBase - - -def probs_to_colors(probs: Float[torch.Tensor, "next_pos"]) -> list[str]: - # for the endoftext token - # no prediction, no color - colors = ["white"] - for p in probs.tolist(): - red_gap = 150 # the higher it is, the less red the tokens will be - green_blue_val = red_gap + int((255 - red_gap) * (1 - p)) - colors.append(f"rgb(255, {green_blue_val}, {green_blue_val})") - return colors - - -def single_loss_diff_to_color(loss_diff: float) -> str: - # if loss_diff is negative, we want the color to be red - # if loss_diff is positive, we want the color to be green - # if loss_diff is 0, we want the color to be white - # the color should be more intense the larger the absolute value of loss_diff - - def sigmoid(x: float) -> float: - return 1 / (1 + math.exp(-x)) - - scaled_loss_diff = sigmoid(loss_diff) # scale to 0-1 - - if scaled_loss_diff < 0.5: # red - red_val = 255 - green_blue_val = min(int(255 * 2 * scaled_loss_diff), 255) - return f"rgb({red_val}, {green_blue_val}, {green_blue_val})" - else: # green - green_val = 255 - red_blue_val = min(int(255 * 2 * (1 - scaled_loss_diff)), 255) - return f"rgb({red_blue_val}, {green_val}, {red_blue_val})" - - -def to_tok_prob_str(tok: int, prob: float, tokenizer: PreTrainedTokenizerBase) -> str: - tok_str = tokenizer.decode(tok).replace(" ", " ").replace("\n", r"\n") - prob_str = f"{prob:.2%}" - return f"{prob_str:>6} |{tok_str}|" - - -def token_to_html( - token: int, - tokenizer: PreTrainedTokenizerBase, - bg_color: str, - data: dict, -) -> str: - data = data or {} # equivalent to if not data: data = {} - # non-breakable space, w/o it leading spaces wouldn't be displayed - str_token = tokenizer.decode(token).replace(" ", " ") - - # background or user-select (for \n) goes here - specific_styles = {} - # for now just adds line break or doesn't - br = "" - - if bg_color: - specific_styles["background-color"] = bg_color - if str_token == "\n": - # replace new line character with two characters: \ and n - str_token = r"\n" - # add line break in html - br += "
" - # this is so we can copy the prompt without "\n"s - specific_styles["user-select"] = "none" - - style_str = data_str = "" - # converting style dict into the style attribute - if specific_styles: - inside_style_str = "; ".join(f"{k}: {v}" for k, v in specific_styles.items()) - style_str = f" style='{inside_style_str}'" - if data: - data_str = "".join( - f" data-{k}='{v.replace(' ', ' ')}'" for k, v in data.items() - ) - return f"
{str_token}
{br}" - - -_token_style = { - "border": "1px solid #888", - "display": "inline-block", - # each character of the same width, so we can easily spot a space - "font-family": "monospace", - "font-size": "14px", - "color": "black", - "background-color": "white", - "margin": "1px 0px 1px 1px", - "padding": "0px 1px 1px 1px", -} -_token_style_str = " ".join([f"{k}: {v};" for k, v in _token_style.items()]) - - -def vis_sample_prediction_probs( - sample_tok: Int[torch.Tensor, "pos"], - correct_probs: Float[torch.Tensor, "pos"], - top_k_probs: torch.return_types.topk, - tokenizer: PreTrainedTokenizerBase, -) -> str: - colors = probs_to_colors(correct_probs) - token_htmls = [] - - # Generate a unique ID for this instance (so we can have multiple instances on the same page) - unique_id = str(uuid.uuid4()) - - token_class = f"token_{unique_id}" - hover_div_id = f"hover_info_{unique_id}" - - for i in range(sample_tok.shape[0]): - tok = cast(int, sample_tok[i].item()) - data = {} - if i > 0: - correct_prob = correct_probs[i - 1].item() - data["next"] = to_tok_prob_str(tok, correct_prob, tokenizer) - top_k_probs_tokens = top_k_probs.indices[i - 1] - top_k_probs_values = top_k_probs.values[i - 1] - for j in range(top_k_probs_tokens.shape[0]): - top_tok = top_k_probs_tokens[j].item() - top_tok = cast(int, top_tok) - top_prob = top_k_probs_values[j].item() - data[f"top{j}"] = to_tok_prob_str(top_tok, top_prob, tokenizer) - - token_htmls.append( - token_to_html(tok, tokenizer, bg_color=colors[i], data=data).replace( - "class='token'", f"class='{token_class}'" - ) - ) - - html_str = f""" - - {"".join(token_htmls)}
- - """ - display(HTML(html_str)) - return html_str - - -def vis_pos_map( - pos_map: dict[tuple[int, int], float | int], - token_ids: Int[torch.Tensor, "prompt pos"], - tokenizer: PreTrainedTokenizerBase, - sample: int = 3, -): - """ - Randomly sample from pos_map and visualize the loss diff at the corresponding position. - """ - - token_htmls = [] - unique_id = str(uuid.uuid4()) - token_class = f"token_{unique_id}" - hover_div_id = f"hover_info_{unique_id}" - - # choose n random keys from pos_map - keys = random.sample(list(pos_map.keys()), k=sample) - - for key in keys: - prompt, pos = key - pre_toks = token_ids[prompt][:pos] - mask = torch.isin(pre_toks, torch.tensor([0, 1], dtype=torch.int8)) - pre_toks = pre_toks[ - ~mask - ] # remove and tokens, cause strikethrough in html - - for i in range(pre_toks.shape[0]): - pre_tok = cast(int, pre_toks[i].item()) - token_htmls.append( - token_to_html(pre_tok, tokenizer, bg_color="white", data={}).replace( - "class='token'", f"class='{token_class}'" - ) - ) - - tok = cast(int, token_ids[prompt][pos].item()) - value = cast(float, pos_map[key]) - - token_htmls.append( - token_to_html( - tok, - tokenizer, - bg_color=single_loss_diff_to_color(value), - data={"loss-diff": f"{value:.2f}"}, - ).replace("class='token'", f"class='{token_class}'") - ) - - # add break line - token_htmls.append("

") - - html_str = f""" - - {"".join(token_htmls)}
- - """ - display(HTML(html_str)) - return html_str - - -def token_selector( - vocab_map: dict[str, int] -) -> tuple[pn.widgets.MultiChoice, list[int]]: - tokens = list(vocab_map.keys()) - token_selector = pn.widgets.MultiChoice(name="Tokens", options=tokens) - token_ids = [vocab_map[token] for token in cast(list[str], token_selector.value)] - - def update_tokens(event): - token_ids.clear() - token_ids.extend([vocab_map[token] for token in event.new]) - - token_selector.param.watch(update_tokens, "value") - return token_selector, token_ids diff --git a/src/delphi/eval/vis_per_token_model.py b/src/delphi/eval/vis_per_token_model.py deleted file mode 100644 index 8daaa96f..00000000 --- a/src/delphi/eval/vis_per_token_model.py +++ /dev/null @@ -1,99 +0,0 @@ -from typing import Union - -import ipywidgets -import numpy as np -import plotly.graph_objects as go - - -def visualize_per_token_category( - input: dict[Union[str, int], dict[str, tuple]], - log_scale=False, - line_metric="Means", - checkpoint_mode=True, - shade_color="rgba(68, 68, 68, 0.3)", - line_color="rgb(31, 119, 180)", - bar_color="purple", - marker_color="SkyBlue", - background_color="AliceBlue", -) -> go.FigureWidget: - input_x = list(input.keys()) - categories = list(input[input_x[0]].keys()) - category = categories[0] - - def get_hovertexts(mid: np.ndarray, lo: np.ndarray, hi: np.ndarray) -> list[str]: - return [f"Loss: {m:.3f} ({l:.3f}, {h:.3f})" for m, l, h in zip(mid, lo, hi)] - - def get_plot_values(category: str) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - x = np.array([input[x][category] for x in input_x]).T - means, err_lo, err_hi = x[0], x[1], x[2] - return means, err_lo, err_hi - - means, err_lo, err_hi = get_plot_values(category) - - if checkpoint_mode: - scatter_plot = go.Figure( - [ - go.Scatter( - name="Upper Bound", - x=input_x, - y=means + err_hi, - mode="lines", - marker=dict(color=shade_color), - line=dict(width=0), - showlegend=False, - ), - go.Scatter( - name="Lower Bound", - x=input_x, - y=means - err_lo, - marker=dict(color=shade_color), - line=dict(width=0), - mode="lines", - fillcolor=shade_color, - fill="tonexty", - showlegend=False, - ), - go.Scatter( - name=line_metric, - x=input_x, - y=means, - mode="lines", - marker=dict( - color=line_color, - size=0, - line=dict(color=line_color, width=1), - ), - ), - ] - ) - else: - scatter_plot = go.Scatter( - x=input_x, - y=means, - error_y=dict( - type="data", - symmetric=False, - array=err_hi, - arrayminus=err_lo, - color=bar_color, - ), - marker=dict( - color=marker_color, - size=15, - line=dict(color=line_color, width=2), - ), - hovertext=get_hovertexts(means, err_lo, err_hi), - hoverinfo="text+x", - ) - g = go.FigureWidget( - data=scatter_plot, - layout=go.Layout( - yaxis=dict( - title="Loss", - type="log" if log_scale else "linear", - ), - plot_bgcolor=background_color, - ), - ) - - return g diff --git a/src/delphi/test_configs/__init__.py b/src/delphi/test_configs/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/delphi/test_configs/debug_transformers_bloom.json b/src/delphi/test_configs/debug_transformers_bloom.json deleted file mode 100644 index 2d72396e..00000000 --- a/src/delphi/test_configs/debug_transformers_bloom.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "max_seq_len": 512, - "max_epochs": 2, - "eval_iters": 1, - "batch_size": 64, - "model_config": { - "model_class": "BloomForCausalLM", - "apply_residual_connection_post_layernorm": false, - "attention_dropout": 0.0, - "bos_token_id": 1, - "eos_token_id": 2, - "hidden_dropout": 0.0, - "hidden_size": 8, - "initializer_range": 0.02, - "layer_norm_epsilon": 1e-05, - "n_head": 2, - "n_layer": 2, - "pretraining_tp": 1, - "slow_but_exact": false, - "use_cache": true, - "vocab_size": 4096 - }, - "batch_ordering_seed": 42, - "torch_seed": 1337, - "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" - }, - "out_repo_id": "" -} \ No newline at end of file diff --git a/src/delphi/test_configs/v0-llama2-100k.json b/src/delphi/test_configs/v0-llama2-100k.json deleted file mode 100644 index 584b5017..00000000 --- a/src/delphi/test_configs/v0-llama2-100k.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "max_seq_len": 512, - "model_config": { - "model_class": "LlamaForCausalLM", - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 1, - "eos_token_id": 2, - "hidden_act": "silu", - "hidden_size": 48, - "initializer_range": 0.02, - "intermediate_size": 128, - "max_position_embeddings": 512, - "num_attention_heads": 8, - "num_hidden_layers": 4, - "num_key_value_heads": 4, - "pretraining_tp": 1, - "rms_norm_eps": 1e-05, - "rope_scaling": null, - "rope_theta": 10000.0, - "tie_word_embeddings": true, - "use_cache": true, - "vocab_size": 4096 - }, - "batch_ordering_seed": 42, - "torch_seed": 1337, - "dataset": { - "name": "delphi-suite/v0-tinystories-v2-clean-tokenized" - }, - "out_repo_id": "" -} \ No newline at end of file diff --git a/src/delphi/train/__init__.py b/src/delphi/train/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/delphi/train/config/dataset_config.py b/src/delphi/train/config/dataset_config.py deleted file mode 100644 index 0c1e356e..00000000 --- a/src/delphi/train/config/dataset_config.py +++ /dev/null @@ -1,46 +0,0 @@ -from dataclasses import dataclass, field -from typing import cast - -import datasets -from beartype import beartype -from datasets import Dataset, load_dataset - - -@beartype -@dataclass(frozen=True) -class DatasetConfig: - name: str = field( - metadata={"help": "tokenized dataset on huggingface to use for train"}, - ) - feature: str = field( - default="tokens", - metadata={ - "help": "feature in the train dataset to use for train; should be a list of max_seq_len+1 token ints" - }, - ) - train_split: str = field( - default="train", - metadata={"help": "split of the dataset to use for training"}, - ) - validation_split: str = field( - default="validation", - metadata={"help": "split of the dataset to use for validation"}, - ) - - def _load(self, split) -> Dataset: - ds = load_dataset( - self.name, - split=split, - features=datasets.Features( - {self.feature: datasets.Sequence(datasets.Value("int32"))} - ), - ) - ds = cast(Dataset, ds) - ds.set_format("torch") - return ds - - def load_train(self) -> Dataset: - return self._load(self.train_split) - - def load_validation(self) -> Dataset: - return self._load(self.validation_split) diff --git a/src/delphi/train/config/training_config.py b/src/delphi/train/config/training_config.py deleted file mode 100644 index c68896d4..00000000 --- a/src/delphi/train/config/training_config.py +++ /dev/null @@ -1,106 +0,0 @@ -import os -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Optional - -import platformdirs -from beartype import beartype - -from .adam_config import AdamConfig -from .dataset_config import DatasetConfig -from .debug_config import DebugConfig -from .wandb_config import WandbConfig - - -@beartype -@dataclass(frozen=True, kw_only=True) -class TrainingConfig: - model_config: dict[str, Any] = field( - metadata={ - "help": "model config; class_name=name of model class in transformers, everything else is kwargs for the corresponding model config" - }, - ) - max_seq_len: int = field(metadata={"help": "max sequence length"}) - # meta - run_name: str = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - output_dir: str = field( - default=os.path.join(platformdirs.user_data_dir(appname="delphi"), run_name), - metadata={"help": "output directory"}, - ) - - # device - device: str = field( - default="auto", metadata={"help": "device to use (cuda, mps, cpu)"} - ) - - # checkpoints, logging, eval - checkpoint_interval: int = field( - default=2000, metadata={"help": "checkpoint every N iters"} - ) - extra_checkpoint_iters: list[int] = field( - default_factory=list, - metadata={"help": "manually list iterations to save checkpoints on"}, - ) - log_interval: int = field(default=1, metadata={"help": "log every N iters"}) - eval_iters: int = field(default=100, metadata={"help": "use N iters for each eval"}) - - # resume from checkpoint - resume_from_path: Optional[str] = field( - default=None, - metadata={ - "help": "path to a checkpoint to resume from (if init_from=='resume')" - }, - ) - - # data - batch_size: int = field( - default=64, - metadata={ - "help": "number of samples used to compute the gradient for a single optimizer step" - }, - ) - - # training - max_epochs: int = field( - default=10, metadata={"help": "total number of training epochs"} - ) - grad_clip: float = field( - default=1.0, - metadata={"help": "clip gradients at this value, or disable if == 0.0"}, - ) - gradient_accumulation_steps: int = field( - default=1, - metadata={ - "help": "if > 1 reduces memory usage by computing gradient in microbatches" - }, - ) - # (adamw) optimizer - adam: AdamConfig = field(default_factory=AdamConfig) - - # reproducibility - batch_ordering_seed: int = field( - metadata={"help": "seed used for pseudorandomly sampling data during training"}, - ) - torch_seed: int = field(metadata={"help": "seed used for torch"}) - save_optimizer: bool = True - - # data - dataset: DatasetConfig = field( - metadata={"help": "specify training and validation data"}, - ) - - tokenizer: str = field( - default="", - metadata={ - "help": "HF repo id or local directory containing the tokenizer. Used only to upload it to HF with the model, not for training" - }, - ) - - # third party - wandb: Optional[WandbConfig] = None - out_repo_id: str = field( - metadata={"help": "set to empty string to not push to repo"}, - ) - - # debug - debug_config: DebugConfig = field(default_factory=DebugConfig) diff --git a/src/delphi/train/config/wandb_config.py b/src/delphi/train/config/wandb_config.py deleted file mode 100644 index 9b4e3c55..00000000 --- a/src/delphi/train/config/wandb_config.py +++ /dev/null @@ -1,11 +0,0 @@ -from dataclasses import dataclass - -from beartype import beartype - - -@beartype -@dataclass -class WandbConfig: - project: str - entity: str - silence: bool = False diff --git a/tests/eval/test_compare_models.py b/tests/eval/test_compare_models.py deleted file mode 100644 index 0521b0cb..00000000 --- a/tests/eval/test_compare_models.py +++ /dev/null @@ -1,23 +0,0 @@ -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer - -from delphi.eval.compare_models import NextTokenStats, compare_models -from delphi.eval.utils import load_validation_dataset, tokenize - - -def test_compare_models(): - with torch.set_grad_enabled(False): - model = AutoModelForCausalLM.from_pretrained("roneneldan/TinyStories-1M") - model_instruct = AutoModelForCausalLM.from_pretrained( - "roneneldan/TinyStories-Instruct-1M" - ) - ds_txt = load_validation_dataset("tinystories-v2-clean")["story"] - tokenizer = AutoTokenizer.from_pretrained("roneneldan/TinyStories-1M") - sample_tok = tokenize(tokenizer, ds_txt[0]) - K = 3 - model_comparison = compare_models(model, model_instruct, sample_tok, top_k=K) - # ignore the first element comparison - assert model_comparison[0] is None - assert isinstance(model_comparison[1], NextTokenStats) - assert len(model_comparison) == sample_tok.shape[0] - assert len(model_comparison[1].topk) == K diff --git a/tests/eval/test_spacy_token_labelling.py b/tests/eval/test_spacy_token_labelling.py deleted file mode 100644 index 8a799b95..00000000 --- a/tests/eval/test_spacy_token_labelling.py +++ /dev/null @@ -1,189 +0,0 @@ -import pickle -from pathlib import Path - -import pytest -from spacy.language import Language -from spacy.tokens import Doc -from transformers import AutoTokenizer - -import delphi.eval.spacy_token_labelling as tl - -# skip all tests in this module -pytestmark = pytest.mark.skip( - "tests are slow and we're not using this module currently" -) - -labelled_token_ids_dict: dict[int, dict[str, bool]] = {} - - -@pytest.fixture -def dummy_doc() -> tuple[str, Doc, dict[str, bool]]: - """ - Create a dummy Doc (list of Tokens) with specific attributes for testing purposes. - """ - nlp_dummy = Language() - - # Assume we're creating a dummy token with specific attributes - words = ["Peter", "is", "a", "person"] - spaces = [True, True, True, True] # No space after "dummy_token" - pos_tags = ["PROPN", "AUX", "DET", "NOUN"] # Part-of-speech tag - dep_tags = ["nsubj", "ROOT", "det", "attr"] # Dependency tag - ner_tags = ["PERSON", "", "", ""] # Named entity tag - - # Ensure the length of pos_tags and dep_tags matches the length of words - assert len(words) == len(pos_tags) == len(dep_tags) == len(ner_tags) - - # Create a Doc with one dummy token - doc = Doc(nlp_dummy.vocab, words=words, spaces=spaces) - - # Manually set POS, dependency and NER tags - for token, pos, dep, ner_tag in zip(doc, pos_tags, dep_tags, ner_tags): - token.pos_, token.dep_, token.ent_type_ = pos, dep, ner_tag - - # Token labels for "Peter" in the dummy doc - PETER_TOKEN_LABEL = { - "Starts with space": False, - "Capitalized": True, - "Is Adjective": False, - "Is Adposition": False, - "Is Adverb": False, - "Is Auxiliary": False, - "Is Coordinating conjuction": False, - "Is Determiner": False, - "Is Interjunction": False, - "Is Noun": False, - "Is Numeral": False, - "Is Particle": False, - "Is Pronoun": False, - "Is Proper Noun": True, - "Is Punctuation": False, - "Is Subordinating conjuction": False, - "Is Symbol": False, - "Is Verb": False, - "Is Other": False, - "Is Named Entity": True, - } - text = " ".join(words) - return text, doc, PETER_TOKEN_LABEL - - -def test_explain_token_labels(dummy_doc): - """ - Test the explain_token_labels function. - """ - # explain all labels - tl.explain_token_labels() - # print explanations for the first token in doc - text, doc, PETER_TOKEN_LABEL = dummy_doc - tl.explain_token_labels(doc[0]) - - -def test_label_single_token(dummy_doc): - """ - Test the label_single_token function. - """ - # create a dummy token - text, doc, PETER_TOKEN_LABEL = dummy_doc - token = doc[0] - # label the token - labels = tl.label_single_token(token) - # check if the labels are correct - assert labels == PETER_TOKEN_LABEL - - -def test_label_sentence(dummy_doc): - """ - Test the label_sentence function. - """ - text, doc, PETER_TOKEN_LABEL = dummy_doc - # label the sentence - labels = tl.label_sentence(doc) - # assert the first token is labeled correctly - assert labels[0] == PETER_TOKEN_LABEL - # iterate through tokens in doc - for token, label in zip(doc, labels): - assert label == tl.label_single_token(token) - - -def test_label_batch_sentences(dummy_doc): - """ - Test the label_batch_sentences function. - """ - # create a batch of sentences - text, doc, PETER_TOKEN_LABEL = dummy_doc - text = text.split(" ") - batch = [text, text, text] - # label the batch - labels = tl.label_batch_sentences(batch, tokenized=True) - # assert the first token is labeled correctly - assert labels[0][0] == PETER_TOKEN_LABEL - assert labels[1][0] == PETER_TOKEN_LABEL - assert labels[2][0] == PETER_TOKEN_LABEL - # iterate through tokens in doc - for token, label in zip(doc, labels[0]): - assert label == tl.label_single_token(token) - - -def is_valid_structure(obj: dict[int, dict[str, bool]]) -> bool: - """ - Checks whether the obj fits the structure of `dict[int, dict[str, bool]]`. Returns True, if it fits, False otherwise. - """ - if not isinstance(obj, dict): - print(f"Main structure is not dict! Instead is type {type(obj)}") - return False - for key, value in obj.items(): - if not isinstance(key, int) or not isinstance(value, dict): - print( - f"Main structure is dict, but its keys are either not int or its values are not dicts. Instead key is type {type(key)} and value is type {type(value)}" - ) - return False - for sub_key, sub_value in value.items(): - if not isinstance(sub_key, str) or not isinstance(sub_value, bool): - print( - f"The structure dict[int, dict[X, Y]] is True, but either X is not str or Y is not bool. Instead X is type {type(sub_key)} and Y is type {type(sub_value)}" - ) - return False - return True - - -def test_label_tokens_from_tokenizer(): - """ - Simple test, checking if download of tokinzer and the labelling of all tokens in its vocabulary works. - """ - global labelled_token_ids_dict - # get a tokinzer - model_name = "delphi-suite/delphi-llama2-100k" - tokenizer = AutoTokenizer.from_pretrained(model_name) - vocab_size = tokenizer.vocab_size - - tokens_str, labelled_token_ids_dict = tl.label_tokens_from_tokenizer(tokenizer) - # count the number of lines in the token_str - assert tokens_str.count("\n") == (vocab_size + 1) # + 1, because of token '\n' - assert len(labelled_token_ids_dict.keys()) == vocab_size - assert is_valid_structure(labelled_token_ids_dict) == True - - -@pytest.mark.parametrize("path", [Path("temp/token_labels.csv")]) -def test_import_token_labels(path: Path): - """ - Simple test, checking if the import of token labels works. - - Note: Because we want to use pure pytest and not install any extra dependencies (e.g. pytest-depencency) we recreate the `labelled_tokens_dict` in this test as we did in `test_label_tokens_from_tokenizer`. This duplication is not ideal, but it is the best quick&dirty solution for now. - """ - # create the labelled_token_ids_dict - model_name = "delphi-suite/delphi-llama2-100k" - tokenizer = AutoTokenizer.from_pretrained(model_name) - _, labelled_token_ids_dict = tl.label_tokens_from_tokenizer(tokenizer) - - # create the path - path.parent.mkdir(parents=True, exist_ok=True) - # save the file - df = tl.convert_label_dict_to_df(labelled_token_ids_dict) - df.to_csv(path, index=False) - - # load the file with our function to be tested - loaded_dict = tl.import_token_labels(path) - - # assure that the structure is correct - assert loaded_dict == labelled_token_ids_dict - assert is_valid_structure(loaded_dict) == True diff --git a/tests/eval/test_token_map.py b/tests/eval/test_token_map.py deleted file mode 100644 index 2f896326..00000000 --- a/tests/eval/test_token_map.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest -from datasets import Dataset - -from delphi.eval.token_map import token_map - - -def test_token_map(): - tokenized_dataset = Dataset.from_dict( - { - "tokens": [ - [0, 1, 2, 3, 4, 5, 0, 6, 7], - [0, 1, 2, 3, 4, 5, 0, 6, 7], - [0, 1, 2, 3, 4, 5, 0, 6, 7], - ] - } - ) - assert token_map(tokenized_dataset, tokenizer_size=9) == [ - [(0, 0), (0, 6), (1, 0), (1, 6), (2, 0), (2, 6)], - [(0, 1), (1, 1), (2, 1)], - [(0, 2), (1, 2), (2, 2)], - [(0, 3), (1, 3), (2, 3)], - [(0, 4), (1, 4), (2, 4)], - [(0, 5), (1, 5), (2, 5)], - [(0, 7), (1, 7), (2, 7)], - [(0, 8), (1, 8), (2, 8)], - [], # token 8 is not present in the dataset - ] - - # fmt: off - tokenized_dataset = Dataset.from_dict( - { # one really long prompt - "tokens": [ - [0, 1, 2, 3, 4, 5, 0, 6, 7, 0, 1, 2, 3, 4, 5, 0, 6, 7, 0, 1, 2, 3, 4, 5, 0, 6, 7] - ] - } - ) - # fmt: on - assert token_map(tokenized_dataset, tokenizer_size=8) == [ - [(0, 0), (0, 6), (0, 9), (0, 15), (0, 18), (0, 24)], - [(0, 1), (0, 10), (0, 19)], - [(0, 2), (0, 11), (0, 20)], - [(0, 3), (0, 12), (0, 21)], - [(0, 4), (0, 13), (0, 22)], - [(0, 5), (0, 14), (0, 23)], - [(0, 7), (0, 16), (0, 25)], - [(0, 8), (0, 17), (0, 26)], - ] diff --git a/tests/eval/test_token_positions.py b/tests/eval/test_token_positions.py deleted file mode 100644 index 1adef6b7..00000000 --- a/tests/eval/test_token_positions.py +++ /dev/null @@ -1,51 +0,0 @@ -from math import isclose -from typing import cast - -import pytest -from datasets import Dataset - -from delphi.eval.token_positions import * - - -@pytest.fixture -def mock_data(): - token_ids = Dataset.from_dict( - {"tokens": [[1, 2, 3], [4, 5, 6], [7, 8, 9]]} - ).with_format("torch") - token_labels = { - 1: {"Is Noun": False, "Is Verb": True}, - 2: {"Is Noun": True, "Is Verb": True}, - 3: {"Is Noun": False, "Is Verb": False}, - 4: {"Is Noun": True, "Is Verb": False}, - 5: {"Is Noun": False, "Is Verb": True}, - 6: {"Is Noun": True, "Is Verb": True}, - 7: {"Is Noun": False, "Is Verb": False}, - 8: {"Is Noun": True, "Is Verb": False}, - 9: {"Is Noun": False, "Is Verb": True}, - } - metrics = torch.tensor([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]) - return token_ids, token_labels, metrics - - -def test_get_all_tok_metrics_in_label(mock_data): - token_ids, token_labels, metrics = mock_data - result = get_all_tok_metrics_in_label( - token_ids["tokens"], token_labels, metrics, "Is Noun" - ) - expected = { - (0, 1): 0.2, - (1, 0): 0.4, - (1, 2): 0.6, - (2, 1): 0.8, - } - # use isclose to compare floating point numbers - for k in result: - assert isclose(cast(float, result[k]), expected[k], rel_tol=1e-6) # type: ignore - - # test with quantile filtering - result_q = get_all_tok_metrics_in_label( - token_ids["tokens"], token_labels, metrics, "Is Noun", q_start=0.3, q_end=1.0 - ) - expected_q = {(1, 2): 0.6, (2, 1): 0.8, (1, 0): 0.4} - for k in result_q: - assert isclose(cast(float, result_q[k]), expected_q[k], rel_tol=1e-6) # type: ignore diff --git a/tests/eval/test_utils_eval.py b/tests/eval/test_utils_eval.py deleted file mode 100644 index ad0f54b8..00000000 --- a/tests/eval/test_utils_eval.py +++ /dev/null @@ -1,78 +0,0 @@ -from math import isclose - -import pytest -import torch - -from delphi.eval.utils import ( - dict_filter_quantile, - gather_logprobs, - load_validation_dataset, -) - - -def test_gather_logprobs(): - # vocab size = 3 - logprobs = torch.tensor( - [ - # batch 0 - [ - # seq 0 - [0.00, 0.01, 0.02], - # seq 1 - [0.10, 0.11, 0.12], - ], - # batch 1 - [ - # seq 0 - [1.00, 1.01, 1.02], - # seq 1 - [1.10, 1.11, 1.12], - ], - ] - ) - tokens = torch.tensor( - [ - # batch 0 - [0, 2], - # batch 1 - [1, 2], - ] - ) - expected_output = torch.tensor( - [ - # batch 0 - [0.00, 0.12], - # batch 1 - [1.01, 1.12], - ] - ) - result = gather_logprobs(logprobs, tokens) - assert torch.allclose(result, expected_output) - - -def test_load_validation_dataset(): - text = load_validation_dataset("tinystories-v2-clean") - tokenized = load_validation_dataset("tinystories-v2-clean-tokenized-v0") - - -@pytest.mark.filterwarnings( - "ignore::RuntimeWarning" -) # ignore warnings from numpy empty slice -def test_dict_filter_quantile(): - d = {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.4, 5: 0.5} - result = dict_filter_quantile(d, 0.2, 0.6) - expected = {2: 0.2, 3: 0.3, 4: 0.4} - for k in result: - assert isclose(result[k], expected[k], rel_tol=1e-6) - - # test invalid quantile range - with pytest.raises(ValueError): - dict_filter_quantile(d, 0.6, 0.2) - with pytest.raises(ValueError): - dict_filter_quantile(d, 0.1, 1.1) - with pytest.raises(ValueError): - dict_filter_quantile(d, -0.1, 0.6) - - # test empty dict, will raise a warning - result = dict_filter_quantile({}, 0.2, 0.6) - assert result == {} diff --git a/tests/test_dummy.py b/tests/test_dummy.py deleted file mode 100644 index 88261b12..00000000 --- a/tests/test_dummy.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest -import torch -from beartype.roar import BeartypeCallHintViolation - -from delphi.dummy import dummy - - -def test_dummy(): - tensor1 = torch.tensor([1.0, 2.0, 3.0]) - tensor2 = torch.tensor([[1, 2, 3], [4, 5, 6]]) - assert torch.allclose(dummy(tensor1), torch.tensor([2.0, 3.0, 4.0])) - assert torch.allclose(dummy(tensor2), torch.tensor([0.9, 1.9, 2.9])) - tensor3 = torch.tensor([1, 2, 3]) - with pytest.raises(BeartypeCallHintViolation): - dummy(tensor3) diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 00000000..cdf88413 --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,91 @@ +from math import isclose +from typing import cast + +import pytest +import torch +from datasets import Dataset + +from delphi.eval import dict_filter_quantile, get_all_tok_metrics_in_label + + +@pytest.mark.filterwarnings( + "ignore::RuntimeWarning" +) # ignore warnings from numpy empty slice +def test_dict_filter_quantile(): + d = {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.4, 5: 0.5} + result = dict_filter_quantile(d, 0.2, 0.6) + expected = {2: 0.2, 3: 0.3} + + # compare keys + assert result.keys() == expected.keys() + # compare values + for k in result: + assert isclose(result[k], expected[k], rel_tol=1e-6) + + # test with negative values + d = {1: -0.1, 2: -0.2, 3: -0.3, 4: -0.4, 5: -0.5} + result = dict_filter_quantile(d, 0.2, 0.6) + expected = {3: -0.3, 4: -0.4} + + # compare keys + assert result.keys() == expected.keys() + # compare values + for k in result: + assert isclose(result[k], expected[k], rel_tol=1e-6) + + # test invalid quantile range + with pytest.raises(ValueError): + dict_filter_quantile(d, 0.6, 0.2) + with pytest.raises(ValueError): + dict_filter_quantile(d, 0.1, 1.1) + with pytest.raises(ValueError): + dict_filter_quantile(d, -0.1, 0.6) + + # test empty dict, will raise a warning + result = dict_filter_quantile({}, 0.2, 0.6) + assert result == {} + + +def test_get_all_tok_metrics_in_label(): + token_ids = Dataset.from_dict( + {"tokens": [[1, 2, 3], [4, 5, 6], [7, 8, 9]]} + ).with_format("torch") + selected_tokens = [2, 4, 6, 8] + metrics = torch.tensor([[-1, 0.45, -0.33], [-1.31, 2.3, 0.6], [0.2, 0.8, 0.1]]) + result = get_all_tok_metrics_in_label( + token_ids["tokens"], # type: ignore + selected_tokens, + metrics, + ) + # key: (prompt_pos, tok_pos), value: logprob + expected = { + (0, 1): 0.45, + (1, 0): -1.31, + (1, 2): 0.6, + (2, 1): 0.8, + } + + # compare keys + assert result.keys() == expected.keys() + # compare values + for k in result: + assert isclose(cast(float, result[k]), expected[k], rel_tol=1e-6) # type: ignore + + # test with quantile filtering + result_q = get_all_tok_metrics_in_label( + token_ids["tokens"], # type: ignore + selected_tokens, + metrics, + q_start=0.6, + q_end=1.0, + ) + expected_q = { + (1, 2): 0.6, + (2, 1): 0.8, + } + + # compare keys + assert result_q.keys() == expected_q.keys() + # compare values + for k in result_q: + assert isclose(cast(float, result_q[k]), expected_q[k], rel_tol=1e-6) # type: ignore diff --git a/tests/dataset/test_tokeniation.py b/tests/test_tokeniation.py similarity index 97% rename from tests/dataset/test_tokeniation.py rename to tests/test_tokeniation.py index bb4180ba..cc9494b2 100644 --- a/tests/dataset/test_tokeniation.py +++ b/tests/test_tokeniation.py @@ -5,7 +5,7 @@ from datasets import Dataset from transformers import AutoTokenizer -from delphi.dataset.tokenization import extend_deque, make_new_sample, tokenize_dataset +from delphi.tokenization import extend_deque, make_new_sample, tokenize_dataset @pytest.fixture diff --git a/tests/test_utils.py b/tests/test_utils.py index 597438ca..79b639ad 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,13 @@ -from delphi.utils import hf_split_to_split_name +import random +import string -from .utils import random_string +import torch + +from delphi.utils import gather_logprobs, hf_split_to_split_name + + +def random_string(length: int) -> str: + return "".join(random.choices(string.ascii_lowercase, k=length)) def test_hf_split_to_split_name(): @@ -12,3 +19,43 @@ def test_hf_split_to_split_name(): assert hf_split_to_split_name(f"{random_split_name}[:200]") == random_split_name assert hf_split_to_split_name(f"{random_split_name}[200:]") == random_split_name assert hf_split_to_split_name(f"{random_split_name}[200:400]") == random_split_name + + +def test_gather_logprobs(): + # vocab size = 3 + logprobs = torch.tensor( + [ + # batch 0 + [ + # seq 0 + [0.00, 0.01, 0.02], + # seq 1 + [0.10, 0.11, 0.12], + ], + # batch 1 + [ + # seq 0 + [1.00, 1.01, 1.02], + # seq 1 + [1.10, 1.11, 1.12], + ], + ] + ) + tokens = torch.tensor( + [ + # batch 0 + [0, 2], + # batch 1 + [1, 2], + ] + ) + expected_output = torch.tensor( + [ + # batch 0 + [0.00, 0.12], + # batch 1 + [1.01, 1.12], + ] + ) + result = gather_logprobs(logprobs, tokens) + assert torch.allclose(result, expected_output) diff --git a/tests/train/config/test_config_utils.py b/tests/train/config/test_config_utils.py index 710aa404..b67791c6 100644 --- a/tests/train/config/test_config_utils.py +++ b/tests/train/config/test_config_utils.py @@ -1,8 +1,6 @@ from typing import Optional -import pytest - -from delphi.constants import TEST_CONFIGS_DIR +from delphi import TEST_CONFIGS_DIR from delphi.train.config.utils import ( _unoptionalize, build_config_from_files_and_overrides, @@ -48,7 +46,7 @@ def test_build_config_from_files_and_overrides(): assert config.eval_iters == 5 # check base values assert config.max_epochs == 2 - assert config.dataset.name == "delphi-suite/v0-tinystories-v2-clean-tokenized" + assert config.dataset.path == "delphi-suite/stories-tokenized" def test_unoptionalize(): diff --git a/tests/train/test_train_step.py b/tests/train/test_train_step.py index 1a7db8cb..e06fa1af 100644 --- a/tests/train/test_train_step.py +++ b/tests/train/test_train_step.py @@ -7,8 +7,7 @@ from jaxtyping import Float from transformers import PreTrainedModel -from delphi.constants import TEST_CONFIGS_DIR -from delphi.eval.utils import get_all_and_next_logprobs +from delphi import TEST_CONFIGS_DIR from delphi.train.config import TrainingConfig from delphi.train.config.utils import build_config_from_files_and_overrides from delphi.train.train_step import accumulate_gradients, train_step @@ -18,6 +17,7 @@ init_model, setup_determinism, ) +from delphi.utils import get_all_and_next_logprobs def load_test_config(preset_name: str) -> TrainingConfig: diff --git a/tests/train/test_wandb_utils.py b/tests/train/test_wandb_utils.py index 4ca89670..70179304 100644 --- a/tests/train/test_wandb_utils.py +++ b/tests/train/test_wandb_utils.py @@ -7,11 +7,11 @@ import transformers from dacite import from_dict -from delphi.constants import TEST_CONFIGS_DIR +from delphi import TEST_CONFIGS_DIR from delphi.train.config import TrainingConfig from delphi.train.config.utils import build_config_from_files_and_overrides from delphi.train.utils import ModelTrainingState, initialize_model_training_state -from delphi.train.wandb_utils import init_wandb, log_to_wandb, silence_wandb +from delphi.train.wandb_utils import init_wandb, log_to_wandb @pytest.fixture @@ -19,10 +19,7 @@ def mock_training_config() -> TrainingConfig: preset_path = TEST_CONFIGS_DIR / "debug.json" overrides = { "run_name": "test_run", - "wandb": { - "entity": "test_entity", - "project": "test_project", - }, + "wandb": "test_entity/test_project", } return build_config_from_files_and_overrides([preset_path], overrides) @@ -39,12 +36,6 @@ def mock_model_training_state(mock_training_config): return mts -@patch.dict("os.environ", {}, clear=True) -def test_silence_wandb(): - silence_wandb() - assert os.environ["WANDB_SILENT"] == "true" - - @patch("wandb.init") def test_init_wandb(mock_wandb_init: MagicMock, mock_training_config): init_wandb(mock_training_config) diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index ed81b58a..00000000 --- a/tests/utils.py +++ /dev/null @@ -1,6 +0,0 @@ -import random -import string - - -def random_string(length: int) -> str: - return "".join(random.choices(string.ascii_lowercase, k=length))