From f75d9f93bb14ed56058f4c70ae4670b6cf3a8ed1 Mon Sep 17 00:00:00 2001
From: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com>
Date: Thu, 30 Nov 2023 05:26:04 +0100
Subject: [PATCH] Update llama examples (#90)
---
.gitignore | 4 +-
examples/running-llamas/README.md | 49 ++-
.../A100-80GB/forward_latency_plot.png | Bin 39612 -> 0 bytes
.../A100-80GB/forward_memory_plot.png | Bin 37099 -> 0 bytes
.../artifacts/A100-80GB/full_report.csv | 11 -
.../A100-80GB/generate_memory_plot.png | Bin 41656 -> 0 bytes
.../A100-80GB/generate_throughput_plot.png | Bin 37626 -> 0 bytes
.../artifacts/A100-80GB/rich_table.svg | 163 -------
.../artifacts/A100-80GB/short_report.csv | 11 -
.../Llama-13b/decode_throughput_bar_plot.png | Bin 0 -> 41316 bytes
.../Llama-13b/decode_throughput_line_plot.png | Bin 0 -> 64674 bytes
.../artifacts/Llama-13b/full_report.csv | 31 ++
...generate_max_memory_allocated_bar_plot.png | Bin 0 -> 44083 bytes
...enerate_max_memory_allocated_line_plot.png | Bin 0 -> 61570 bytes
.../generate_max_memory_reserved_bar_plot.png | Bin 0 -> 46918 bytes
...generate_max_memory_reserved_line_plot.png | Bin 0 -> 61567 bytes
.../peak_decode_throughput_bar_plot.png | Bin 0 -> 40338 bytes
.../Llama-13b/prefill_latency_bar_plot.png | Bin 0 -> 36349 bytes
.../Llama-13b/prefill_latency_line_plot.png | Bin 0 -> 59534 bytes
.../artifacts/Llama-13b/short_report.csv | 31 ++
.../Llama-65b/decode_throughput_bar_plot.png | Bin 0 -> 32942 bytes
.../Llama-65b/decode_throughput_line_plot.png | Bin 0 -> 43750 bytes
.../artifacts/Llama-65b/full_report.csv | 9 +
...generate_max_memory_allocated_bar_plot.png | Bin 0 -> 39213 bytes
...enerate_max_memory_allocated_line_plot.png | Bin 0 -> 47608 bytes
.../generate_max_memory_reserved_bar_plot.png | Bin 0 -> 39831 bytes
...generate_max_memory_reserved_line_plot.png | Bin 0 -> 48540 bytes
.../peak_decode_throughput_bar_plot.png | Bin 0 -> 30405 bytes
.../Llama-65b/prefill_latency_bar_plot.png | Bin 0 -> 30959 bytes
.../Llama-65b/prefill_latency_line_plot.png | Bin 0 -> 41751 bytes
.../artifacts/Llama-65b/short_report.csv | 9 +
.../Llama-7b/decode_throughput_bar_plot.png | Bin 0 -> 41663 bytes
.../Llama-7b/decode_throughput_line_plot.png | Bin 0 -> 70079 bytes
.../artifacts/Llama-7b/full_report.csv | 36 ++
...generate_max_memory_allocated_bar_plot.png | Bin 0 -> 44722 bytes
...enerate_max_memory_allocated_line_plot.png | Bin 0 -> 64550 bytes
.../generate_max_memory_reserved_bar_plot.png | Bin 0 -> 47072 bytes
...generate_max_memory_reserved_line_plot.png | Bin 0 -> 63481 bytes
.../peak_decode_throughput_bar_plot.png | Bin 0 -> 40604 bytes
.../Llama-7b/prefill_latency_bar_plot.png | Bin 0 -> 36913 bytes
.../Llama-7b/prefill_latency_line_plot.png | Bin 0 -> 58741 bytes
.../artifacts/Llama-7b/short_report.csv | 36 ++
examples/running-llamas/configs/_base_.yaml | 43 +-
examples/running-llamas/configs/bnb.yaml | 11 -
.../running-llamas/configs/fp16+bt+tp=2.yaml | 22 +
examples/running-llamas/configs/fp16+bt.yaml | 9 +
.../running-llamas/configs/fp16+dp=2.yaml | 15 +
.../running-llamas/configs/fp16+fa2+tp=2.yaml | 22 +
examples/running-llamas/configs/fp16+fa2.yaml | 9 +
.../configs/fp16+gptq+exllamav1+dp=2.yaml | 27 ++
.../configs/fp16+gptq+exllamav1.yaml | 20 +
.../configs/fp16+gptq+exllamav2+dp=2.yaml | 27 ++
.../configs/fp16+gptq+exllamav2.yaml | 20 +
.../running-llamas/configs/fp16+tp=2.yaml | 21 +
examples/running-llamas/configs/fp16.yaml | 6 +
examples/running-llamas/configs/gptq.yaml | 6 -
.../0/.hydra/config.yaml | 73 ----
.../0/.hydra/hydra.yaml | 174 --------
.../0/.hydra/overrides.yaml | 1 -
.../0/hydra_config.yaml | 79 ----
.../0/inference_results.csv | 2 -
.../4/.hydra/config.yaml | 73 ----
.../4/.hydra/hydra.yaml | 174 --------
.../4/.hydra/overrides.yaml | 1 -
.../4/hydra_config.yaml | 79 ----
.../4/inference_results.csv | 2 -
.../1/.hydra/config.yaml | 73 ----
.../1/.hydra/hydra.yaml | 174 --------
.../1/.hydra/overrides.yaml | 1 -
.../1/hydra_config.yaml | 79 ----
.../1/inference_results.csv | 2 -
.../multirun.yaml | 246 -----------
.../2/.hydra/config.yaml | 73 ----
.../2/.hydra/hydra.yaml | 174 --------
.../2/.hydra/overrides.yaml | 1 -
.../2/hydra_config.yaml | 79 ----
.../2/inference_results.csv | 2 -
.../3/.hydra/config.yaml | 73 ----
.../3/.hydra/hydra.yaml | 174 --------
.../3/.hydra/overrides.yaml | 1 -
.../3/hydra_config.yaml | 79 ----
.../3/inference_results.csv | 2 -
.../0/.hydra/config.yaml | 73 ----
.../0/.hydra/hydra.yaml | 174 --------
.../0/.hydra/overrides.yaml | 1 -
.../0/hydra_config.yaml | 79 ----
.../0/inference_results.csv | 2 -
.../4/.hydra/config.yaml | 73 ----
.../4/.hydra/hydra.yaml | 174 --------
.../4/.hydra/overrides.yaml | 1 -
.../4/hydra_config.yaml | 79 ----
.../4/inference_results.csv | 2 -
.../1/.hydra/config.yaml | 73 ----
.../1/.hydra/hydra.yaml | 174 --------
.../1/.hydra/overrides.yaml | 1 -
.../1/hydra_config.yaml | 79 ----
.../1/inference_results.csv | 2 -
.../multirun.yaml | 246 -----------
.../2/.hydra/config.yaml | 73 ----
.../2/.hydra/hydra.yaml | 174 --------
.../2/.hydra/overrides.yaml | 1 -
.../2/hydra_config.yaml | 79 ----
.../2/inference_results.csv | 2 -
.../3/.hydra/config.yaml | 73 ----
.../3/.hydra/hydra.yaml | 174 --------
.../3/.hydra/overrides.yaml | 1 -
.../3/hydra_config.yaml | 79 ----
.../3/inference_results.csv | 2 -
examples/running-llamas/report.py | 405 +++++++++---------
optimum_benchmark/backends/isolation_utils.py | 83 ++--
optimum_benchmark/backends/pytorch/backend.py | 25 +-
optimum_benchmark/backends/pytorch/config.py | 4 +-
.../benchmarks/inference/benchmark.py | 36 +-
optimum_benchmark/experiment.py | 11 +-
.../launchers/inline/__init__.py | 0
optimum_benchmark/launchers/inline/config.py | 12 +
.../launchers/inline/launcher.py | 20 +
.../launchers/process/launcher.py | 8 +-
optimum_benchmark/trackers/memory.py | 8 +-
119 files changed, 744 insertions(+), 4298 deletions(-)
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/forward_latency_plot.png
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/forward_memory_plot.png
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/full_report.csv
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/generate_memory_plot.png
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/generate_throughput_plot.png
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/rich_table.svg
delete mode 100644 examples/running-llamas/artifacts/A100-80GB/short_report.csv
create mode 100644 examples/running-llamas/artifacts/Llama-13b/decode_throughput_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/decode_throughput_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/full_report.csv
create mode 100644 examples/running-llamas/artifacts/Llama-13b/generate_max_memory_allocated_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/generate_max_memory_allocated_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/generate_max_memory_reserved_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/generate_max_memory_reserved_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/peak_decode_throughput_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/prefill_latency_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/prefill_latency_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-13b/short_report.csv
create mode 100644 examples/running-llamas/artifacts/Llama-65b/decode_throughput_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/decode_throughput_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/full_report.csv
create mode 100644 examples/running-llamas/artifacts/Llama-65b/generate_max_memory_allocated_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/generate_max_memory_allocated_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/generate_max_memory_reserved_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/generate_max_memory_reserved_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/peak_decode_throughput_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/prefill_latency_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/prefill_latency_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-65b/short_report.csv
create mode 100644 examples/running-llamas/artifacts/Llama-7b/decode_throughput_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/decode_throughput_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/full_report.csv
create mode 100644 examples/running-llamas/artifacts/Llama-7b/generate_max_memory_allocated_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/generate_max_memory_allocated_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/generate_max_memory_reserved_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/generate_max_memory_reserved_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/peak_decode_throughput_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/prefill_latency_bar_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/prefill_latency_line_plot.png
create mode 100644 examples/running-llamas/artifacts/Llama-7b/short_report.csv
delete mode 100644 examples/running-llamas/configs/bnb.yaml
create mode 100644 examples/running-llamas/configs/fp16+bt+tp=2.yaml
create mode 100644 examples/running-llamas/configs/fp16+bt.yaml
create mode 100644 examples/running-llamas/configs/fp16+dp=2.yaml
create mode 100644 examples/running-llamas/configs/fp16+fa2+tp=2.yaml
create mode 100644 examples/running-llamas/configs/fp16+fa2.yaml
create mode 100644 examples/running-llamas/configs/fp16+gptq+exllamav1+dp=2.yaml
create mode 100644 examples/running-llamas/configs/fp16+gptq+exllamav1.yaml
create mode 100644 examples/running-llamas/configs/fp16+gptq+exllamav2+dp=2.yaml
create mode 100644 examples/running-llamas/configs/fp16+gptq+exllamav2.yaml
create mode 100644 examples/running-llamas/configs/fp16+tp=2.yaml
create mode 100644 examples/running-llamas/configs/fp16.yaml
delete mode 100644 examples/running-llamas/configs/gptq.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(1)-sequence_length(256)-new_tokens(512)/0/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(1)-sequence_length(256)-new_tokens(512)/0/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(1)-sequence_length(256)-new_tokens(512)/0/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(1)-sequence_length(256)-new_tokens(512)/0/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(1)-sequence_length(256)-new_tokens(512)/0/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(16)-sequence_length(256)-new_tokens(512)/4/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(16)-sequence_length(256)-new_tokens(512)/4/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(16)-sequence_length(256)-new_tokens(512)/4/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(16)-sequence_length(256)-new_tokens(512)/4/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(16)-sequence_length(256)-new_tokens(512)/4/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(2)-sequence_length(256)-new_tokens(512)/1/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(2)-sequence_length(256)-new_tokens(512)/1/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(2)-sequence_length(256)-new_tokens(512)/1/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(2)-sequence_length(256)-new_tokens(512)/1/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(2)-sequence_length(256)-new_tokens(512)/1/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(2)-sequence_length(256)-new_tokens(512)/multirun.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(4)-sequence_length(256)-new_tokens(512)/2/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(4)-sequence_length(256)-new_tokens(512)/2/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(4)-sequence_length(256)-new_tokens(512)/2/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(4)-sequence_length(256)-new_tokens(512)/2/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(4)-sequence_length(256)-new_tokens(512)/2/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(8)-sequence_length(256)-new_tokens(512)/3/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(8)-sequence_length(256)-new_tokens(512)/3/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(8)-sequence_length(256)-new_tokens(512)/3/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(8)-sequence_length(256)-new_tokens(512)/3/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/fp16-batch_size(8)-sequence_length(256)-new_tokens(512)/3/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(1)-sequence_length(256)-new_tokens(512)/0/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(1)-sequence_length(256)-new_tokens(512)/0/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(1)-sequence_length(256)-new_tokens(512)/0/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(1)-sequence_length(256)-new_tokens(512)/0/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(1)-sequence_length(256)-new_tokens(512)/0/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(16)-sequence_length(256)-new_tokens(512)/4/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(16)-sequence_length(256)-new_tokens(512)/4/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(16)-sequence_length(256)-new_tokens(512)/4/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(16)-sequence_length(256)-new_tokens(512)/4/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(16)-sequence_length(256)-new_tokens(512)/4/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(2)-sequence_length(256)-new_tokens(512)/1/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(2)-sequence_length(256)-new_tokens(512)/1/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(2)-sequence_length(256)-new_tokens(512)/1/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(2)-sequence_length(256)-new_tokens(512)/1/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(2)-sequence_length(256)-new_tokens(512)/1/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(2)-sequence_length(256)-new_tokens(512)/multirun.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(4)-sequence_length(256)-new_tokens(512)/2/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(4)-sequence_length(256)-new_tokens(512)/2/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(4)-sequence_length(256)-new_tokens(512)/2/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(4)-sequence_length(256)-new_tokens(512)/2/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(4)-sequence_length(256)-new_tokens(512)/2/inference_results.csv
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(8)-sequence_length(256)-new_tokens(512)/3/.hydra/config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(8)-sequence_length(256)-new_tokens(512)/3/.hydra/hydra.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(8)-sequence_length(256)-new_tokens(512)/3/.hydra/overrides.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(8)-sequence_length(256)-new_tokens(512)/3/hydra_config.yaml
delete mode 100644 examples/running-llamas/experiments/A100-80GB/gptq-batch_size(8)-sequence_length(256)-new_tokens(512)/3/inference_results.csv
create mode 100644 optimum_benchmark/launchers/inline/__init__.py
create mode 100644 optimum_benchmark/launchers/inline/config.py
create mode 100644 optimum_benchmark/launchers/inline/launcher.py
diff --git a/.gitignore b/.gitignore
index a589733f..30063812 100644
--- a/.gitignore
+++ b/.gitignore
@@ -168,6 +168,4 @@ data/
version.txt
actions-runner/
-experiments/
-examples/
-results/
\ No newline at end of file
+experiments/
\ No newline at end of file
diff --git a/examples/running-llamas/README.md b/examples/running-llamas/README.md
index dca82a4e..f4c9036c 100644
--- a/examples/running-llamas/README.md
+++ b/examples/running-llamas/README.md
@@ -1,61 +1,66 @@
-# Optimum-Benchmark x LLaMAs x GPTQ
+# Optimum-Benchmark x LLaMA
A set of benchmarks on Meta's LLaMA2's inference.
## Setup
-You will need to install these quantization packages:
+You will need to install any necessary third-party libraries like `deepspeed` or `auto-gptq` depending on the hardware and benchmarks you want to run.
-```bash
-pip install auto-gptq # or install it from source
-```
+For example running FlashAttentionV2 on two devices with Tensor Parallelism (i.e. `fp16+fa2+tp=2`) will require: `deepspeed` and `flash-attn`
## Running
-Then run these commands from this directory:
+Then run the benchmarks from this directory with:
```bash
-optimum-benchmark --config-dir configs/ --config-name _base_ --multirun
-optimum-benchmark --config-dir configs/ --config-name gptq --multirun
+optimum-benchmark --config-dir configs/ --config-name fp16 --multirun
+optimum-benchmark --config-dir configs/ --config-name fp16+fa2+tp=2 --multirun
+[...]
```
-This will create a folder called `experiments` with the results of the benchmarks with an inference `batch_size` ranging from 1 to 16 and an input `sequence_length` (prompt size) of 256.
+This will create a folder called `experiments` with the results of the benchmarks with an inference `batch_size` ranging from 1 to 128 and an input `sequence_length` (prompt size) of 256.
## Reporting
-To create a report run:
+To create a report for 7B models on A100-80GB, run:
```bash
-python report.py -e experiments -m allocated
+python report.py -e experiments/hf-dgx-01/NousResearch/Llama-2-7b-hf/ experiments/hf-dgx-01/TheBloke/LLaMa-7B-GPTQ/ -r artifacts/Llama-7b/
+python report.py -e experiments/hf-dgx-01/NousResearch/Llama-2-13b-hf/ experiments/hf-dgx-01/TheBloke/LLaMa-13B-GPTQ/ -r artifacts/Llama-13b/
+python report.py -e experiments/hf-dgx-01/NousResearch/Llama-2-65b-hf/ experiments/hf-dgx-01/TheBloke/LLaMa-65B-GPTQ/ -r artifacts/Llama-65b/
```
-Which will create some quick reporting artifacts like a `full_report.csv`, `short_report.csv`, some plots and a `rich_table.svg`.
-
-`-e` is the experiments folder from which to read the results.
-`-r` is the report folder to which to write the resulting artifacts.
-`-m` is the memory type to use for the reporting. It can be `used`, `allocated` or `reserved`.
+Which will create some quick reporting artifacts like a `full_report.csv`, `short_report.csv`, and some interesting analysis plots.
## Results
-### On A100-80GB
+### LLaMA-7B on A100-80GB
-
+
-
+
+### LLaMA-13B on A100-80GB
+
-
+
-
+
+### LLaMA-65B on A100-80GB
+
-
+
+
+
+
+
\ No newline at end of file
diff --git a/examples/running-llamas/artifacts/A100-80GB/forward_latency_plot.png b/examples/running-llamas/artifacts/A100-80GB/forward_latency_plot.png
deleted file mode 100644
index 4e7dd369a376ab509811c02853f15fc09153d0c7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 39612
zcmeFZbySsW`#-n|1rbCEX%Ll0x+N4u5s;Qn=|)13RtZ4?r8@;gxx3{qq
zU}d%ZKOewiZEMWRonM>@FLJ|1Ld_0=z|}+kgYr`(-4ua{W_bDRsghId`lO?(Qp<4d
z&empM=CzFb;_EoinsWT;54A(inc2;x-#mErYTb`i-=f##4KqWtcJlN5rn};=l=5_m
zDs-=*|Je9@6cliFJzwtK>uJ|`;{=1vE+NNvL=z1iaU@~M^jM;p)Lyrfd*o`?uOojF
z{Y)f<_vGIXqR;>TA3py-E<#<7R)t)K()w8Onjx<^0b7;JQFB*U*ZlDJ1U>447qEo!
zx;wK?Kb3yb3H#tuT^7U~^$9t9T%If?2)0Pd$QZ4U-&BF%x0xu_%gkJXRliXpMc<5{+p(}($aK`(SrT;(GWC@Ta}?a
zH@+uCRl#lCVI&iA7Zes2jtYF8qj^G8oTiI>qlvP&2rjFM;{~2HM1E1xd<&Ha`n?Aa
zKHVf@VYB`#SYKcN+26l@a#HBUjupIZ+1t78!^7sOsVVeF?j?1fFgT67$j6G!TVkxM
z{i32sjJsm>_7*#jPfryp-dQ3@c`W`k_~Pd0!qf#FR-i`HXs(f&}&Meca3QJy}vN
z6VL4IN8=}Vg(!+731>tTapzr;G@k3@cvDND;mhvNjO*=PEp99^mUhRqN46oOfj*}1uLyCs=4eHj@UgvC^K+}%g&(`#$(aBSWm
zZ&OdXpFKKQ9i02~{ehE*cB(0Dj8-nUh>Gu$jD+O@59kw7u-0R7P$QhE5G3^
zB)tft#3p9*biX*2&(Y+k6n2#ep?y_myMXlx1Gjl{GET?+#9QR*yx(L?OH>q9lZglr_vNsCBG80e}rvev%f44C*El;fcJRn>ayi{XHKo!^?llr;G;(YuQQcWj4l&qc{w)^z`(;l5qNb{fgZjL}mSxSGN`cH(WYF*o|1wiLJM{wn?Mgm4<2(c9PCNJ4hO$%Ob{roc
z-zFn#KbrE8O?e)GE!ITHsHA&-yz`Y(C>pk6f8tX##OhG?&Ds*nag)j4+*Vdr*d!e2
zus3Dm1e%%y$VfO0B+`y(9zOI+OuUyS6W4aQF@CZbV@+0I}?k=kCLR*;q
zO5ZgyvY4gn!xzYc#aeS7yfU)T9&vKM*Ud>22=|BvE+N@`IHz*2>+wXjTa0zhMYsL<7QqvKht<$zdjA5WPSR^~_a8jKIS393DQv{L
zaYNqK&5e^{Z+BM#LBq_<+`mJ@X+-Z?@TLi?As@ez`%Ht{<(UvU!g=4Vu&mTEg
zS62d7R@Mb%If_5qayk9^>M9_E-XHJw?L_qJ*M*O`57tK&Q9|Jfj1&U)a&YI(jww!Z
zVQqx>?3|CclkkX%f0k8LF!5UdRU#0VXVLoo!-R`;p`Ep+L+J>%ynj@bTKJqO0Xhdd
zb0cR*6sND{c!{AiVxsbW$mh?P&PSWCpwVIUvkKS8A?4ED
z`dfuSo?a+~r$<{sAt50%$528T6n;pgwbETx9j(c;8p3`mgT219vlEpwGdCwDAt8~b
zAH!o=-`N?^75A7daCBs(r^e&TqK3CSrPlpvjpNI+Eo4nMY!8nWav75A
z?(4%VR=;|kIA~Vxa>VlP-MdgG6&8(z{@#=qeu05T%RNbI&aIbc@xr^uTTh`S!t%`8
zW#%^+V`-%yRNVj>Mgm5do0v`GM~obEVWNE=_&BI
zDB+t>2iigzLKX^dC*S)=|jMW*N3j?9;~8ym}DUQ_JxN_xUElo>v^Q?8Yjm2Qjvxvm6}pk(^5q}=}UDbI!SyP(u<
zZ*Q}ON`*0ot(?F{MJ3+Ma0%@ZvZi$`wVEUru&0*_XV%+a?rm&oDSSKk97@e$}3rleci!XMpS-f}_bbKE3HZXq9PDi6!6u(+f38ZJ_ty1Kfkz+#KhU=G7Jwl|HK
z%DLJ&%OQJFoW@Ahbccc>ZpFQFw&{z=k-(^%?9+r$;onPp-{CTh%9nOt4w!Lo)m#Z9
z|AFddI92V|5yKnQ8N=IO<>J^tCEdMyaWsXzE!9_-E)7|FJsx-cljYK+qv+y~rjnef
zv9Mkuyq%uLbLkgC*F&t27WzV?QK)iuXfW_oD>V8Z8Tkk9`N?iOE6$Q;nau~bxo(&4Sag#EMl|bXlTDiCu(h(>2AXO
zuPdG2j+IzSjh9%OPL#1%IBqTOF0}i&TuYemNfLwpUAc*&Si*zMB
z;>@~_8o$uqNW5LXz5Hbn3W{3%W7r)4O-9uhQeCrHzdVO6QVD<@Asq*)GM&ZY@>S+t_qq)s}s;Y#L
zo2wwSeXh#uk$O>WYpY*w^{GEp)upl{YuG4JZ@S}!$n5OwqPa{_pA6?{^_mq+N3v13
zmy?i?sQty&=W{t^MuI=MDL(q70krMrI_0*8;b*Iq7f<(`Dt@Wt20}dpEcRztQ^rL*
zftHqb(eH8OkJMD4CAS954ixd7vz4V_6<Aqg{i
zT1cLPq!@0d+~z~rH*ek?D>7~HL_w2~mNvLNKLOk#!Y=O|45ydf<783Q9`SfVeR0RL2$=
zwkrToXH?0N2EYZS=oWzNbO0awYs0uYIy!Ik^>5(fE&*kV8v50K3j+fqO*WzH@m8f+
zXEZm?8preJ&x2!QJ7h&J41SthS;@fKL9CnKx(joS?jKFwx
zJT3@(jwG~y+xx+}P3m9aIH)hNnvC9=Yw1syr2=r&4nM+yc^U3P3;^}j_xs|Jyw;Sk
zAxFl>@CXRJ;HKIwb)~+F;W<7#Lq2C>VglI)=w#y!+QT1}yW<3DR=f*(i
zHYe9vGPLitvl+px(!auw(I&Ybii5JX3NJ2y8ptzd-!bD$y)CC+YLyKKno-Jy@~Sevl+(k
zjIV#uC%EqV+0HfNh{YMZ1G{u&t{XDTTvksSQ>It-`-Py5+!*^3&g
z?%;3V5`hIP0$h`q3}?=WjEP}_?Ial|;1GJUw`8ntwYbxLE`LrZMCsuMoKAUlI5!tI
z8zb_?fWw8s`|0q*9_AdUd*=Q_f#AVG?@j7wDEK5Ke#nY?bKo+d(8;Cb#qR^Pw^aznyEYT3-Zlu=n5dKpXV_I)Mfc62+AN^Y79B%MX7ZeiQ6j
zu?MvY&Bf&`eKUjssVb^`Nu~#=jBs>x99XR%C8E8dONo4ltZaA0BsNlSoUaFL$Dmd)
z-)_|`MGb$3r9?ZWx`%<`8!9t-
z8uX5tZjq}WFGFbi@L@@3!|sJVk2?Vi71k9>Qx$Koe#4`t%7+
zB}emrRJ4dma=6dMD@8W%MIJVPjJxSHc&&KRu=LjxP~hg7*xA|7w>+-&uP!fO%i$9d
z9TmicK7wKjBwj&Lkp_i8^s}*_I<8)VXnnwy$l~;^&YF+@mO9!MSi~d=u1%&6STjVM
zyDp6m^{f-(nJ(;NpJ!%fpgqBH3|lA%SrDI?_;YG14fK6k!7C75xJ-Hwzzqb2gg`Qo
z3>RQh$@$@Wva715CZ8%v$YVLSKk@EWf^b|6uQlGCJ9VRl#)TKo)#tlXP1KW2x)N8c(?`Kzho_r4pKbva_+5`Hm_(S3ja5~&tsJAPkzwK8<;ZUpQ>DJ+|NyqR6SXZC;Ky`=)X(Bi{7^E46z?=tOx9>;T
ztGjhm$*wNTAqc&2e0Q@A_(m|5{3pWV&7nwuV2DRW74+$oSlSW@1E3ZVfGSek+)Ue&
z-ykz#FrE9N;G9%YUBYNnWAIftIsgl(Rn1Tb8bAR02_+ORu|vTc)a5LV(!kH3KTEeP
zb;Y$CC%8SAmc|C`jliZ7j)jl(_H^GA0VVdif&v~HuN7CU3dp(V7Z*`Ln(@aMHa0qe
z_24W3GR$|{c>@C7l-m)-+HkHhz@f78a(2Tu0s+U(SejoxfJ%;Vkz9SF2)o2yGVvAF5!Y3U_zFR$B_l$Qf#9#hxMBC9EAvjdh5+-nTOG)-D18sef7xCDDOy6uq*p6=2vX+)h{dCUP>k;5;^_xy
z$O?vFqtraX?Jrg$ucGOGOK^{t+M7|o)F{(^;P|!$bHSB7t7@KS>0jrYo{OC^_*7Kg
z@cO_f-j;L)`T5ngw9xgl6A=+L*a4)ap`~rMS+8+;^X5%FX(?!cs_N=*Kr_$GG*4TC
zZB-8#39S~`%1DVN8B*4S8=+M)<_c88UVs8Pf&mZ?*f+bN6ZC^p0jhF1LlmJXrXK1$
zmhRk;5a-#b9~g25wu>Dkw=FWy4j^6J2zA(>>xO>f`#s|Bn6>%
zeP}=XS0rR*jRE+1d3njf)(B_OY6HO)bZp}s?N3nJEffgK?82{VdRJa5O;0E3;60y@
z+DBAB8CoPwZuS){{hPGxvah{=WViNcQ!zwcxT(Nla$P-77X>hM6s;S*|3@evg~r{@
zy}di-?0}Q}0y-zeuYR8YT-r2spcJ|SaD)`k8tj5(McH|Hc`q(P37M3+$eBPv2WHj%
zO4u7J!_!$y)hm=(oU`tygi&;#&SQvq5}5Iye0}DO(76{T=^fxbYDb0B7FyfcyPDyF
zklL7?bNOirr(wl^)A9mr7h6>>TJ79VfL
z2MQ!lTNu*|khXCg-mQ;(U#SE@MB=;)CEq6@Az?<(=y1K}^(}I~Z>gF|qTUw8M2T>6
zJgYJGzq=f5z6IR|kWc}|3+Pd|VsCsZci3-DXhs#K6^%%!o5_C}cx$ugz*r5V=kpJY
zZNeY&nZ*5R_t89rc8h90Ab%uvIK-=1A1izN;Am?q-Fm88&G{6VK;y@2H=T|q9r;1m
z@bmL)Y;4@#o)K$co>eJ-s?I>q?}bvpinPn&5uQfTUfpS3tO(ylo924s3+a$T9`G
zNj^|9@TLI!%=l^+y8(h9_$mJhP*@>Q3)nTGrX#1Y0239C$bAA5502&G-d;mjS0Ld2
zz8tN`KG-DnfG9knNj-S}H4!R*<>8olxmgJ8lynKHkB{JUik}wP(JUXMm0ea~t9lO=
z;D63wxIYX+KH(F;prGzuO$CMEU1*E+FJHba53D&k#z{SF-0&YxC&LLE!!Kou37YyN
zRhaM#s+T07?S$hADCY27P+v*8&3+_yOK}P%r=;K0rBS?Tp7$9
z#viW)t3fR8_}FOy^mJuVgp&!Fl%-dCl0HE-3R@~!QKP&dKL^F54V3awD1=K4@vu2v
zhA-tqqfq=gnx9@_l4a1F;_8^-#~)m1ss0i0e6QQ)fS~GKCAcL%H8=E$Aa*1B=YnF#
zvXI9~oMh@75Z|HYTpIGy+*HGm!=%1U?ewfktIKIkG^zVbA<%Jcs#z^F`nW=N2qlzy
zS|$E@cCyUMENcY`68}KC!8E^KTP${Jn|iCqjC8~x?$0rYV;!UP?%>0304S$7HUh%J
z2%kQEN)BR_#uBh<>@dl&R9^{sg6Bb$0WmRTLqkL87#Mb&6a4r^nt%i8y$G%ao}=!O
zE{BLnVJvT>*=z3)%v(cC;-K+BuO{rAVqJ%LIS@K8X>@(g&W@+GbQCiRcwz{
zFsZSMe^lh1Kf1tUHT9c*jeC9bliB3-jZ8%o2dp`sXF_&FS#QabdsgSWxb^?=-k2^j
z?Oz_wrE)vnPXdAp9Ud15!d({4;{m%O^+&MlX@MX&3=i}6+xaK6={I8wU!B_lj(Y(J
z{sby%`P;ec0KqtSGVj28V&&P5AGBA*{>3!2!=%Kh;(792K~`u%Zoi6I$m1#%!8@nO
zN8r|tj>F=*D~=i&9^MRunsb99Q#td-ty}JkfAaxfw2Y2MRlA)Ty=M^+h>nbmlxC-(
zptymBy^r(z@f?93}abn2Tj$oI!l4t#|jNg_0G3wTe%C!J?Ucbgu5d+fLg
zr!S$*aUBG-1g9?Gv`tV2u-B@DHhbd>`A36zxVggy29(73si~?Xf1gX;0@?p``b_noqcy7D6rPJmh)th-=taqZ}wVK8D3Tl55&q
z_(J&vxFQ#6C+AV%MJ@;m|2S=tsXkv?{|W*y_#Eg(K6@Yw;GES8GoGEDiR4p2>A1(m
zH6SM=A@S`Y?D=Oi4p$smZbUij!x2nHA9xDm`5KO15{S>FUqnQfop70w2#{xWNJz9g
zB~|DWDxV}{^wigXtgWq;c4=9)7l2BwH}d$frY(ZC8<;)khA}jE4Q-bVZm9O~nLvES
zP*E9{W(O?D=d?YIua636e|CEM+Y0iI5D56Aes9;+c5~Y`*2CCXd@`(45O+TLN_rtK
z;ZkqihrW-)o{M3tqbrD%b1QqbOu&e8(ZQkU4L~zPiYE`+h=e>5Z~*pK2fsjllFL+n
zWYiJ)6z~@8N|Z$7?sz9BC!}ZrX!oaPxoz4W)kgL9)Ep=+3tw2P6Q!d#z#?Na8=!}t
zWccz9T+*y*@KzmFl8tS4^*RYUMYZo6;n7?9Tjvx$kAkL$J}EJG47d&zpbD)yB`IW>
zXpaV@5o#F{G2^%M-p~czxyxtMI@
z6Zh_G1lP`EH;zkQAP6Tof_}ELsHR1*`gl?yz0@+-6Ap04$w8$9MBHEpW%mu_BV>gf
z;s4c!jubPLz?+Vbj3ic2P#CFpdknS$xBQwNwV6=IPi&E5pFy{St2lWzkBLSg?b
zMP`GH?RMa4(6X`JmXc~u|I7LwynJL70O}ghMOZ!>sW^~;6hM1MF0n!?7Wgl1%H!bA
zAA>xTp<>0{m-6GW%q2)!2yZbqV0XZ
z>AtuzMCT!77WSqAac!(gKWdH}{Le8OJ
zh4xpwxyZ!w?R30LO-t(oWJoI$D*-YAP%qq;ASxub0|&_$DOV_708F*pTYLr1R13V^
zA0TumutZV(cC^3;xC9a4dm7MR0U<+>^VxX9EkrKjT#yJ-2}Rnc-~gn7h6M`e!m=Bo
z=x0~DMf462oLe2e5$2URXosFJbW6mtgq#DyIgjw(KcakLZns$VI_@P!rmiLR19euZ
zY7$hI+XUD83VY3e^w%%SPG^wK{=B!YxNR9*HKS`6LuXa7#|0nE{d^#x7(YfHS`lV6N}v+#BL7TmQwC0lT~?0_(PPo)3c(PQF^f%SW0J%M0EwZuv*
zTkj<12k0=<^A;>mId`wCju<^Y1(ZjxQ!NMpWG=h(?;9}Kz~x7M%@guxd)o$lZut1#
zGpZ62m^y9;gh=2BjtQ7s#_9@7@^MR+4W&Rz0q;aJoSfw@759#fMT6a8wC3>a*)ya-
zgtWjHF8d(Iafh6|1MCH)YCs^kGrZqkp*fbx8exfIS#zxdZ}irkx7>ksQbD#0+8kzt
z*rp2MxY|b{t>n3wkkCxw
zoA`Y?-#u}Ctiy{^sxa<+#U9JfQ7dt8;zF}L;_Hh$=E~SV8@^3*1fN%l0>l>QDD6FJ
z7V2KO$osLkw^mgyDZ<3}c(^kSNg$VdsE2X}Ww&@aIa4%Bt&rCbqP+2A8SPiGeBU8J
zLfR%TFCWTyF#oEt=4|sZ(ECVk^Sdq}qwi0^Bf%7XAS6TqDG?<8hLdDB5Z>Y&cC}Vh
z6$^@0uF%*1tvMH0@9OnCm=sH5JAJT#G2cX&{V@;YhJ%WVe$6&F2EMlGSmd#uZTgPB
zEv}GJz)PTU>R@9tEPL!U>*mRLv6rgDdm79J4h=-P&>WAhsD3FET$7TL0xLCcp}L&z
zt%HNxY|f*gco*fk7u|4~6;QJVwrK0Cd+
z85pH@=rP|O%79i2VUT#fc~{`YD&|9FP~3pJUwpH!=aw_wFLWVuM~_~ovwAGkw7c_5
ze?m&KVNA1Oq=axqy@7MQ6xWYP%Ewbj+zf~7YIUl9u7>X_JVutsc{k$w_c{Ri9Z1$~
zN)-10igI06+-y$~2zsZ-0FIrzySqv7LFdSP|E1<0Y*0y=@m+V0N33}@Bp0tCeQ
z>Xewn023lMu6v!lvBGZA;1D56&~_aLm0)2&gAYy$yeRnLULaf8&i}as3MJmG|8CF{R0LTWOf&$2bJO46)9eA_
zg_RZu)h92%_3|P1j#>YT{r<*y1aPA~mm{kNw~w-W%S+QB3`|tHM8IW{9H!odhH+`>O-4X<>^?x^JFksT)N*#&o&r=1<>-^;DU|nIMr;9rzFmS@6(hS
z-S7);>D)DfHXjs3W`l27>6B_fOqinTosRBk_mlNGjLqiLB|86zW_8
zq&T_Cm={Ua1j_DxNAGzCI=+pq?JURyh>_9JB;|HiSv{}>!6Kc9-!(QhB@LRP7`%3+
z_A(Xx?0rC0Tb_Mduw~wJ`S@ZHW%LI<*3BcW+04%Lm;I|YZ}SYCc|BIc-*L#NG_^mx
zk$3}82UhJjbpLuf7M8$~5mo3Bc|}E`+|=H;n;lPaRYfp^w>~!GMF)OTN4tj+cgqn^
z@x;mS;bUlF_sTxrR1{E2jkG+Z`||l*9PbIfLWUBciSIxw0TDt>r(tYN-T4C+3b+qa
zQev5d@am!v8v<(GG$D9KmU)x9ku#OHKKzc&ys_J8DwcgTZWo~~fl>G9xL3~Sd6_qi
zca6-R7yqi&D6X2X_~i-J?V3}Os*RHqA0Vthi%i~MK*xbNU@912tQ8ERpFhs7PMt1j
zZEB$_$vNJxyyTBmGkW|?-6znxB!1>hhzNy#?M+J-B}%B{qu9s#1U!IR?!hUq>+TNP
z-?x8>xJd$F`b0_{Q;&QMdt|TpGRdSa>7jijhA@sT0qv;_o>(}=!bq_b;_ojl|89-f
zjcuBTysXR*QCvSDjqP`NNU!ICBP)`|UQoG}OA?vz6J~uecRi&_Rk~wLng-(Vqk)D=lWgCIGM}p1>ri3i2)C&_KoL}}`@{wMl_QE0@_
zJ1hvxU*&s-zHNM}t+q;{|MK~>gKsZByC=6y+MU{N93{3Avz4~VM_S6~yH+v=m(AN-
zFmru1*YNB5Kw~J5s`sB;wo|ZDv^~r7PU6`{
zd8=cU($o6JXo$AKe`eYa$7BsUXi0-;r?yRpJ%S8NtYRQ&=-KTNpZ6SJh195DBwcNi(|M?D(exjj*;HRBCY(FmU~ct!
zdO&l6&mV5>jEG>>kufo$2fp%vp8hjf+sLpmzyR>Zx)f?u^WV50jv52dp@hT5V>KZG
ziCp*-kiL0n1qR=y1I0eLhGB?j^yhUmdS^^{GPb&E3(o6%zvV$t?jLif0
z?s22JNO97>%My(-+oUxqqQ_Q(I4Enyol*FV&*0M`vls~QGG%3Dk<1j*oP~rbkZJ`Q
z?tcTH7I8{hTUN$5l&ukw$peYp>E&e!h|fV{yxbp}OR}eb1*$nl{X)(D!ftj_Cv)DT
z_!PF686;n|Jc7e~b5Z1~lO)bBh0cbLQGI_*%1DUqJQ+lYwkokpx67=9jWShkhmoxeIa;Z-ITjWcz+$Hv
zOHF_dW@es(KMdw6L_h=6UR~9USHbGS)bhH^g
z-NT2(GlMrkfMcWl>nkGcg`at_5L}!%cO=)r`$ErNz}x#Q`Gj$uYeoGU?+d0h$G!9t
z>n=PA>Udh6ho0qz*49W22^AS=?Wdj|IS2?NGieA2D)A0vgPeh^&=75gA(StWu`}@F
z1u+Q-$Wd}MD@b)BT&xcJY0RbAW*!&P}+u(WQ85M?aiex3e{
zAU~f?JX4%Hj1})c$qok+08xQ0&CRo`tEU5dkQW5Vjex`pIAjdSHD62?@bp
z>Q3-@gUSa~SX4|5XTi%09SmT*9e<>HMof~Cr!VNWwk4mgYI?EDLW5>UZgw;}?pk*L
z<6}Ql)#wb8!Qc0-uZM|-eTsYP^(WBvJ3?Jm^)@l_AMmh{A@{@t^a8lk!jMk2Xo2in
zl#dOSMx^i$^d1K&ZWGAV5|Wb1vI)Wn7_GsF$TECV
z^krXj^MRjFSNc*RX>ABuV5G(mA$=mx#>Qu`)Gq)H+c-ENQz)*xt#m*^4Ap6$Ljn9F
zn*QmNo!%K=Puj--{Gu3~1@(Ep0q=1?Y;EVz5Z?Wxu%oQGra$3?c^rVZ?}^SotA2^7
za9C>wqJq?ufH}Z|}v>p4M;e(@7@vWhVb{^l7sgGJi+@=ZQ@&wA1
z)y?4UQVM`upd6_${d`{~)aDzW7PvTE
zJj11{A@>zHzbDP`eW|=Dms!*@wKn;Gz0kP4l4=Os^Oz}7LF`jd
zVlQ1Hd$&Asw#-aL+)`Bj9JzYjM`Lz#BM01jiAwVL+?t=$L;mYI!2m|N?_5Ts(8~>w
z37jKRKiZP78l8*zuzZ=;-4{ZPom6lpxfzmY*@i#E(qsOwa|Bg5kPhvCiqt=~t<~sp
z#Ra;qa)7{*SYo=8jKe5*c3&Uq^Y&B|67lr2YGs!f|VFX%02d|1FI2p
z3tFInN1#QV%myj`FkGhBvu-WQ88neg^YSBQdA9j4o0GgP>4wm2m|4y}Eiiw*y#FAS
zK4MPpD7f#~V355ep>Ip$O1&JoN~ve*U*y<(<`#1O+5XNl31p5zkxTl#soic^|Nd(q?3)
zP0b5U*o$-$(w((`+rG9^WcK~~74vqS+3UtX0GO;RRe5a6|XOinxo4!%chYs;ig8so5toWjbBOG|QScf9wn4ZEEkpe?Y5)}Fq
z7uMZ_a883|f(eT3$X+2&@Wa238&PZD`|sVDhi|RXIZsq1vJO+A=LVq`BHxUO+8bG_
zk0vM(>Pj$J+VYEUERw9SNwm`D-`=GsE+WZ0A{)6Nq*u4u?wML!8mCv8c^6;Z^n&R@
zc}%EJSa~$NC1Y<@S~_4JEcAbGHScl6QV%G4!`iLfmd@k=N0q4a?au-1MN)B|Q
zBbki6%
zBQsvRj%1Oek)9?iw?D&`&-=GM?t7HoNz(bd(52?aZ<6S)XKc4d?%US08d(K=`CY77
zx~IxpG9)&cD*P-T!wi<>zwMjL6L$y=U>5T%%nM?v>?AOzqW7KJ=U}1Lcbc9X@j)tfE6!U(YiZP4Kq#?LDW^;_Q72i|K#odODX9I*XSZ_
zimt8(XsArU%!F3o=w70xEBjx*LeF$P9;f5exF4}l_{FAg-lD`Fzs=rRNI=k^~V
zqC*{xB$s{>looO+fARh^{BL9J*fx>-4qi+jGPsp5K86CPIx^2VQN8C#Tq5{U16LuR
zO>s}H9cw>p&YO|w=oXf|+`pSPyU??8&z0w#AEn4tR|9p-4Z|>Ifl(8R4zC8xFT~%Z
z$2#U8-?0orFqZKXm(PgKd;GVSJF~fUV~#U@+HU2VOi;Nk>>NG*E>yfkw&g$OJBaA(JeV-Jb+STLLF3fU9n
zbSMI;$j#2yL$GY+Z`GOI-2atN*oNdoBJ}%rEDv{-vzluJ^out^LvJN5$4c*K=oq*A
zV5-p_oTRu-|E9BJ^KxLjYjVgY*(P{v%e4>fe|&jx&hfdDU
zi}ZhCz^ccrxU8(qKwuL-Gt$&|)&}N8NlE!5EiJj%|E;(3!ox0N=il1sU3yQXa!COJlK<@bX*bb4tuqxdP2qKe)0wxxZR!e9z$)
zG_-`PYW*qX38Q`U;4s(r_lLmkFM#og+1c416Rubhk9ZgyoQ82V$k*`OOnb7kUOa=m
zGB`rpkVM{Fh|s|!C2fP*cUubU#?N=)PLUxa;N05!-Y|wGQHE0U&MR-`bUs&_lbASu
zZJw1zt9|5vZsn|B1Pp?dx1bUoeII{H@;&$-#}s3C}DP39ngcZ1TM=l0z11s
zW(O9X>JBiUp2G;h$$DWo;)idi-JrG4-RrGb^tlLMJX7W5(&BK
z<mt<39^|8kQ|rX|DSCiate
zK`<%({I>BMh8sdz;x1OM{@-K^o_N$~LvkHC5)1Y_a_|ajjyF^o)I^00&`W#
zQrFVr3sh)ka}znMU3irY_mi!H{FeEd*aV?mMv^-{7v-&JI!9DNSxsBg1{Es0^&gko
z>(>oqkJHS#A-aTY`&dFD8|Zj<@A`m)2M#I5{ujRP1d#;rDLLRv1Yk-zI>iZe|2c5<
zkVC#;;X>f_#?70Y(zD2kX|sbXM-y5anw`HcTW^8p!9W88_?yT{aCjEyz{z}wERv&n
zTwQ`bD-UDx5MjtG2=*`mt`Wu2r+Z{@Y%V}AKtFqp!oyRc*p6cNe=D13E@+ETf9*6_
zLnvn`cq2!D;1t-NAHOvl%6bWU3uJ7N(PgB3LJ;AWgd{=FlI+izv)*G^Jzs6Vv=-fK
z)eWogz~C$rFuHgjKk5+=D8Ug>0!T$>t@oSF3vXXhPfgLTqkIhvL=MOxvb3wBK^)9>!9@*D_2xn4`|F``wnh*
z+2-Zjb=rnu^0}vy*WZO3xsE1rY@7>y9nht2ER1AE$~s|HomuE)N4#;MC)J>?^W
zBw;DLQKt9pZruWO8}QSSbDJ>G;SU>ab|&d6(+c^m1t5YcMIeKz%o%>C*W!XMlUXa2
zd$;D+VlN1(C>>{=1wTblEX-~^RAO|>eQWA-J)%i7g!LUQVR9|o=)-UIPps|}iV~=*
zT={TcOuKU>esmU_F8;0?UeUA-C48n@%C#6GLFyDNxBkJC$v&B8%hKDU_ok+vZ=ygJ
zzaA281_vOwMn>{b97=l&>3mAQ_Cod_V_6cIk84yCRwrrA;VebL%$cqa?Rw2%WUiN`
ze3RPt&ri2a&_G};Eed=$;o`t#dZhITp$InfA!dDTZBX{KC#n!P#rD_E7}S6E5&pek
zl5L%eFDx|`)@QBgnrN<(}*W7CFT!2faa?1M12q$Tr^}yx@Q|`mfiSptN&CO1-76
z%^%`}_=JSwpvJ?O4j^j{3^x()?Lzyp6_cOdegg!fFQl&Rj{2)ljB$1p9FN3W2C|Kh
z^}59oZNEp@fp{)x-CVMXxt&WIGJ)m<>Va~OW-E*mmqQ2#YJVdPym@*eAdQFoq7Kqm
zms8#Ho7BkcieIVT#XY_XHJw9K%#Edw`kN}|pQ}R}r&C|>y9-fzmYPM(?kN-Ib97%_
zZGQRqw7SL+Pa^&g?l;JVJ+2V1DbeY(>`{FZ!Y+4KrOCPlU$l@Jq-ngrMJFNCuh7^W
zn=9~|=0|YmcQTe|fBzhn-iY`B;kq4vNvV(72@N7Q4-A+5k=i%Yl7NAzq(DNhP~GVt
zG6ebw!M{KRd3HeW8)~ob{{WzO>p??an#hDuF6;0KLYwN^qH_7I_y4OrZ>ivOq`fDW
zqD+#}D~hTAPbI)Yv%G%JttHZ+5`l8K*`vpD%+uv25m#S1NfcSK!K@x9+L}Dnu}=9U
zQAK{PSOZfJ4~ZL}J#}Oy9lGV2iE*$XNwHt(YU#fqVDNv@=d&|^v*+s*LKyAeHTwO#
z#%p28Pa5$?&KZ8td`v|vip6?e@saVUp>RC;&56_Nhi_X~MpR9R$4l_EP^FUF#H2y|
zHY`Bz86S@+opO(X`=tm&n~GW>CpCsVoce_GX=dQM+h}II`-`CaZwe%~9PsSr4w
zm$kfAr=K+JBS_<6tWVWrbto*hJDmFgHxpwhY+m-WYuo%9f|B3X3&t`EAuNJSPFC(*
zM<#|~Xoq*fPzO`(g#-+wy#yJ(xi)F*z%bpm$5goNb-ROvxV)ngPzFY1JqzO5~%|ky`5u{;{>IPT_K>{)TdiUl|-=CwSn~~_j
z@$p@d1JJgnX02d;AMWA35^n6y%eAaeeCn59D1Z
z-z%71*B?qS7@RRjD6ziwikd{*W;W9Q@%-NlYG7Sx7pNib(NF`
z@P|)tVq)H7WNiEIRcH=>xWA9kpgUL~wH7O6t*z%7rgr$>YVMnKL73FLL?JYukQj&W
z9-3WSi&;5_i4k_r5Hf#n02hF$;q0@%5Vmc3*;&pWD*0<4AFGr>LvUcP1*FykmEVOm`GB3i7*+V1BO=ro*9v
zBOuvk^Y-lwjP0@y8hYQp@9{XFl;_r`P$#z158wQ5k35!bulQkPaw4#fEpkbS^W?pX
z3lgJbeW`+D2vIcYaD=mylMUo1!_i?hEqH5D2GCd~O(
zpUx?T&}{BUNgN!JArY?fD}A;98{zr6McM89H5{g>PLMf&w>i-YU!wpcg;DWjLe51D
ztdG2I=T?KndqT4wa|0w^41r5TeDN*;&!wI~Lbgy>vpz4%&pD0QmC5tB4n-WIDc-lQ
zARCfoCD?eD)HoDUdOoGdtcTm{p~)?
zoED*HDav<18`W2Qk-0KgjDOR(xsAZAs#`q*YCn%Dfp
zb>S;_U;buEb*xrziJYL_oy~XGtA37T)<0I`JsWm2^)a=hf4(~`=gSWpxK2j6nLo*Dqqo7
z@<~q@fxf5@7^r?ie|>v`+uASm*|XC)HeWt}`0(LK^ImVB_zOGsT1p(}s}KME?%3Cb
z^Rc?^UtV8-nkp2)xhaKozO?ue`b^_DzpXIbMCGLTDs?Aei$Ov})By-ndlaP3M^BT+
zer0)SCYF|RPEAt&(`}TzEJU1y5PPpzD6Y5X9%YqsK#J{0pxSV9UHWKt3_&6D*(TAn
zf0Y2ze&2xW)v=HC{x9v#a55C_HwGzt>a>5-lVS}iB;vL`kyM4AzWGGqlfi^5C+RYk
zS|K8TBqm}S11yo!PQpm_jbZB3)>&B-rXo(7We><#2rnU64kq{s0b@Dy9MUX0X6E&-
zCg?HdKg+Pqv0F(FuWp`orwE{7IdkmmSW)7Qs4s^Cz6fmMpDJZfI9XZOb=7peIiW3P
zeYu2s-)8oZWD2Uqt~LOG07MFUsx!*p-`nebB5t3Um{`LEq%yjC(QZ@I5LrFNwGnU%
zbmwZ{5B+<;(ahZZV90TAx>|>RmiK=*i3bl>Hag1(rWAe?QZ*Xmd^J(=&e&4!)tcjY
zhAroO_8HOV!@V5`_8qTzYMHzD=uyIj0pEcl*2rgDNJGBG9P^5hcCrn=qVu2a$E{yc
zx`8k)29hvU8EaTn0D!oYnhH7on~7gTgv%daD7=?X;%2G$Z3rOuSfg5Z#-J^6O~^hX
zyua6aZ9(c|M3&!3!j%toZ+8FA=GfxW9NMc&_2@bdQ|0?i$F5$z`oyTYv(pe_z^Ilggme6$f@4Ml>94@8Sv?TrbB5FhC%;(_x1^?dDN<0
zxx&Hju`INShNTliU7}x%pr@<9Um4gTHiJC6*z>1yDGP3Sp$UE8ud|J*eT&~|R&1Sm
ziN>c}%-eNs9=3R2bUo9OHoKCYtJk4yvaosIWXV;?|;Y0rci&ul(MXZZ%v^tLl
zH-nZ(kkSU~Xft&7KQ00Q&Z5-3I*;gaAUMqjrKgoooW
z7@**HErisd^c9^C{UlaYl`%3u*XC*f1
zqAR>Q-$SdopeE1>D9wD;(k$ZUFvk)0`TU}Sis;RQGz`Wk8G*+_P&?oo7Zw(dfSMl4;o*(~xyo#aT^j#!G_sv1wYy6Lk#5xP>*37T{5E+?%xLI+f+)
zuJ`^9ic8IA|0R^_-YqG|LT3#2N`V~O|M=t_(250_dVp3k<^cy`f7hzL$N
z`tL`PPADr=jQ_lz@&Ua86v!-w&p6
z)tdZ#ZHh!g_F_->!_4>#2wlHiog+cO-Ei@x58apBqXu_HDLMzq+_J7nnaNOC2J{`B
zmJ1&hx~j7Q!Nirpm9J@gl_zj0MSDy&sh}I$Xp=*{f*6Kh|G
ze-~yPdfpZ*AQk3=j+cvkl)#+e*~RhfeYWcle#_76%`+U^R2&dhXdGBC_xY^{>BsPf!2_YO
zh=F7B>6=oOx?iESXRN?yPDx4WTC=jU+Lq&iBhO~2>9D$-%E^(x~aH@=AXy_O8*?i$y<=quzTxSl9SR_Ra&Z%x|U&
z_aNspS1YZKNMgiCJ%`5d@zt?ll_u-c8gVhlcU?D)(5Lk3UVh;
z5d|?Fbj?T^xq(7?IRLW04_Y~d)5S-`nfx{mN>f=6c=e49E>Nj4{g$QRT`Eip@!ZVp
ztU@4oF$-z#ss^s4*?P^H$F#r_8yXc-Og4bE?AJ0H+?wb;6~W(wKQlMUO+CeGdbKz$I(
zV#AJw9MZafg+{~XqfjledERbhk|CR(oD_CAxzF1(^3meH4i`rGS9h9ED6E`6-|tqL
z!J2?_SfkJ==PM(tWFyPA0Q;n_gK?+>ke}hYgLk(IAou%I+*B(cxsitREh!0lwT~f$7Y8W!-GFek9
zYgqR^?s>2isSq`t`HxCR`Kqs?Eq1(!Q2ll{+V|~!Ke}Z?>zFawS#J3{Qh876`&b@6
z8T+t_J*tl%C&ztGqEyeh*po3?bHW0SnA}C8L7aQZUb01;Vs)9E{xJQNPfcLqZ#iY7
zkCHx$^>bL(l%_7c77O&K!Aq_;POfexZTluMakkk458m#zTUd9zDp)%uiTfuQ-m{Vz
zSAT>j+TUqvDEcm|Z1e@#k6*q#H_8&2f_J57$S3fWa3&Z-&wBIZ(e}4vftQc~DbOpN
zI7^}+Cr^{PK8n6?8Fp@tf}GD=nx>ltep#AqJ6z!FnQ1HCyT_rM3tWLOJ-mbI{d-LYnMgE}Fu<%rj
z!r|=QybjW439TQr3zFfrPCPHZK-Ho1xM1!3o5qW_j#anph^}Dy68WVyA(JYmVxsYb
zlq>@FG{6AM;L&b@YY}xyP6Yp26a7_By)rY!LHa`9^(O3W2~98LJY+$&B>2rl77eTi
z-wbN~04IV$Vq|4?*2YGV;G1Z)dDCUld2DmH*yfTzzInr}FM_i}3lwl=3pAvTg+Kyq
z!?UQL+zoirF{2+F#tTpqKLrX^I*SJ&)Y^a-2KO9>wGc8(SW${mmw%&~#0(4dFECQ>
z%^*Eg0yqQo3vQ@9(YpN(8zvYbH{OTmBfa`ObWI~AD9Y=@C&aoUcfzm6p~c6UE^Ds}`~{yJ6=a*VB-la1hT=qxIMe)aSKSyg$tA%c6tntSxztcuZOZv|S}_i^X@
zEan#;{U6EleXh-FHm^7pTN2hC8HsH6&uN3%oHi%&Z04mkFdkkEIgnufAye4+iXTyO
zT`jpq(ql~`$RE|bIemY#aRcZg4v-xY5iv$43drf>J#bf<9eKu=?q_UlOql)tWFcp~
zhRYyW*ytxQ2g^zRs}a7YfVMg0CTsUT@Etx3L|au~B3U{9u~1ciIw2yN)bwV0I^e
zmb=ohGyHj(zv$OiSA~wt?7iyu&3`XX@We*TL)x!2N#>z;f0;te!~f-igOnY~oOC8g(5iICUUW&$m;eEK<@!?;;P>AXsl5>#b3nYp&i`mDC4^3P5oLay~am!)7P(dy(X8~heO4(dI3
zSY^W>xKp`Q&4GqLs{S0J3_-LBNwN#Q
zJBgYAGDv_l38H;!YO3^aZl2>W9agB`4^qqt-1)f`z7t6=`4^c*zEhcOg15~#sEN;r
zjCD{QD(z1?WN5sv?J0J_8*k{9oSKqPb?*ip-Ubh0>K6s7)V@S?`=hpKSBJUKdEmo^NX)
za19U|->OHs7?N{UE%u(;hQ;6R%%L~CBl#O`93HYcs%z3+dxhqJ?xrA4Z&zy34l<2)F5O{ix4;KlN<@Ji(PD9Ac!Hyz^$+1k
zFjw6Fg1iIOPV?3@PSo}eaB`}FNQu123w$wT3h3lLivIgC)scDl85Im`OUL?^IyGk&
z`ehYQXFs_Yd2YgP{u=39R*OiwpB&Lbr3*qj9Z>P)4#D=f`uVX7R4{0?>^cj{pixnQ
z#}Ut*C{cNTJ-c_KyE&~?K^?aTN>OXSozAVEYxhr>$@JL$xIlI$cVWfL8Jd@U-qaEA
zHwnU!4*V`##-$(>zC=ZLfsY9ZmI4@A11%ro-tr&z77UweGpZ(?%%}MC35odl*rSAp
zk}tkI@X+fWI&6Bf`A}$T=2eSuizA03Gp`xriIjd(u|pon(&98ed{p2h;A1=l8%8-g
zb?eRigJum~wE~76Tj2;3w%6H;&4ONxOO}wBrj9M^k)CPV8`*w6M4uVL>gY}PB^14Z
zjT<%)vLE%=D!T!&C_%9Y{d^i&TebkJAVz%dGBmj~XMC>vP4_(;lTQ~YlD1C2Db8L{
zF)sS`o<~G*MwYKy67To8C*1F(&+R3;y{@Qz0Yp@eP9JF=V*0D(yT#sk-Yx)w65
zT?$MMZ?}yvEj`>piz3A6z$6ua$JrCF?%oMHP1pcY35|RJ
zSBwtPg?1rNnD`_CJ~|(7cBnYTO{FWJMFRTdtWy2K7C+V5q-b?*!?7w{0VD<)ywcpF@a7p>OwZtlnlw7rH=H
zGv!F7)kGie+7-V=r}s?r>KzTY=`GTU)%$Mq{cvKRv@vKW%o}QT4WBBHnu3@UW4F)gY>Mk0K7{7yG;2(?N~&$ut#X%B=7f6z
zG6S+Vm$M}Z=W%3QhR`KGJl5WLtUsAI__{64K6?02KEtBX43;7y*u>XkD+kWxg=u~5
z23odX8$@``V8U|gH!pWYi_`V9@Z&bx3qs9r<|X1;#(vBxB|YiWBWy;3I^^02MK^9`
zk_Xk@#=go;z-ygz*djwYt3sx<>0Dn-C5O%1(4mJ1|6Y3S{Q^=QGZIKK5%ty4-`FOl
z_`T^C?w#6E2sr>Djsmk12Dt8PP!lFD6%m#(&&-%o`qb3wTjdQx)Wmo2H)0cMIpq`M
z{>z$$KRM=xC{ZW8eN+X=&B#bVK!nu285E>zTa!{p^-kt*VQ{E(-gt53@sJI6&~A`}
zSR4oybE(zgi0@g4&K~;Alu}KIvw1?2sul^dAS~JHK8y;|{QI||4_$BzoPZmRhJ{g(
zNo4N0FpxKIU1ois1G1azeW&Tf*2TAwUp*aWD+Evu+>vtRIqT7g*i;>md<{g4d~>u*
zeQLIVuH1WpO?g{T*?D>`S@DLd6+ubw`uy`<6Uk&$}?OfXWxqMzs8229-I0u_etv7UfO7|d^4bfMxQL^ZX^YC;Ob?i
z?I*Vp#GW*S&8WdAyj9q;kMx~=aBe~-_oTt>`=pM_On=j4MYaNl+K1Pavu<-!d#-MC
zTS(Q>K|+izLV^B(z}4v2$~o?#d^(d#y661wj?M1YZrya_#fFP@d+C`UY&bjW{whP#
zXKYT_@MlvxtPQ#>d<(AwOIZs72uF8;Wh-?|6
zP=GecLbz35!@)%&2DjV?^^izeVY^h1!R$aFH5DFYV6S!(lCG
zJ93jP{;HwzOCw8^o!_cm7;PXb4d-6*1Y8LyM6N*eQGxq2kwn|RV2V&8t}NbKZzWX6
zmu|5Wxggi;GDiO9$oQh^d26OOcFp(JpvM*}h`w5_wlZcUTFX&2{O3WY`Y2-n*Z|84
zXeYE#+{PdsC`6+BZleU9g)zvD%*Jm40zEFJ|323}}=(klz7(7Gz;n6jnsCeE?N
zY8GuuSrB@2YIl_=SI++9M46xrJ0YIg2)Hp(;QMCpij@J8cQtVX8gbNSSjggdv)})_N+jtfH>BKb;u