From 26896b0f877d6964ccce53308567df7485ad99f9 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Tue, 1 Mar 2022 12:50:13 -0500 Subject: [PATCH 1/7] sweeper initial work --- tango/sweep-config.jsonnet | 8 ++ tango/sweep-config.yaml | 10 +++ tango/sweeps.py | 45 ++++++++++++ tango/sweeps_test.ipynb | 146 +++++++++++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 tango/sweep-config.jsonnet create mode 100644 tango/sweep-config.yaml create mode 100644 tango/sweeps.py create mode 100644 tango/sweeps_test.ipynb diff --git a/tango/sweep-config.jsonnet b/tango/sweep-config.jsonnet new file mode 100644 index 000000000..415988705 --- /dev/null +++ b/tango/sweep-config.jsonnet @@ -0,0 +1,8 @@ +{ + sweeper: "default", + n_proc: 1, + hyperparameters: { + batch_size: [16, 32, 64], + lr: [0.01, 0.001], + }, +} \ No newline at end of file diff --git a/tango/sweep-config.yaml b/tango/sweep-config.yaml new file mode 100644 index 000000000..c029c8755 --- /dev/null +++ b/tango/sweep-config.yaml @@ -0,0 +1,10 @@ +sweeper: "default" +n_proc: 1 +hyperparameters: + batc_size: + - 16 + - 32 + - 64 + lr: + - 0.001 + - 0.0001 \ No newline at end of file diff --git a/tango/sweeps.py b/tango/sweeps.py new file mode 100644 index 000000000..3e19f89c6 --- /dev/null +++ b/tango/sweeps.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass, asdict +from tango.common import Params +from tango.common import Registrable +import itertools + +config_path = "/Users/sabhyac/Desktop/sabhya/tango/tango/sweep-config.yaml" + +# main sweeper class +class Sweeper(Registrable): + def __init__(self, config_path): + super(Registrable, self).__init__() + self.config_path = config_path + self.sweep_config = load_config(config_path) + + # returns all the combinations of hyperparameters in the form of a list of lists + def get_combinations(self): + hyperparams = self.sweep_config.config["hyperparameters"] + hyperparams_lsts = [] + for key, val in hyperparams.items(): + hyperparams_lsts.append(val) + hyperparam_combos = list(itertools.product(*hyperparams_lsts)) + return hyperparam_combos + + + +# wandb sweeper class +@Sweeper.register("wandb") +class WandbSweeper(Sweeper): + pass + +# function that loads the config from a specified yaml or jasonnet file +# TODO: how do I read "wandb" form config and call appropriate class +def load_config(config_path: str): + return SweepConfig.from_file(config_path) + +# data class that loads the parameters +# TODO: unsure about how to specify a default here? +@dataclass(frozen=True) +class SweepConfig(Params): + config: dict + + +if __name__ == "__main__": + sw = Sweeper(config_path=config_path) + print(sw.sweep_config) \ No newline at end of file diff --git a/tango/sweeps_test.ipynb b/tango/sweeps_test.ipynb new file mode 100644 index 000000000..e8b57f659 --- /dev/null +++ b/tango/sweeps_test.ipynb @@ -0,0 +1,146 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "41db2d6b", + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass, asdict\n", + "from tango.common import Params\n", + "from tango.common import Registrable\n", + "import itertools" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "6b1f94e1", + "metadata": {}, + "outputs": [], + "source": [ + "config_path = \"/Users/sabhyac/Desktop/sabhya/tango/tango/sweep-config.yaml\"" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "4642e40d", + "metadata": {}, + "outputs": [], + "source": [ + "class Sweeper(Registrable):\n", + " def __init__(self, config_path):\n", + " super(Registrable, self).__init__()\n", + " self.config_path = config_path\n", + " self.sweep_config = load_config(config_path)\n", + " \n", + " def get_combinations(self):\n", + " hyperparams = self.sweep_config.config[\"hyperparameters\"]\n", + " hyperparams_lsts = []\n", + " for key, val in hyperparams.items():\n", + " hyperparams_lsts.append(val) \n", + " hyperparam_combos = list(itertools.product(*hyperparams_lsts))\n", + " print(hyperparam_combos)\n", + "\n", + "@Sweeper.register(\"wandb\")\n", + "class WandbSweeper(Sweeper):\n", + " pass\n", + "\n", + "def load_config(config_path: str):\n", + " return SweepConfig.from_file(config_path)\n", + "\n", + "@dataclass(frozen=True)\n", + "class SweepConfig(Params):\n", + " config: dict" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "034c3f37", + "metadata": {}, + "outputs": [], + "source": [ + "sw = Sweeper(config_path=config_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "1cc17d35", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "SweepConfig(config={'sweeper': 'default', 'n_proc': 1, 'hyperparameters': {'batc_size': [16, 32, 64], 'lr': [0.001, 0.0001]}})" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sw.sweep_config" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "cb348871", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[(16, 0.001), (16, 0.0001), (32, 0.001), (32, 0.0001), (64, 0.001), (64, 0.0001)]\n" + ] + } + ], + "source": [ + "sw.get_combinations()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f0d1249", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b70bae90", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 3d4f320c9fd0d62fc03cd3d3fe010f7742dd4096 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Tue, 1 Mar 2022 13:05:11 -0500 Subject: [PATCH 2/7] new functions --- tango/sweeps.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tango/sweeps.py b/tango/sweeps.py index 3e19f89c6..2ba320dd3 100644 --- a/tango/sweeps.py +++ b/tango/sweeps.py @@ -20,8 +20,14 @@ def get_combinations(self): hyperparams_lsts.append(val) hyperparam_combos = list(itertools.product(*hyperparams_lsts)) return hyperparam_combos - + # TODO: wondering if this function should be here or in a test_file? + def override_hyperparameters(self): + pass + + # TODO: trying to figure the best path forward? should i use tests? + def run_experiments(self): + pass # wandb sweeper class @Sweeper.register("wandb") From 2d8afa94e0ad035bd99033c617ec2b69b02e4993 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Tue, 1 Mar 2022 13:07:48 -0500 Subject: [PATCH 3/7] remove test file --- tango/sweeps_test.ipynb | 146 ---------------------------------------- 1 file changed, 146 deletions(-) delete mode 100644 tango/sweeps_test.ipynb diff --git a/tango/sweeps_test.ipynb b/tango/sweeps_test.ipynb deleted file mode 100644 index e8b57f659..000000000 --- a/tango/sweeps_test.ipynb +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "id": "41db2d6b", - "metadata": {}, - "outputs": [], - "source": [ - "from dataclasses import dataclass, asdict\n", - "from tango.common import Params\n", - "from tango.common import Registrable\n", - "import itertools" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "6b1f94e1", - "metadata": {}, - "outputs": [], - "source": [ - "config_path = \"/Users/sabhyac/Desktop/sabhya/tango/tango/sweep-config.yaml\"" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "id": "4642e40d", - "metadata": {}, - "outputs": [], - "source": [ - "class Sweeper(Registrable):\n", - " def __init__(self, config_path):\n", - " super(Registrable, self).__init__()\n", - " self.config_path = config_path\n", - " self.sweep_config = load_config(config_path)\n", - " \n", - " def get_combinations(self):\n", - " hyperparams = self.sweep_config.config[\"hyperparameters\"]\n", - " hyperparams_lsts = []\n", - " for key, val in hyperparams.items():\n", - " hyperparams_lsts.append(val) \n", - " hyperparam_combos = list(itertools.product(*hyperparams_lsts))\n", - " print(hyperparam_combos)\n", - "\n", - "@Sweeper.register(\"wandb\")\n", - "class WandbSweeper(Sweeper):\n", - " pass\n", - "\n", - "def load_config(config_path: str):\n", - " return SweepConfig.from_file(config_path)\n", - "\n", - "@dataclass(frozen=True)\n", - "class SweepConfig(Params):\n", - " config: dict" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "034c3f37", - "metadata": {}, - "outputs": [], - "source": [ - "sw = Sweeper(config_path=config_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "id": "1cc17d35", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "SweepConfig(config={'sweeper': 'default', 'n_proc': 1, 'hyperparameters': {'batc_size': [16, 32, 64], 'lr': [0.001, 0.0001]}})" - ] - }, - "execution_count": 73, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sw.sweep_config" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "id": "cb348871", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[(16, 0.001), (16, 0.0001), (32, 0.001), (32, 0.0001), (64, 0.001), (64, 0.0001)]\n" - ] - } - ], - "source": [ - "sw.get_combinations()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6f0d1249", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b70bae90", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 804275d362e2da7d160cd25f5b7d0712aa2cd1f2 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Mon, 7 Mar 2022 17:57:28 -0500 Subject: [PATCH 4/7] sweeps working preliminary implementation --- tango/sweep-config.jsonnet | 8 ----- tango/sweep-config.yaml | 10 ------ tango/sweeps.py | 71 +++++++++++++++++++++++--------------- 3 files changed, 43 insertions(+), 46 deletions(-) delete mode 100644 tango/sweep-config.jsonnet delete mode 100644 tango/sweep-config.yaml diff --git a/tango/sweep-config.jsonnet b/tango/sweep-config.jsonnet deleted file mode 100644 index 415988705..000000000 --- a/tango/sweep-config.jsonnet +++ /dev/null @@ -1,8 +0,0 @@ -{ - sweeper: "default", - n_proc: 1, - hyperparameters: { - batch_size: [16, 32, 64], - lr: [0.01, 0.001], - }, -} \ No newline at end of file diff --git a/tango/sweep-config.yaml b/tango/sweep-config.yaml deleted file mode 100644 index c029c8755..000000000 --- a/tango/sweep-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -sweeper: "default" -n_proc: 1 -hyperparameters: - batc_size: - - 16 - - 32 - - 64 - lr: - - 0.001 - - 0.0001 \ No newline at end of file diff --git a/tango/sweeps.py b/tango/sweeps.py index 2ba320dd3..c2efdd085 100644 --- a/tango/sweeps.py +++ b/tango/sweeps.py @@ -1,51 +1,66 @@ -from dataclasses import dataclass, asdict -from tango.common import Params -from tango.common import Registrable import itertools +from collections import OrderedDict +from dataclasses import dataclass + +from tango.common import Params, Registrable +from tango.common.testing import run_experiment + +main_config_path = ( + "/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/config.jsonnet" +) +sweeps_config_path = ( + "/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet" +) +components = ( + "/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/basic_arithmetic.py" +) -config_path = "/Users/sabhyac/Desktop/sabhya/tango/tango/sweep-config.yaml" -# main sweeper class class Sweeper(Registrable): - def __init__(self, config_path): + def __init__(self, main_config_path: str, sweeps_config_path: str, components: str): super(Registrable, self).__init__() - self.config_path = config_path - self.sweep_config = load_config(config_path) - + self.main_config_path = main_config_path + self.sweep_config = load_config(sweeps_config_path) + self.main_config_path = main_config_path + self.components = components + # returns all the combinations of hyperparameters in the form of a list of lists def get_combinations(self): hyperparams = self.sweep_config.config["hyperparameters"] hyperparams_lsts = [] for key, val in hyperparams.items(): - hyperparams_lsts.append(val) + hyperparams_lsts.append(val) hyperparam_combos = list(itertools.product(*hyperparams_lsts)) return hyperparam_combos - - # TODO: wondering if this function should be here or in a test_file? - def override_hyperparameters(self): - pass - + # TODO: trying to figure the best path forward? should i use tests? def run_experiments(self): - pass + hyperparam_combos = self.get_combinations() + for combination in hyperparam_combos: + main_config = self.override_hyperparameters(combination) + with run_experiment(main_config, include_package=[self.components]) as run_dir: + # TODO: fill in something here? + pass + + # TODO: wondering if this function should be here or in a test_file? + def override_hyperparameters(self, experiment_tuple: dict): + # Override all the hyperparameters in the current experiment_config + overrides = {} + for (i, key) in enumerate(self.sweep_config.config["hyperparameters"].keys()): + overrides[key] = experiment_tuple[i] + # load the config & override it + main_config = Params.from_file(self.main_config_path, params_overrides=overrides) + return main_config -# wandb sweeper class -@Sweeper.register("wandb") -class WandbSweeper(Sweeper): - pass # function that loads the config from a specified yaml or jasonnet file # TODO: how do I read "wandb" form config and call appropriate class -def load_config(config_path: str): - return SweepConfig.from_file(config_path) +def load_config(sweeps_config_path: str): + return SweepConfig.from_file(sweeps_config_path) + # data class that loads the parameters # TODO: unsure about how to specify a default here? @dataclass(frozen=True) class SweepConfig(Params): - config: dict - - -if __name__ == "__main__": - sw = Sweeper(config_path=config_path) - print(sw.sweep_config) \ No newline at end of file + config: OrderedDict From 31f06fb84a484a47b9bbb91ef5ee338c63ba84c5 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Tue, 8 Mar 2022 16:14:23 -0500 Subject: [PATCH 5/7] typing added --- tango/sweeps.py | 5 +- tango/sweeps_test.ipynb | 393 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 tango/sweeps_test.ipynb diff --git a/tango/sweeps.py b/tango/sweeps.py index c2efdd085..d8497f5b7 100644 --- a/tango/sweeps.py +++ b/tango/sweeps.py @@ -28,7 +28,7 @@ def __init__(self, main_config_path: str, sweeps_config_path: str, components: s def get_combinations(self): hyperparams = self.sweep_config.config["hyperparameters"] hyperparams_lsts = [] - for key, val in hyperparams.items(): + for val in hyperparams.values(): hyperparams_lsts.append(val) hyperparam_combos = list(itertools.product(*hyperparams_lsts)) return hyperparam_combos @@ -38,12 +38,13 @@ def run_experiments(self): hyperparam_combos = self.get_combinations() for combination in hyperparam_combos: main_config = self.override_hyperparameters(combination) + # TODO: need to figure where & how to store results / way to track runs with run_experiment(main_config, include_package=[self.components]) as run_dir: # TODO: fill in something here? pass # TODO: wondering if this function should be here or in a test_file? - def override_hyperparameters(self, experiment_tuple: dict): + def override_hyperparameters(self, experiment_tuple: tuple): # Override all the hyperparameters in the current experiment_config overrides = {} for (i, key) in enumerate(self.sweep_config.config["hyperparameters"].keys()): diff --git a/tango/sweeps_test.ipynb b/tango/sweeps_test.ipynb new file mode 100644 index 000000000..2d0fa9e7d --- /dev/null +++ b/tango/sweeps_test.ipynb @@ -0,0 +1,393 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 49, + "id": "41db2d6b", + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass, asdict\n", + "from tango.common import Params\n", + "from tango.common import Registrable\n", + "from tango.common.testing import run_experiment\n", + "from collections import OrderedDict\n", + "import itertools" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "6b1f94e1", + "metadata": {}, + "outputs": [], + "source": [ + "sweeps_config_path = \"/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet\"\n", + "main_config_path = \"/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/config.jsonnet\"\n", + "components = \"/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/basic_arithmetic.py\"" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "4642e40d", + "metadata": {}, + "outputs": [], + "source": [ + "class Sweeper(Registrable):\n", + " def __init__(self, main_config_path: str , sweeps_config_path: str, components: str):\n", + " super(Registrable, self).__init__()\n", + " self.config_path = main_config_path\n", + " self.sweep_config = load_config(sweeps_config_path)\n", + " self.main_config_path = main_config_path\n", + " self.components = components\n", + " \n", + " # returns all the combinations of hyperparameters in the form of a list of lists\n", + " def get_combinations(self):\n", + " hyperparams = self.sweep_config.config[\"hyperparameters\"]\n", + " hyperparams_lsts = []\n", + " for key, val in hyperparams.items():\n", + " hyperparams_lsts.append(val) \n", + " hyperparam_combos = list(itertools.product(*hyperparams_lsts))\n", + " return hyperparam_combos\n", + " \n", + " # TODO: trying to figure the best path forward? should i use tests?\n", + " def run_experiments(self):\n", + " hyperparam_combos = self.get_combinations()\n", + " for combination in hyperparam_combos:\n", + " main_config = self.override_hyperparameters(combination)\n", + " with run_experiment(main_config, include_package=[self.components]) as run_dir:\n", + " # TODO: fill in something here?\n", + " pass\n", + " \n", + " # TODO: wondering if this function should be here or in a test_file?\n", + " def override_hyperparameters(self, experiment_tuple: dict):\n", + " # Override all the hyperparameters in the current experiment_config\n", + " overrides = {}\n", + " for (i, key) in enumerate(self.sweep_config.config[\"hyperparameters\"].keys()):\n", + " overrides[key] = experiment_tuple[i]\n", + " # load the config & override it\n", + " main_config = Params.from_file(self.main_config_path, params_overrides=overrides)\n", + " return main_config\n", + " \n", + "\n", + "# function that loads the config from a specified yaml or jasonnet file\n", + "# TODO: how do I read \"wandb\" form config and call appropriate class\n", + "def load_config(config_path: str):\n", + " return SweepConfig.from_file(config_path)\n", + "\n", + "# data class that loads the parameters\n", + "# TODO: unsure about how to specify a default here?\n", + "@dataclass(frozen=True)\n", + "class SweepConfig(Params):\n", + " config: OrderedDict" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "034c3f37", + "metadata": {}, + "outputs": [], + "source": [ + "sw = Sweeper(main_config_path=main_config_path, sweeps_config_path=config_path, components=components)" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "1cc17d35", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'steps.add_numbers.num1': [8, 16],\n", + " 'steps.add_numbers.num2': [2, 4],\n", + " 'steps.add_x.num2': [1, 2],\n", + " 'steps.divide_result.factor': [5, 10],\n", + " 'steps.multiply_result.factor': [1, 10]}" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sw.sweep_config.config[\"hyperparameters\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "cb348871", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(8, 2, 1, 5, 1)" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sw.get_combinations()[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "6f0d1249", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting new run proper-osprey\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "3.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/AdditionStep-3jJeYfWuajwTjmC2LRLmvQKkg7dDgbVD\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/ScaleDown-2NFgy7ynbLViMVyvVfTF5MyFMwxt35Ha\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/Print-4mQjAMijYXe8JXyps2os5r63wpjDzKav\n", + "Finished run proper-osprey\n", + "Starting new run just-cub\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "21.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/AdditionStep-66vynTtLtrqj7Rx1zbiuWSPXSavPdNp4\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/ScaleDown-3FuQAfg28T9ooWBNyza3UKLJhmMizmKk\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/Print-eqmMX7w962dRL1pZyjcABTVDAmwsCJCQ\n", + "Finished run just-cub\n", + "Starting new run moral-oriole\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "2.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/AdditionStep-67YMU8ECPbQfX9SxjCKwMNzH5Vy8aVCR\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/ScaleDown-62vFyrKe2rkSvJbtgGvQo2JTStnA2DYk\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/Print-4iD4zgH2L2ehVcb3fS2xzGQtBsem4ypg\n", + "Finished run moral-oriole\n", + "Starting new run pumped-hen\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "11.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/AdditionStep-5iwCiJkNh4maFguKuDx3quWG3wovYPyj\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/ScaleDown-VUZiioYuUzCN2nwZL7znwdqd8P1Hw5Y5\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/Print-4XnusrC4M7DRgsXKyaZzhfTAQboH1eJL\n", + "Finished run pumped-hen\n", + "Starting new run native-goblin\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "4.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/AdditionStep-Z7zxedA8H1zSNAEUxxwo5uhmHkTdaHMa\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/ScaleDown-2NFgy7ynbLViMVyvVfTF5MyFMwxt35Ha\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/Print-4KRcAJsTvXQ3uuey7kaf3vLwVXW3imto\n", + "Finished run native-goblin\n", + "Starting new run on-joey\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "22.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/AdditionStep-3WmejhakQcjZ6s1Lpd9Pju2E3keswSr5\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/ScaleDown-3FuQAfg28T9ooWBNyza3UKLJhmMizmKk\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/Print-3cu2d5W4bFu6PkxbNC3cCQrCo8LAdwTb\n", + "Finished run on-joey\n", + "Starting new run deep-stud\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "3.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/AdditionStep-2dAwX9hfbdp3HGqjH2WeLSkFtWp8h2Te\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/ScaleDown-62vFyrKe2rkSvJbtgGvQo2JTStnA2DYk\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/Print-3RofxL3QzEcPTSugGJv5xnFGLppD8wKd\n", + "Finished run deep-stud\n", + "Starting new run sacred-shad\n", + "● Starting step \"add_numbers\" ...\n", + "✓ Finished step \"add_numbers\"\n", + "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", + "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", + "✓ Finished step \"multiply_result\"\n", + "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", + "✓ Finished step \"divide_result\"\n", + "● Starting step \"add_x\" ...\n", + "✓ Finished step \"add_x\"\n", + "✓ Found output for step \"divide_result\" in cache ...\n", + "✓ Found output for step \"multiply_result\" in cache ...\n", + "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", + "● Starting step \"print\" ...\n", + "12.0\n", + "✓ Finished step \"print\"\n", + "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", + "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/AdditionStep-2zPGoYJAd3LRon9yd5SSkXqczbZiTpBL\n", + "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/ScaleDown-VUZiioYuUzCN2nwZL7znwdqd8P1Hw5Y5\n", + "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", + "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/Print-55ZhHbf9HnzdtjLgaEEabkh3H8xcwVZC\n", + "Finished run sacred-shad\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Input \u001b[0;32mIn [81]\u001b[0m, in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43msw\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_experiments\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "Input \u001b[0;32mIn [77]\u001b[0m, in \u001b[0;36mSweeper.run_experiments\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 20\u001b[0m hyperparam_combos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_combinations()\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m combination \u001b[38;5;129;01min\u001b[39;00m hyperparam_combos:\n\u001b[0;32m---> 22\u001b[0m main_config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moverride_hyperparameters\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcombination\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 23\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m run_experiment(main_config, include_package\u001b[38;5;241m=\u001b[39m[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcomponents]) \u001b[38;5;28;01mas\u001b[39;00m run_dir:\n\u001b[1;32m 24\u001b[0m \u001b[38;5;66;03m# TODO: fill in something here?\u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n", + "Input \u001b[0;32mIn [77]\u001b[0m, in \u001b[0;36mSweeper.override_hyperparameters\u001b[0;34m(self, experiment_tuple)\u001b[0m\n\u001b[1;32m 32\u001b[0m overrides[key] \u001b[38;5;241m=\u001b[39m experiment_tuple[i]\n\u001b[1;32m 33\u001b[0m \u001b[38;5;66;03m# load the config & override it\u001b[39;00m\n\u001b[0;32m---> 34\u001b[0m main_config \u001b[38;5;241m=\u001b[39m \u001b[43mParams\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_file\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmain_config_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams_overrides\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moverrides\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m main_config\n", + "File \u001b[0;32m~/Desktop/sabhya/tango/tango/common/params.py:520\u001b[0m, in \u001b[0;36mParams.from_file\u001b[0;34m(cls, params_file, params_overrides, ext_vars)\u001b[0m\n\u001b[1;32m 517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 518\u001b[0m \u001b[38;5;66;03m# Fall back to JSON/Jsonnet.\u001b[39;00m\n\u001b[1;32m 519\u001b[0m ext_vars \u001b[38;5;241m=\u001b[39m {\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m_environment_variables(), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mext_vars}\n\u001b[0;32m--> 520\u001b[0m file_dict \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(\u001b[43mevaluate_file\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mparams_file\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mext_vars\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mext_vars\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 522\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(params_overrides, \u001b[38;5;28mdict\u001b[39m):\n\u001b[1;32m 523\u001b[0m params_overrides \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mdumps(params_overrides)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "sw.run_experiments()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b70bae90", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c38010b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 6a7feabe322c0afc12ea5df00d215734e4748112 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Tue, 22 Mar 2022 14:50:17 -0400 Subject: [PATCH 6/7] functioning sweeper --- tango/sweeps.py | 57 +++--- tango/sweeps_test.ipynb | 393 ---------------------------------------- 2 files changed, 29 insertions(+), 421 deletions(-) delete mode 100644 tango/sweeps_test.ipynb diff --git a/tango/sweeps.py b/tango/sweeps.py index d8497f5b7..d1ffe5d36 100644 --- a/tango/sweeps.py +++ b/tango/sweeps.py @@ -1,23 +1,19 @@ import itertools +import json +import subprocess from collections import OrderedDict from dataclasses import dataclass from tango.common import Params, Registrable -from tango.common.testing import run_experiment - -main_config_path = ( - "/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/config.jsonnet" -) -sweeps_config_path = ( - "/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet" -) -components = ( - "/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/basic_arithmetic.py" -) class Sweeper(Registrable): - def __init__(self, main_config_path: str, sweeps_config_path: str, components: str): + def __init__( + self, + main_config_path: str, + sweeps_config_path: str, + components: str, + ): super(Registrable, self).__init__() self.main_config_path = main_config_path self.sweep_config = load_config(sweeps_config_path) @@ -25,43 +21,48 @@ def __init__(self, main_config_path: str, sweeps_config_path: str, components: s self.components = components # returns all the combinations of hyperparameters in the form of a list of lists - def get_combinations(self): - hyperparams = self.sweep_config.config["hyperparameters"] + def get_combinations(self) -> list: + hyperparams = self.sweep_config.config["config"]["hyperparameters"] hyperparams_lsts = [] for val in hyperparams.values(): hyperparams_lsts.append(val) hyperparam_combos = list(itertools.product(*hyperparams_lsts)) return hyperparam_combos - # TODO: trying to figure the best path forward? should i use tests? + # loops through all combinations of hyperparameters and creates a run for each def run_experiments(self): hyperparam_combos = self.get_combinations() for combination in hyperparam_combos: - main_config = self.override_hyperparameters(combination) + # main_config = self.override_hyperparameters(combination) + overrides = self.override_hyperparameters(combination) # TODO: need to figure where & how to store results / way to track runs - with run_experiment(main_config, include_package=[self.components]) as run_dir: - # TODO: fill in something here? - pass + # specify what workspace to use + subprocess.call( + [ + "tango", + "run", + self.main_config_path, + "--include-package", + self.components, + "--overrides", + json.dumps(overrides), + ] + ) - # TODO: wondering if this function should be here or in a test_file? - def override_hyperparameters(self, experiment_tuple: tuple): - # Override all the hyperparameters in the current experiment_config + # function to override all the hyperparameters in the current experiment_config + def override_hyperparameters(self, experiment_tuple: tuple) -> dict: overrides = {} - for (i, key) in enumerate(self.sweep_config.config["hyperparameters"].keys()): + for (i, key) in enumerate(self.sweep_config.config["config"]["hyperparameters"].keys()): overrides[key] = experiment_tuple[i] - # load the config & override it - main_config = Params.from_file(self.main_config_path, params_overrides=overrides) - return main_config + return overrides # function that loads the config from a specified yaml or jasonnet file -# TODO: how do I read "wandb" form config and call appropriate class def load_config(sweeps_config_path: str): return SweepConfig.from_file(sweeps_config_path) # data class that loads the parameters -# TODO: unsure about how to specify a default here? @dataclass(frozen=True) class SweepConfig(Params): config: OrderedDict diff --git a/tango/sweeps_test.ipynb b/tango/sweeps_test.ipynb deleted file mode 100644 index 2d0fa9e7d..000000000 --- a/tango/sweeps_test.ipynb +++ /dev/null @@ -1,393 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 49, - "id": "41db2d6b", - "metadata": {}, - "outputs": [], - "source": [ - "from dataclasses import dataclass, asdict\n", - "from tango.common import Params\n", - "from tango.common import Registrable\n", - "from tango.common.testing import run_experiment\n", - "from collections import OrderedDict\n", - "import itertools" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "id": "6b1f94e1", - "metadata": {}, - "outputs": [], - "source": [ - "sweeps_config_path = \"/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet\"\n", - "main_config_path = \"/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/config.jsonnet\"\n", - "components = \"/Users/sabhyac/Desktop/sabhya/tango/test_fixtures/sweeps/basic_test/basic_arithmetic.py\"" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "id": "4642e40d", - "metadata": {}, - "outputs": [], - "source": [ - "class Sweeper(Registrable):\n", - " def __init__(self, main_config_path: str , sweeps_config_path: str, components: str):\n", - " super(Registrable, self).__init__()\n", - " self.config_path = main_config_path\n", - " self.sweep_config = load_config(sweeps_config_path)\n", - " self.main_config_path = main_config_path\n", - " self.components = components\n", - " \n", - " # returns all the combinations of hyperparameters in the form of a list of lists\n", - " def get_combinations(self):\n", - " hyperparams = self.sweep_config.config[\"hyperparameters\"]\n", - " hyperparams_lsts = []\n", - " for key, val in hyperparams.items():\n", - " hyperparams_lsts.append(val) \n", - " hyperparam_combos = list(itertools.product(*hyperparams_lsts))\n", - " return hyperparam_combos\n", - " \n", - " # TODO: trying to figure the best path forward? should i use tests?\n", - " def run_experiments(self):\n", - " hyperparam_combos = self.get_combinations()\n", - " for combination in hyperparam_combos:\n", - " main_config = self.override_hyperparameters(combination)\n", - " with run_experiment(main_config, include_package=[self.components]) as run_dir:\n", - " # TODO: fill in something here?\n", - " pass\n", - " \n", - " # TODO: wondering if this function should be here or in a test_file?\n", - " def override_hyperparameters(self, experiment_tuple: dict):\n", - " # Override all the hyperparameters in the current experiment_config\n", - " overrides = {}\n", - " for (i, key) in enumerate(self.sweep_config.config[\"hyperparameters\"].keys()):\n", - " overrides[key] = experiment_tuple[i]\n", - " # load the config & override it\n", - " main_config = Params.from_file(self.main_config_path, params_overrides=overrides)\n", - " return main_config\n", - " \n", - "\n", - "# function that loads the config from a specified yaml or jasonnet file\n", - "# TODO: how do I read \"wandb\" form config and call appropriate class\n", - "def load_config(config_path: str):\n", - " return SweepConfig.from_file(config_path)\n", - "\n", - "# data class that loads the parameters\n", - "# TODO: unsure about how to specify a default here?\n", - "@dataclass(frozen=True)\n", - "class SweepConfig(Params):\n", - " config: OrderedDict" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "id": "034c3f37", - "metadata": {}, - "outputs": [], - "source": [ - "sw = Sweeper(main_config_path=main_config_path, sweeps_config_path=config_path, components=components)" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "id": "1cc17d35", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'steps.add_numbers.num1': [8, 16],\n", - " 'steps.add_numbers.num2': [2, 4],\n", - " 'steps.add_x.num2': [1, 2],\n", - " 'steps.divide_result.factor': [5, 10],\n", - " 'steps.multiply_result.factor': [1, 10]}" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sw.sweep_config.config[\"hyperparameters\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "id": "cb348871", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(8, 2, 1, 5, 1)" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sw.get_combinations()[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "id": "6f0d1249", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting new run proper-osprey\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "3.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/AdditionStep-3jJeYfWuajwTjmC2LRLmvQKkg7dDgbVD\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/ScaleDown-2NFgy7ynbLViMVyvVfTF5MyFMwxt35Ha\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsngn5yg16/workspace/cache/Print-4mQjAMijYXe8JXyps2os5r63wpjDzKav\n", - "Finished run proper-osprey\n", - "Starting new run just-cub\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "21.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/AdditionStep-66vynTtLtrqj7Rx1zbiuWSPXSavPdNp4\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/ScaleDown-3FuQAfg28T9ooWBNyza3UKLJhmMizmKk\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxqz1gqlq/workspace/cache/Print-eqmMX7w962dRL1pZyjcABTVDAmwsCJCQ\n", - "Finished run just-cub\n", - "Starting new run moral-oriole\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "2.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/AdditionStep-67YMU8ECPbQfX9SxjCKwMNzH5Vy8aVCR\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/ScaleDown-62vFyrKe2rkSvJbtgGvQo2JTStnA2DYk\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsnvrtci6d/workspace/cache/Print-4iD4zgH2L2ehVcb3fS2xzGQtBsem4ypg\n", - "Finished run moral-oriole\n", - "Starting new run pumped-hen\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "11.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/AdditionStep-5iwCiJkNh4maFguKuDx3quWG3wovYPyj\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/ScaleDown-VUZiioYuUzCN2nwZL7znwdqd8P1Hw5Y5\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_tests0lxpi5nh/workspace/cache/Print-4XnusrC4M7DRgsXKyaZzhfTAQboH1eJL\n", - "Finished run pumped-hen\n", - "Starting new run native-goblin\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "4.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/AdditionStep-Z7zxedA8H1zSNAEUxxwo5uhmHkTdaHMa\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/ScaleDown-2NFgy7ynbLViMVyvVfTF5MyFMwxt35Ha\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testswcf0r45o/workspace/cache/Print-4KRcAJsTvXQ3uuey7kaf3vLwVXW3imto\n", - "Finished run native-goblin\n", - "Starting new run on-joey\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "22.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/AdditionStep-3WmejhakQcjZ6s1Lpd9Pju2E3keswSr5\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/ScaleDown-3FuQAfg28T9ooWBNyza3UKLJhmMizmKk\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testss0_bni68/workspace/cache/Print-3cu2d5W4bFu6PkxbNC3cCQrCo8LAdwTb\n", - "Finished run on-joey\n", - "Starting new run deep-stud\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "3.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/AdditionStep-2dAwX9hfbdp3HGqjH2WeLSkFtWp8h2Te\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/ScaleDown-62vFyrKe2rkSvJbtgGvQo2JTStnA2DYk\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/ScaleUp-c88UVQjQ9GEuSGBUD1rNVsFq2ULbxaR6\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testskxyythyj/workspace/cache/Print-3RofxL3QzEcPTSugGJv5xnFGLppD8wKd\n", - "Finished run deep-stud\n", - "Starting new run sacred-shad\n", - "● Starting step \"add_numbers\" ...\n", - "✓ Finished step \"add_numbers\"\n", - "✓ Found output for step \"add_numbers\" in cache (needed by \"multiply_result\") ...\n", - "● Starting step \"multiply_result\" (needed by \"divide_result\") ...\n", - "✓ Finished step \"multiply_result\"\n", - "● Starting step \"divide_result\" (needed by \"add_x\") ...\n", - "✓ Finished step \"divide_result\"\n", - "● Starting step \"add_x\" ...\n", - "✓ Finished step \"add_x\"\n", - "✓ Found output for step \"divide_result\" in cache ...\n", - "✓ Found output for step \"multiply_result\" in cache ...\n", - "✓ Found output for step \"add_x\" in cache (needed by \"print\") ...\n", - "● Starting step \"print\" ...\n", - "12.0\n", - "✓ Finished step \"print\"\n", - "✓ The output for \"add_numbers\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/AdditionStep-5SnWM66M3F7uD2UpCqGAbwLfzHMxmkqA\n", - "✓ The output for \"add_x\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/AdditionStep-2zPGoYJAd3LRon9yd5SSkXqczbZiTpBL\n", - "✓ The output for \"divide_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/ScaleDown-VUZiioYuUzCN2nwZL7znwdqd8P1Hw5Y5\n", - "✓ The output for \"multiply_result\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/ScaleUp-5Fsy9AbNHQZTREHfunhfwbpt6AjFP7T4\n", - "✓ The output for \"print\" is in /var/folders/st/xl2yk49d6v50p42_9spxgtlr0000gp/T/tango_testsxjk07_8r/workspace/cache/Print-55ZhHbf9HnzdtjLgaEEabkh3H8xcwVZC\n", - "Finished run sacred-shad\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Input \u001b[0;32mIn [81]\u001b[0m, in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43msw\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_experiments\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "Input \u001b[0;32mIn [77]\u001b[0m, in \u001b[0;36mSweeper.run_experiments\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 20\u001b[0m hyperparam_combos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_combinations()\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m combination \u001b[38;5;129;01min\u001b[39;00m hyperparam_combos:\n\u001b[0;32m---> 22\u001b[0m main_config \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moverride_hyperparameters\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcombination\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 23\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m run_experiment(main_config, include_package\u001b[38;5;241m=\u001b[39m[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcomponents]) \u001b[38;5;28;01mas\u001b[39;00m run_dir:\n\u001b[1;32m 24\u001b[0m \u001b[38;5;66;03m# TODO: fill in something here?\u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m\n", - "Input \u001b[0;32mIn [77]\u001b[0m, in \u001b[0;36mSweeper.override_hyperparameters\u001b[0;34m(self, experiment_tuple)\u001b[0m\n\u001b[1;32m 32\u001b[0m overrides[key] \u001b[38;5;241m=\u001b[39m experiment_tuple[i]\n\u001b[1;32m 33\u001b[0m \u001b[38;5;66;03m# load the config & override it\u001b[39;00m\n\u001b[0;32m---> 34\u001b[0m main_config \u001b[38;5;241m=\u001b[39m \u001b[43mParams\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_file\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmain_config_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams_overrides\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moverrides\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m main_config\n", - "File \u001b[0;32m~/Desktop/sabhya/tango/tango/common/params.py:520\u001b[0m, in \u001b[0;36mParams.from_file\u001b[0;34m(cls, params_file, params_overrides, ext_vars)\u001b[0m\n\u001b[1;32m 517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 518\u001b[0m \u001b[38;5;66;03m# Fall back to JSON/Jsonnet.\u001b[39;00m\n\u001b[1;32m 519\u001b[0m ext_vars \u001b[38;5;241m=\u001b[39m {\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m_environment_variables(), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mext_vars}\n\u001b[0;32m--> 520\u001b[0m file_dict \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(\u001b[43mevaluate_file\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mparams_file\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mext_vars\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mext_vars\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 522\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(params_overrides, \u001b[38;5;28mdict\u001b[39m):\n\u001b[1;32m 523\u001b[0m params_overrides \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mdumps(params_overrides)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "sw.run_experiments()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b70bae90", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c38010b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From c59b394b280aebcce39955d97c97d44704a1eae2 Mon Sep 17 00:00:00 2001 From: Sabhya Chhabria Date: Tue, 22 Mar 2022 15:06:40 -0400 Subject: [PATCH 7/7] sweeps test files --- .../sweeps/basic_test/basic_arithmetic.py | 25 +++++++++++++++++ .../sweeps/basic_test/config.jsonnet | 28 +++++++++++++++++++ .../sweeps/basic_test/sweeps-config.jsonnet | 13 +++++++++ .../sweeps/basic_test/sweeps-config.yaml | 0 .../cv_example_test/sweep-config.jsonnet | 8 ++++++ .../sweeps/cv_example_test/sweep-config.yaml | 10 +++++++ 6 files changed, 84 insertions(+) create mode 100644 test_fixtures/sweeps/basic_test/basic_arithmetic.py create mode 100644 test_fixtures/sweeps/basic_test/config.jsonnet create mode 100644 test_fixtures/sweeps/basic_test/sweeps-config.jsonnet create mode 100644 test_fixtures/sweeps/basic_test/sweeps-config.yaml create mode 100644 test_fixtures/sweeps/cv_example_test/sweep-config.jsonnet create mode 100644 test_fixtures/sweeps/cv_example_test/sweep-config.yaml diff --git a/test_fixtures/sweeps/basic_test/basic_arithmetic.py b/test_fixtures/sweeps/basic_test/basic_arithmetic.py new file mode 100644 index 000000000..42aed8cf5 --- /dev/null +++ b/test_fixtures/sweeps/basic_test/basic_arithmetic.py @@ -0,0 +1,25 @@ +from typing import Union + +from tango import Step + +IntOrFloat = Union[int, float] + +@Step.register("addition") +class AdditionStep(Step): + def run(self, num1: int, num2: int) -> int: + return num1 + num2 + +@Step.register("scale_up") +class ScaleUp(Step): + def run(self, num1: int, factor: int) -> int: + return num1 * factor + +@Step.register("scale_down") +class ScaleDown(Step): + def run(self, num1: int, factor: int) -> IntOrFloat: + return num1 / factor + +@Step.register("print") +class Print(Step): + def run(self, num: IntOrFloat) -> None: + print(num) \ No newline at end of file diff --git a/test_fixtures/sweeps/basic_test/config.jsonnet b/test_fixtures/sweeps/basic_test/config.jsonnet new file mode 100644 index 000000000..dffd55793 --- /dev/null +++ b/test_fixtures/sweeps/basic_test/config.jsonnet @@ -0,0 +1,28 @@ +{ + "steps": { + "add_numbers": { + "type": "addition", + "num1": 34, + "num2": 8 + }, + "multiply_result": { + "type": "scale_up", + "num1": {"type": "ref", "ref": "add_numbers"}, + "factor": 10, + }, + "divide_result": { + "type": "scale_down", + "num1": {"type": "ref", "ref": "multiply_result"}, + "factor": 5, + }, + "add_x": { + "type": "addition", + "num1": {"type": "ref", "ref": "divide_result"}, + "num2": 1, + }, + "print": { + "type": "print", + "num": {"type": "ref", "ref": "add_x"}, + }, + }, +} diff --git a/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet b/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet new file mode 100644 index 000000000..cdb432357 --- /dev/null +++ b/test_fixtures/sweeps/basic_test/sweeps-config.jsonnet @@ -0,0 +1,13 @@ +{ + "config": { + "sweeper": "default", + "n_proc": 1, + "hyperparameters": { + "steps.add_numbers.num1": [8, 16], + "steps.add_numbers.num2": [2, 4], + "steps.multiply_result.factor": [1, 10], + "steps.divide_result.factor": [5, 10], + "steps.add_x.num2": [1, 2], + }, + }, +} \ No newline at end of file diff --git a/test_fixtures/sweeps/basic_test/sweeps-config.yaml b/test_fixtures/sweeps/basic_test/sweeps-config.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/test_fixtures/sweeps/cv_example_test/sweep-config.jsonnet b/test_fixtures/sweeps/cv_example_test/sweep-config.jsonnet new file mode 100644 index 000000000..415988705 --- /dev/null +++ b/test_fixtures/sweeps/cv_example_test/sweep-config.jsonnet @@ -0,0 +1,8 @@ +{ + sweeper: "default", + n_proc: 1, + hyperparameters: { + batch_size: [16, 32, 64], + lr: [0.01, 0.001], + }, +} \ No newline at end of file diff --git a/test_fixtures/sweeps/cv_example_test/sweep-config.yaml b/test_fixtures/sweeps/cv_example_test/sweep-config.yaml new file mode 100644 index 000000000..c029c8755 --- /dev/null +++ b/test_fixtures/sweeps/cv_example_test/sweep-config.yaml @@ -0,0 +1,10 @@ +sweeper: "default" +n_proc: 1 +hyperparameters: + batc_size: + - 16 + - 32 + - 64 + lr: + - 0.001 + - 0.0001 \ No newline at end of file