From 4bc45101000989bda9616e58cc1d98bf5fb8e668 Mon Sep 17 00:00:00 2001 From: Federico Siciliano <77665712+federicosiciliano@users.noreply.github.com> Date: Mon, 8 Apr 2024 09:13:42 +0200 Subject: [PATCH] Release (#2) * Initial commit * Update README.md * integration exp_utils integration exp_utils uncommented * Integration data_utils Integration data_utils uncommented * integration torch_utils integration torch_utils uncommented * resnet18+minst_ntb notebook for ResNet18 and MNIST dataset * commenting code partially commenting code partially * exp_utils commented exp_utils commented * exp_utils and data_utils commented exp_utils and data_utils commented * comments, readme and information_ntb comments, readme and information_ntb * update updating requirement.txt * Update requirements.txt * Update torch.py * custom losses handling * GCE loss in custom losses * Rename losses to losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update process.py * GCE classical * Update losses.py * Update losses.py * Update losses.py * Noise Robust Loss * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * NCODLoss manual optimization * Update model.py * Update preparation.py * Update model.py * Update model.py * NCODLoss encodings * Update losses.py * Update preparation.py * Update preparation.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update model.py * Update model.py * Update model.py * Update model.py * LR scheduler and optimization of NCODLoss as in the paper * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * NCODLoss * Update model.py * Update preparation.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update losses.py * Update losses.py * Update losses.py * Update losses.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update model.py * Update losses.py * Update model.py * Update model.py * Prova (#1) * delete data_utils * prova1 * prova2 * prova3 * prova4 * prova5 * prova6 * prova7 * prova8 * prova9 * prova10 * exp * exp1 * torch1 * rec_utils * submodules updated * submodules updated * version1 * submodule branch * solve sequential * new gitignore * new gitignore * changed name from utils to easy * submodule links * updated gitignore and submodules * fix install metadata? * fix rec_torch_utils * easy_rec update + base notebook * change git username * release change account * update username submodules * Update model.py * rebase? * release * relase3 --------- Co-authored-by: Federico Carmignani <82052662+fed21@users.noreply.github.com> Co-authored-by: Federico Siciliano <77665712+siciliano-diag@users.noreply.github.com> --- .gitignore | 2 +- .gitmodules | 24 +- README.md | 6 +- cfg/config_nn.yaml | 4 +- cfg/local_config.yaml | 4 +- cfg/torchvision_config.yaml | 4 +- data_utils | 1 - easy_data | 1 + easy_exp | 1 + easy_rec | 1 + easy_torch | 1 + exp_utils | 1 - ntb/try_data.ipynb | 97 +-- ntb/try_exp.ipynb | 1187 +++++------------------------------ ntb/try_rec.ipynb | 1 + ntb/try_torch.ipynb | 305 ++------- rec_utils | 1 - setup.py | 12 +- torch_utils | 1 - 19 files changed, 255 insertions(+), 1399 deletions(-) delete mode 160000 data_utils create mode 160000 easy_data create mode 160000 easy_exp create mode 160000 easy_rec create mode 160000 easy_torch delete mode 160000 exp_utils create mode 100644 ntb/try_rec.ipynb delete mode 160000 rec_utils delete mode 160000 torch_utils diff --git a/.gitignore b/.gitignore index f875045..091e7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### CUSTOM GITIGNORE # DS_Store files # http://stackoverflow.com/questions/107701/how-can-i-remove-ds-store-files-from-a-git-repository -**/.DS_Store +.DS_Store # Config (personal) cfg/wandb/* diff --git a/.gitmodules b/.gitmodules index 603da62..f293776 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,16 @@ -[submodule "data_utils"] - path = data_utils - url = https://github.com/siciliano-diag/data_utils.git +[submodule "easy_data"] + path = easy_data + url = https://github.com/federicosiciliano/easy_data.git branch = main -[submodule "exp_utils"] - path = exp_utils - url = https://github.com/siciliano-diag/exp_utils.git +[submodule "easy_exp"] + path = easy_exp + url = https://github.com/federicosiciliano/easy_exp.git branch = main -[submodule "torch_utils"] - path = torch_utils - url = https://github.com/siciliano-diag/torch_utils.git +[submodule "easy_torch"] + path = easy_torch + url = https://github.com/federicosiciliano/easy_torch.git branch = main -[submodule "rec_utils"] - path = rec_utils - url = https://github.com/siciliano-diag/rec_utils.git +[submodule "easy_rec"] + path = easy_rec + url = https://github.com/federicosiciliano/easy_rec.git branch = main diff --git a/README.md b/README.md index 262ffef..40a857e 100644 --- a/README.md +++ b/README.md @@ -43,17 +43,17 @@ Below is an outline of key files and folders you'll find in this project, along ### Folders -1. **data_utils** +1. **easy_data** - Contains utilities for data loading, file management, data and data structure management, data splitting, and statistics. -2. **exp_utils** +2. **easy_exp** - Manages experiments by defining unique IDs based on their configuration. - Allows for hashing of each ID to check for previously conducted experiments. - Excludes GPU/CPU usage and training modes from the experiment ID. - Saves experiments in a specific file along with their relative configuration. - Includes methods for parsing YAML configs and handles special characters used in them (e.g., through `var.py`). -3. **torch_utils** +3. **easy_torch** - Includes functions for metrics, loading models, and creating trainers in PyTorch Lightning. - Defines steps, loss, optimizer, and other parameters to use. - Sets callbacks and dataloaders. diff --git a/cfg/config_nn.yaml b/cfg/config_nn.yaml index eda29f2..c35cdcd 100644 --- a/cfg/config_nn.yaml +++ b/cfg/config_nn.yaml @@ -3,7 +3,7 @@ data: name: MNIST source: torchvision #uci #tfds #custom #path_to_local_file # -split_vars: - # value: "data_utils.utils.split_single_column" + # value: "easy_data.utils.split_single_column" # eval: eval merge_before_split: False @@ -50,7 +50,7 @@ __exp__: # key_prefix: "" #Prefix for experiment key, optional, default = "" __imports__: #imports needed for config preparation (e.g. type casting) - torchvision - # - data_utils + # - easy_data # - numpy as np #" as " will be recognized # - name: pandas #" as " will be recognized # fromlist: diff --git a/cfg/local_config.yaml b/cfg/local_config.yaml index 316b7be..f3c3753 100644 --- a/cfg/local_config.yaml +++ b/cfg/local_config.yaml @@ -5,7 +5,7 @@ data: loader_params: delimiter: "," # -split_vars: - # value: "data_utils.utils.split_single_column" + # value: "easy_data.utils.split_single_column" # eval: eval merge_before_split: False @@ -51,7 +51,7 @@ __exp__: # key_len: 16 #Length of experiment key, optional, default = 16 # key_prefix: "" #Prefix for experiment key, optional, default = "" # imports: #imports needed for config preparation (e.g. type casting) - # # - data_utils + # # - easy_data # # - numpy as np #" as " will be recognized # # - name: pandas #" as " will be recognized # # fromlist: diff --git a/cfg/torchvision_config.yaml b/cfg/torchvision_config.yaml index c46eb05..548a1c1 100644 --- a/cfg/torchvision_config.yaml +++ b/cfg/torchvision_config.yaml @@ -3,7 +3,7 @@ data: #[adult, australian, breast-cancer-wisconsin, bupa, car, cleveland, covtype, crx, EEG%20Eye%20State, german, glass, haberman, heart, hepatitis, image, ionosphere, iris, monks-1, monks-2, monks-3, poker, sonar]: source: torchvision #uci #tfds #custom #path_to_local_file # -split_vars: - # value: "data_utils.utils.split_single_column" + # value: "easy_data.utils.split_single_column" # eval: eval merge_before_split: False @@ -48,7 +48,7 @@ __exp__: # key_len: 16 #Length of experiment key, optional, default = 16 # key_prefix: "" #Prefix for experiment key, optional, default = "" # imports: #imports needed for config preparation (e.g. type casting) - # # - data_utils + # # - easy_data # # - numpy as np #" as " will be recognized # # - name: pandas #" as " will be recognized # # fromlist: diff --git a/data_utils b/data_utils deleted file mode 160000 index b971470..0000000 --- a/data_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b97147071d65aec695ee798b0bd8a5705f7a92f1 diff --git a/easy_data b/easy_data new file mode 160000 index 0000000..ce6db97 --- /dev/null +++ b/easy_data @@ -0,0 +1 @@ +Subproject commit ce6db977a18303466a8d44f111b93bdbc66810a6 diff --git a/easy_exp b/easy_exp new file mode 160000 index 0000000..a0e6fe1 --- /dev/null +++ b/easy_exp @@ -0,0 +1 @@ +Subproject commit a0e6fe188fb78cc8dbc0033cc151b2ce0d6ad0b6 diff --git a/easy_rec b/easy_rec new file mode 160000 index 0000000..ae5e5ff --- /dev/null +++ b/easy_rec @@ -0,0 +1 @@ +Subproject commit ae5e5ff385fbb4f2bf62124c46c1e2278264604d diff --git a/easy_torch b/easy_torch new file mode 160000 index 0000000..9983353 --- /dev/null +++ b/easy_torch @@ -0,0 +1 @@ +Subproject commit 9983353c7fe2e78ce88f2e049656c6a6d07787df diff --git a/exp_utils b/exp_utils deleted file mode 160000 index a44f4c4..0000000 --- a/exp_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a44f4c4feae524336c1320b4b2e76e251f16c07b diff --git a/ntb/try_data.ipynb b/ntb/try_data.ipynb index 20ab312..beeb693 100644 --- a/ntb/try_data.ipynb +++ b/ntb/try_data.ipynb @@ -4,12 +4,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Testing Data Utils" + "# Testing Easy Data" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -18,17 +18,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['../', '/Users/federicosiciliano/Desktop/Py_utils/data_utils/ntb', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python38.zip', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/lib-dynload', '', '/Users/federicosiciliano/Library/Python/3.8/lib/python/site-packages', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/site-packages']\n" - ] - } - ], + "outputs": [], "source": [ "project_folder = \"../\"\n", "sys.path.insert(0, project_folder)\n", @@ -37,11 +29,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import exp_utils, data_utils" + "import easy_exp, easy_data" ] }, { @@ -53,112 +45,63 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg = exp_utils.cfg.load_configuration(\"local_config\")" + "cfg = easy_exp.cfg.load_configuration(\"local_config\")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg[\"data\"][\"split_vars\"] = data_utils.utils.separate_single_column" + "cfg[\"data\"][\"split_vars\"] = easy_data.utils.separate_single_column" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "data = data_utils.data.load_data(**cfg[\"data\"])" + "data = easy_data.data.load_data(**cfg[\"data\"])" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'train_x': array([[2., 7., 6., 1.],\n", - " [4., 5., 6., 1.],\n", - " [5., 5., 5., 0.],\n", - " [5., 2., 2., 1.],\n", - " [2., 2., 2., 0.],\n", - " [9., 6., 2., 1.]]),\n", - " 'train_y': array([[0., 1.],\n", - " [0., 1.],\n", - " [1., 0.],\n", - " [0., 1.],\n", - " [1., 0.],\n", - " [0., 1.]]),\n", - " 'val_x': array([[1., 1., 1., 0.],\n", - " [7., 8., 9., 0.]]),\n", - " 'val_y': array([[1., 0.],\n", - " [1., 0.]]),\n", - " 'test_x': array([[8., 9., 6., 1.],\n", - " [1., 2., 3., 0.]]),\n", - " 'test_y': array([[0., 1.],\n", - " [1., 0.]])}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "data" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg = exp_utils.cfg.load_configuration(\"torchvision_config\")" + "cfg = easy_exp.cfg.load_configuration(\"torchvision_config\")" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "data = data_utils.data.load_data(**cfg[\"data\"])" + "data = easy_data.data.load_data(**cfg[\"data\"])" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "train x torch.Size([48000, 1, 28, 28])\n", - "\n", - "train y (48000, 10)\n", - "\n", - "val x torch.Size([12000, 1, 28, 28])\n", - "\n", - "val y (12000, 10)\n", - "\n", - "test x torch.Size([10000, 1, 28, 28])\n", - "\n", - "test y (10000, 10)\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "for split in [\"train\",\"val\",\"test\"]:\n", " for var in [\"x\",\"y\"]:\n", diff --git a/ntb/try_exp.ipynb b/ntb/try_exp.ipynb index 6099648..2e17d0a 100644 --- a/ntb/try_exp.ipynb +++ b/ntb/try_exp.ipynb @@ -5,12 +5,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Test Exp Utils" + "# Test Easy Exp" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -20,17 +20,9 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['../', '/Users/federicosiciliano/Desktop/Py_utils/exp_utils/ntb', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python38.zip', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/lib-dynload', '', '/Users/federicosiciliano/Library/Python/3.8/lib/python/site-packages', '/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/site-packages']\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "project_folder = \"../\"\n", "sys.path.insert(0, project_folder)\n", @@ -39,11 +31,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import exp_utils" + "import easy_exp" ] }, { @@ -56,179 +48,70 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg = exp_utils.cfg.load_configuration()" + "cfg = easy_exp.cfg.load_configuration()" ] }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'app1': {'foo_str': 'string_value',\n", - " 'foo_int': 123,\n", - " 'foo_float': 1.23,\n", - " 'foo_none': None,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': array([1, 2, 3]),\n", - " 'foo_parse3': 0 1 2\n", - " 0 1 2 3\n", - " 1 4 5 6,\n", - " 'foo_parse4': 8,\n", - " 'app1_2': {'foo_str1_2': 'app2_foo'},\n", - " 'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'app_glob1_2': 'glob1_2'},\n", - " '__exp__': {'name': 'prova',\n", - " '__imports__': ['numpy as np',\n", - " {'name': 'pandas', 'fromlist': ['DataFrame']},\n", - " 'multiprocessing'],\n", - " '__nosave__': {'app1.foo_parse4': None,\n", - " 'app1.app1_2': None,\n", - " 'app2.foo_str': None,\n", - " 'app2.foo_int2': None,\n", - " 'app2.app3.foo_int1': None,\n", - " 'app1.app_glob1_2': None},\n", - " 'project_folder': '../',\n", - " 'key_len': 16,\n", - " 'key_prefix': ''},\n", - " 'app2': {'app3': {'foo_int1': 123}, 'foo_str': 'app2_foo', 'foo_int2': 123}}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg" ] }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "exp_utils.cfg.ConfigObject" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "type(cfg)" ] }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(dict, dict)" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "type(cfg[\"app1\"]), type(cfg[\"__exp__\"])" ] }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'name': 'prova',\n", - " '__imports__': ['numpy as np',\n", - " {'name': 'pandas', 'fromlist': ['DataFrame']},\n", - " 'multiprocessing'],\n", - " '__nosave__': {'app1.foo_parse4': None,\n", - " 'app1.app1_2': None,\n", - " 'app2.foo_str': None,\n", - " 'app2.foo_int2': None,\n", - " 'app2.app3.foo_int1': None,\n", - " 'app1.app_glob1_2': None},\n", - " 'project_folder': '../',\n", - " 'key_len': 16,\n", - " 'key_prefix': ''}" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg[\"__exp__\"]" ] }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 2, 3])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg[\"app1\"][\"foo_parse2\"]" ] }, { "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([1, 2, 3])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg[\"app1.foo_parse2\"]" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -237,20 +120,9 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('abc', 'abc')" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg[\"app3.foo_str\"],cfg[\"app3\"][\"foo_str\"]" ] @@ -265,11 +137,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg, exp_cfg = exp_utils.exp.separate_exp_cfg(cfg) # Drop __exp__ key cause it doesn't define an experiment's parameters" + "cfg, exp_cfg = easy_exp.exp.separate_exp_cfg(cfg) # Drop __exp__ key cause it doesn't define an experiment's parameters" ] }, { @@ -282,40 +154,21 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "eNhVVnmZQ7jDMnAp\n", - "B-0BDI2sx6Z6BMgfdk\n", - "zJsQ2euT\n", - "FSajX3\n" - ] - } - ], + "outputs": [], "source": [ - "print(exp_utils.exp.generate_random_id())\n", - "print(exp_utils.exp.generate_random_id(key_prefix=\"B-\"))\n", - "print(exp_utils.exp.generate_random_id(key_len=8))\n", - "print(exp_utils.exp.generate_random_id(key_len=4,key_prefix=\"FS\"))" + "print(easy_exp.exp.generate_random_id())\n", + "print(easy_exp.exp.generate_random_id(key_prefix=\"B-\"))\n", + "print(easy_exp.exp.generate_random_id(key_len=8))\n", + "print(easy_exp.exp.generate_random_id(key_len=4,key_prefix=\"FS\"))" ] }, { "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Default number of possible keys: 10 ^ 28.678267031972062\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import string\n", "import numpy as np\n", @@ -335,98 +188,48 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#The only solution, is save each numpy array / pandas DF / else in the configuration as their own experiment, having a certain experiment ID\n", "numpy_key = \"app1.foo_parse2\"\n", "np_to_save_elsewhere = cfg[numpy_key]\n", - "np_exp_id = 123 #exp_utils.exp.generate_random_id() #get numpy object experiment ID somehow\n", + "np_exp_id = 123 #easy_exp.exp.generate_random_id() #get numpy object experiment ID somehow\n", "cfg[numpy_key] = np_exp_id #now cfg doesn't have the numpy array anymore\n", "\n", "pandas_key = \"app1.foo_parse3\"\n", "pd_to_save_elsewhere = cfg[pandas_key]\n", - "pd_exp_id = 42 #exp_utils.exp.generate_random_id() #get pandas object experiment ID somehow\n", + "pd_exp_id = 42 #easy_exp.exp.generate_random_id() #get pandas object experiment ID somehow\n", "cfg[pandas_key] = pd_exp_id #now cfg doesn't have the pandas DF anymore" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'7d5beb1aeca81e09a51095e098c7172b'" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "exp_utils.exp.hash_config(cfg)" + "easy_exp.exp.hash_config(cfg)" ] }, { "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'app1': {'foo_str': 'string_value',\n", - " 'foo_int': 123,\n", - " 'foo_float': 1.23,\n", - " 'foo_none': None,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_parse4': 8,\n", - " 'app1_2': {'foo_str1_2': 'app2_foo'},\n", - " 'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'app_glob1_2': 'glob1_2'},\n", - " 'app2': {'app3': {'foo_int1': 123}, 'foo_str': 'app2_foo', 'foo_int2': 123},\n", - " 'app3': {'foo_str': 'abc'}}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(True, 'b8rTUqbVFHHqBrJP')" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "#cfg, exp_cfg = exp_utils.exp.separate_exp_cfg(cfg) # Drop __exp__ key cause it doesn't define an experiment's parameters\n", - "exp_utils.exp.get_experiment_id(cfg, exp_cfg)" + "#cfg, exp_cfg = easy_exp.exp.separate_exp_cfg(cfg) # Drop __exp__ key cause it doesn't define an experiment's parameters\n", + "easy_exp.exp.get_experiment_id(cfg, exp_cfg)" ] }, { @@ -439,74 +242,39 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg = exp_utils.exp.combine_exp_cfg(cfg, exp_cfg)" + "cfg = easy_exp.exp.combine_exp_cfg(cfg, exp_cfg)" ] }, { "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'name': 'prova',\n", - " '__imports__': ['numpy as np',\n", - " {'name': 'pandas', 'fromlist': ['DataFrame']},\n", - " 'multiprocessing'],\n", - " '__nosave__': {'app1.foo_parse4': None,\n", - " 'app1.app1_2': None,\n", - " 'app2.foo_str': None,\n", - " 'app2.foo_int2': None,\n", - " 'app2.app3.foo_int1': None,\n", - " 'app1.app_glob1_2': None},\n", - " 'project_folder': '../',\n", - " 'key_len': 16,\n", - " 'key_prefix': '',\n", - " 'hash': '691c996f5bad5a61d0b524a155f2908b'}" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg[\"__exp__\"]" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "exp_utils.exp.save_experiment(cfg)\n", + "easy_exp.exp.save_experiment(cfg)\n", "# Save experiment can be called directly; it will set the experiment id directly" ] }, { "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, 'b8rTUqbVFHHqBrJP'), (True, 'b8rTUqbVFHHqBrJP'))" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "exp_utils.exp.get_experiment_id(cfg), exp_utils.exp.get_experiment_id(cfg)\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "easy_exp.exp.get_experiment_id(cfg), easy_exp.exp.get_experiment_id(cfg)\n", "# If the cfg exists, we get\n", "# exp_found True\n", "# Always the same experiment_id" @@ -514,36 +282,25 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg, exp_cfg = exp_utils.exp.separate_exp_cfg(cfg) # Drop __exp__ key cause it doesn't define an experiment's parameters" + "cfg, exp_cfg = easy_exp.exp.separate_exp_cfg(cfg) # Drop __exp__ key cause it doesn't define an experiment's parameters" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, 'b8rTUqbVFHHqBrJP'), (True, 'b8rTUqbVFHHqBrJP'))" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "exp_utils.exp.get_experiment_id(cfg,exp_cfg), exp_utils.exp.get_experiment_id(cfg,exp_cfg) # Even if the cfg is divided, the result is the same" + "easy_exp.exp.get_experiment_id(cfg,exp_cfg), easy_exp.exp.get_experiment_id(cfg,exp_cfg) # Even if the cfg is divided, the result is the same" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -552,82 +309,38 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, '0sMmmIRL2vbiGUyH'), (True, '0sMmmIRL2vbiGUyH'))" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "exp_utils.exp.get_experiment_id(cfg,exp_cfg), exp_utils.exp.get_experiment_id(cfg,exp_cfg) # If the cfg doesn't exist, we get a different experiment_id. Also, exp_found is False" + "easy_exp.exp.get_experiment_id(cfg,exp_cfg), easy_exp.exp.get_experiment_id(cfg,exp_cfg) # If the cfg doesn't exist, we get a different experiment_id. Also, exp_found is False" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(True, '0sMmmIRL2vbiGUyH')" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "exp_utils.exp.get_set_experiment_id(cfg,exp_cfg)" + "easy_exp.exp.get_set_experiment_id(cfg,exp_cfg)" ] }, { "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'0sMmmIRL2vbiGUyH'" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "exp_cfg[\"experiment_id\"] # Now the experiment_id is saved inside exp_cfg" ] }, { "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, '0sMmmIRL2vbiGUyH'), '0sMmmIRL2vbiGUyH')" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "exp_utils.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "easy_exp.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]\n", "# get_experiment_id is meant to search for the experiment in the file,\n", "# so, if it doesn't find the cfg in the saved configurations,\n", "# It will not set the exp_id inside exp_cfg" @@ -635,22 +348,11 @@ }, { "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, '0sMmmIRL2vbiGUyH'), '0sMmmIRL2vbiGUyH')" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "exp_utils.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "easy_exp.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]\n", "# In the same way, get_set_experiment_id is meant to search for the experiment in the file,\n", "# so, if it doesn't find the cfg in the saved configurations,\n", "# it will not return the exp_id inside exp_cfg\n", @@ -659,7 +361,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -668,120 +370,65 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "exp_utils.exp.save_experiment(cfg,exp_cfg)" + "easy_exp.exp.save_experiment(cfg,exp_cfg)" ] }, { "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "prev_cfg == cfg # Check if cfg has been restored" ] }, { "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, '0sMmmIRL2vbiGUyH'),\n", - " '0sMmmIRL2vbiGUyH',\n", - " (True, '0sMmmIRL2vbiGUyH'),\n", - " '0sMmmIRL2vbiGUyH')" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Now that the experiment is saved, we will always get the same experiment_id\n", - "exp_utils.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"], exp_utils.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]" + "easy_exp.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"], easy_exp.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]" ] }, { "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, 'DO2T84byfPBTAviv'),\n", - " '0sMmmIRL2vbiGUyH',\n", - " (True, 'DO2T84byfPBTAviv'),\n", - " 'DO2T84byfPBTAviv')" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# If we modify cfg so that is new, we get the same results as before\n", "cfg[\"new_key\"] = \"kmasdkasm\"\n", "cfg[\"new_key2\"] = {\"abc\":123}\n", - "exp_utils.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"], exp_utils.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]" + "easy_exp.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"], easy_exp.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "exp_utils.exp.save_experiment(cfg,exp_cfg)" + "easy_exp.exp.save_experiment(cfg,exp_cfg)" ] }, { "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'app1.foo_parse4': None,\n", - " 'app1.app1_2': None,\n", - " 'app2.foo_str': None,\n", - " 'app2.foo_int2': None,\n", - " 'app2.app3.foo_int1': None,\n", - " 'app1.app_glob1_2': None}" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "exp_cfg[\"__nosave__\"] # Check nosave keys" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -790,26 +437,12 @@ }, { "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "((True, 'DO2T84byfPBTAviv'),\n", - " 'DO2T84byfPBTAviv',\n", - " (True, 'DO2T84byfPBTAviv'),\n", - " 'DO2T84byfPBTAviv')" - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# If a nosave key is modified, the cfg will still match a saved one\n", - "exp_utils.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"], exp_utils.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]" + "easy_exp.exp.get_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"], easy_exp.exp.get_set_experiment_id(cfg,exp_cfg), exp_cfg[\"experiment_id\"]" ] }, { @@ -822,435 +455,45 @@ }, { "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'0sMmmIRL2vbiGUyH': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5},\n", - " 'HhN3ZNkPkoxHkDzz': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 123,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'mTzSztICfKt0Voj3': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': '456',\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " '8WNMLIBK5jkrJaIY': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 789.0,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'DO2T84byfPBTAviv': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'b8rTUqbVFHHqBrJP': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'}}}" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_exps = exp_utils.exp.get_experiments(**exp_cfg)\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_exps = easy_exp.exp.get_experiments(**exp_cfg)\n", "all_exps" ] }, { "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'0sMmmIRL2vbiGUyH': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5},\n", - " 'HhN3ZNkPkoxHkDzz': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 123,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'mTzSztICfKt0Voj3': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': '456',\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " '8WNMLIBK5jkrJaIY': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 789.0,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'DO2T84byfPBTAviv': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'b8rTUqbVFHHqBrJP': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'}}}" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_exps = exp_utils.exp.get_experiments(**exp_cfg, sub_cfg={\"abc\":123}) # If not setting check_type, nothing will be checked\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_exps = easy_exp.exp.get_experiments(**exp_cfg, sub_cfg={\"abc\":123}) # If not setting check_type, nothing will be checked\n", "all_exps" ] }, { "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'HhN3ZNkPkoxHkDzz': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 123,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'mTzSztICfKt0Voj3': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': '456',\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " '8WNMLIBK5jkrJaIY': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 789.0,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'DO2T84byfPBTAviv': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}}}" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# With contain, sub_cfg must be a \"subset\" of the dict\n", "to_check = {'new_key': 'kmasdkasm','new_key2': {'abc': 123}}\n", - "all_exps = exp_utils.exp.get_experiments(**exp_cfg, sub_cfg = to_check, check_type=\"contain\") # If not setting check_type, nothing will be checked\n", + "all_exps = easy_exp.exp.get_experiments(**exp_cfg, sub_cfg = to_check, check_type=\"contain\") # If not setting check_type, nothing will be checked\n", "all_exps" ] }, { "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'HhN3ZNkPkoxHkDzz': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 123,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'mTzSztICfKt0Voj3': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': '456',\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " '8WNMLIBK5jkrJaIY': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': 789.0,\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}},\n", - " 'DO2T84byfPBTAviv': {'app1': {'app_glob': 'glob',\n", - " 'app_glob2': 'glob2',\n", - " 'foo_float': 1.23,\n", - " 'foo_int': 123,\n", - " 'foo_list1': [123, '456', 789.0],\n", - " 'foo_list2': [123, '456'],\n", - " 'foo_none': None,\n", - " 'foo_parse1': ['a', 'b', 'c'],\n", - " 'foo_parse2': 123,\n", - " 'foo_parse3': 42,\n", - " 'foo_ref1': 'string_value',\n", - " 'foo_ref2': 'string_value',\n", - " 'foo_str': 'string_value'},\n", - " 'app2': {'app3': {}},\n", - " 'app3': {'foo_str': 'abc'},\n", - " 'app5': 5,\n", - " 'new_key': 'kmasdkasm',\n", - " 'new_key2': {'abc': 123}}}" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "#with match, sub_cfg must use relative keys\n", "to_check = {'new_key2.abc': 123}\n", - "all_exps = exp_utils.exp.get_experiments(**exp_cfg, sub_cfg = to_check, check_type=\"match\")\n", + "all_exps = easy_exp.exp.get_experiments(**exp_cfg, sub_cfg = to_check, check_type=\"match\")\n", "all_exps" ] }, @@ -1264,21 +507,9 @@ }, { "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[123, '456', 789.0]\n", - "123 ___ 123\n", - "456 ___ 456\n", - "789.0 ___ 789.0\n", - "[123, '456', 789.0]\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "#If one parameter is an iterable of parameters to sweep, then use sweep. It will:\n", "#1) cycle on every value in the\n", @@ -1288,7 +519,7 @@ "print(cfg['app1.foo_list1'])\n", "for param_value in cfg.sweep('app1.foo_list1'):\n", " print(param_value,\"___\",cfg['app1.foo_list1'])\n", - " exp_utils.exp.get_set_experiment_id(cfg,exp_cfg),exp_utils.exp.save_experiment(cfg,exp_cfg)\n", + " easy_exp.exp.get_set_experiment_id(cfg,exp_cfg),easy_exp.exp.save_experiment(cfg,exp_cfg)\n", "print(cfg['app1.foo_list1'])" ] }, @@ -1302,56 +533,27 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "cfg = exp_utils.cfg.load_configuration(\"config2\")" + "cfg = easy_exp.cfg.load_configuration(\"config2\")" ] }, { "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'app1': {'+/app1_2': ['app1_2_file']},\n", - " '+app2': ['app2_file', 'app2bis_file'],\n", - " '__exp__': {'name': 'prova',\n", - " 'project_folder': '../',\n", - " 'key_len': 16,\n", - " 'key_prefix': '',\n", - " '__nosave__': {}}}" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg" ] }, { "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'asdsadas' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/Users/federicosiciliano/Desktop/Py_utils/exp_utils/ntb/try_exp.ipynb Cella 63\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfor\u001b[39;00m value1 \u001b[39min\u001b[39;00m cfg\u001b[39m.\u001b[39msweep_additions(\u001b[39m\"\u001b[39m\u001b[39mapp2\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m----> 2\u001b[0m \u001b[39mprint\u001b[39m(asdsadas)\n\u001b[1;32m 3\u001b[0m \u001b[39mfor\u001b[39;00m value2 \u001b[39min\u001b[39;00m cfg\u001b[39m.\u001b[39msweep_additions(\u001b[39m\"\u001b[39m\u001b[39mapp1./app1_2\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m 4\u001b[0m \u001b[39mprint\u001b[39m(value2)\n", - "\u001b[0;31mNameError\u001b[0m: name 'asdsadas' is not defined" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for value1 in cfg.sweep_additions(\"app2\"):\n", " print(asdsadas)\n", @@ -1363,53 +565,18 @@ }, { "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'app3': {'foo_int1': 123},\n", - " 'foo_str': 'app2_foo',\n", - " '__nosave__': ['foo_str', 'foo_int2', 'app3.foo_int1'],\n", - " 'foo_int2': 123}" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg[\"app2\"]" ] }, { "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'app1': {'+/app1_2': ['app1_2_file']},\n", - " '__exp__': {'name': 'prova',\n", - " 'project_folder': '../',\n", - " 'key_len': 16,\n", - " 'key_prefix': '',\n", - " '__nosave__': {}},\n", - " '__global__': {'app1': {'app_glob': 'glob', 'app_glob2': 'glob2'}},\n", - " 'app2': {'app3': {'foo_int1': 123},\n", - " 'foo_str': 'app2_foo',\n", - " '__nosave__': ['foo_str', 'foo_int2', 'app3.foo_int1'],\n", - " 'foo_int2': 123}}" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "cfg" ] @@ -1446,29 +613,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 Time/read: 0.0011250972747802734\n", - "1 Tot time: 0.0011250972747802734\n", - "\n", - "10 Time/read: 0.0002203702926635742\n", - "10 Tot time: 0.002203702926635742\n", - "\n", - "100 Time/read: 6.832838058471679e-05\n", - "100 Tot time: 0.00683283805847168\n", - "\n", - "1000 Time/read: 5.3598880767822266e-05\n", - "1000 Tot time: 0.053598880767822266\n", - "\n", - "10000 Time/read: 5.807058811187744e-05\n", - "10000 Tot time: 0.5807058811187744\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "start_time = time.time()\n", "\n", @@ -1518,18 +663,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "_StoreAction(option_strings=['--foo.foo'], dest='foo.foo', nargs=None, const=None, default=[1, 2, 3], type=None, choices=None, help=None, metavar=None)" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "parser.add_argument(\n", " \"--foo.foo\", #name or flags - Either a name or a list of option strings, e.g. foo or -f, --foo.\n", @@ -1560,18 +694,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[1., 1.]])" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "eval(\"np.ones((1,2))\")" ] diff --git a/ntb/try_rec.ipynb b/ntb/try_rec.ipynb new file mode 100644 index 0000000..0c54fc8 --- /dev/null +++ b/ntb/try_rec.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"markdown","metadata":{"id":"2nRCG0Q8347G"},"source":["# Preparation stuff"]},{"cell_type":"markdown","metadata":{"id":"c81IM9YmWpCp"},"source":["## Connect to Drive"]},{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":11,"status":"ok","timestamp":1686658044012,"user":{"displayName":"Federico Siciliano","userId":"13460778358604487896"},"user_tz":-120},"id":"EoEPCwJGdpz_"},"outputs":[],"source":["connect_to_drive = False"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":31338,"status":"ok","timestamp":1686658075342,"user":{"displayName":"Federico Siciliano","userId":"13460778358604487896"},"user_tz":-120},"id":"QgLWUVMAWpCq","outputId":"539a36c4-47bc-435a-ebb0-d59c874000c6"},"outputs":[],"source":["#Run command and authorize by popup --> other window\n","if connect_to_drive:\n"," from google.colab import drive\n"," drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"markdown","metadata":{"id":"SzMilQQcEL-w"},"source":["## Install packages"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":["if connect_to_drive:\n"," #Install FS code\n"," !pip install --upgrade --no-deps --force-reinstall git+https://github.com/federicosiciliano/easy_lightning.git@fedsic\n","\n"," !pip install pytorch_lightning"]},{"cell_type":"markdown","metadata":{"id":"llsyZg59yyiX"},"source":["## IMPORTS"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":5232,"status":"ok","timestamp":1686658109562,"user":{"displayName":"Federico Siciliano","userId":"13460778358604487896"},"user_tz":-120},"id":"U0GsBSD6yz9y"},"outputs":[],"source":["#Put all imports here\n","import numpy as np\n","import matplotlib.pyplot as plt\n","#from copy import deepcopy\n","#import pickle\n","import os\n","import sys\n","#import cv2\n","import torch"]},{"cell_type":"markdown","metadata":{"id":"mnSShc_Yy4lr"},"source":["## Define paths"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1686658109563,"user":{"displayName":"Federico Siciliano","userId":"13460778358604487896"},"user_tz":-120},"id":"WRYc5NEeyjQ8"},"outputs":[],"source":["#every path should start from the project folder:\n","project_folder = \"../\"\n","if connect_to_drive:\n"," project_folder = \"/content/gdrive/Shareddrives/\" #Name of SharedDrive folder\n"," #project_folder = \"/content/gdrive/MyDrive/\" #Name of MyDrive folder\n","\n","#Config folder should contain hyperparameters configurations\n","cfg_folder = os.path.join(project_folder,\"cfg\")\n","\n","#Data folder should contain raw and preprocessed data\n","data_folder = os.path.join(project_folder,\"data\")\n","raw_data_folder = os.path.join(data_folder,\"raw\")\n","processed_data_folder = os.path.join(data_folder,\"processed\")\n","\n","#Source folder should contain all the (essential) source code\n","source_folder = os.path.join(project_folder,\"src\")\n","\n","#The out folder should contain all outputs: models, results, plots, etc.\n","out_folder = os.path.join(project_folder,\"out\")\n","img_folder = os.path.join(out_folder,\"img\")"]},{"cell_type":"markdown","metadata":{"id":"B4fhGkp14CSb"},"source":["## Import own code"]},{"cell_type":"code","execution_count":6,"metadata":{"executionInfo":{"elapsed":3705,"status":"ok","timestamp":1686658113259,"user":{"displayName":"Federico Siciliano","userId":"13460778358604487896"},"user_tz":-120},"id":"LrX3FM7szllL"},"outputs":[],"source":["#To import from src:\n","\n","#attach the source folder to the start of sys.path\n","sys.path.insert(0, project_folder)\n","\n","#import from src directory\n","\n","import easy_exp, easy_rec, easy_torch #easy_data"]},{"cell_type":"markdown","metadata":{"id":"DIslF_wh31z2"},"source":["# MAIN"]},{"cell_type":"markdown","metadata":{},"source":["## Train"]},{"cell_type":"markdown","metadata":{"id":"o9gCiPA9dp0H"},"source":["### Data"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"background_save":true},"id":"1oW5REUm4GC8"},"outputs":[],"source":["cfg = easy_exp.cfg.load_configuration(\"config_rec\")"]},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[],"source":["cfg[\"data_params\"][\"data_folder\"] = raw_data_folder"]},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["True True\n","../data/raw/ml-1m ml-1m\n","Ratings data already exists. Skip pre-processing\n","Filtering by minimum number of users per item: 5\n","Filtering by minimum number of items per user: 5\n","Densifying index\n","Splitting: leave_n_out\n"]}],"source":["#cfg[\"data_params\"][\"test_sizes\"] = [cfg[\"data_params.dataset_params.out_seq_len.val\"],cfg[\"data_params.dataset_params.out_seq_len.test\"]]\n","\n","data, maps = easy_rec.data_generation_utils.preprocess_dataset(**cfg[\"data_params\"])\n","\n","#TODO: save maps"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[],"source":["datasets = easy_rec.rec_torch.prepare_rec_datasets(data,**cfg[\"data_params\"][\"dataset_params\"])"]},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[],"source":["cfg[\"model\"][\"loader_params\"][\"num_items\"] = np.max(list(maps[\"sid\"].values()))"]},{"cell_type":"code","execution_count":12,"metadata":{},"outputs":[],"source":["loaders = easy_rec.rec_torch.prepare_rec_data_loaders(datasets, data, **cfg[\"model\"][\"loader_params\"])"]},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[],"source":["#for x in loaders[\"train\"]: break"]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[],"source":["cfg[\"model\"][\"rec_model\"][\"num_items\"] = np.max(list(maps[\"sid\"].values()))\n","cfg[\"model\"][\"rec_model\"][\"num_users\"] = np.max(list(maps[\"uid\"].values()))\n","cfg[\"model\"][\"rec_model\"][\"lookback\"] = cfg[\"data_params\"][\"dataset_params\"][\"lookback\"]"]},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":["Seed set to 42\n"]}],"source":["main_module = easy_rec.rec_torch.create_rec_model(**cfg[\"model\"][\"rec_model\"])"]},{"cell_type":"code","execution_count":17,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["../out/exp not found --> creating\n","../out/exp/prova not found --> creating\n","Experiment already found: False ----> The experiment id is: i7wWDu0p6cpVK3vJ\n"]}],"source":["exp_found, experiment_id = easy_exp.exp.get_set_experiment_id(cfg)\n","print(\"Experiment already found:\", exp_found, \"----> The experiment id is:\", experiment_id)"]},{"cell_type":"code","execution_count":17,"metadata":{},"outputs":[],"source":["#if exp_found: exit() #TODO: make the notebook/script stop here if the experiment is already found"]},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":["Seed set to 42\n","Seed set to 42\n","Seed set to 42\n","GPU available: True (mps), used: False\n","TPU available: False, using: 0 TPU cores\n","IPU available: False, using: 0 IPUs\n","HPU available: False, using: 0 HPUs\n","/opt/homebrew/lib/python3.11/site-packages/pytorch_lightning/trainer/setup.py:187: GPU available but not used. You can set it by doing `Trainer(accelerator='gpu')`.\n","Seed set to 42\n","Seed set to 42\n","Seed set to 42\n","Seed set to 42\n"]}],"source":["trainer_params = easy_torch.preparation.prepare_experiment_id(cfg[\"model\"][\"trainer_params\"], experiment_id)\n","\n","# Prepare callbacks and logger using the prepared trainer_params\n","trainer_params[\"callbacks\"] = easy_torch.preparation.prepare_callbacks(trainer_params)\n","trainer_params[\"logger\"] = easy_torch.preparation.prepare_logger(trainer_params)\n","\n","# Prepare the trainer using the prepared trainer_params\n","trainer = easy_torch.preparation.prepare_trainer(**trainer_params)\n","\n","model_params = cfg[\"model\"].copy()\n","\n","model_params[\"loss\"] = easy_torch.preparation.prepare_loss(cfg[\"model\"][\"loss\"], easy_rec.losses)\n","\n","# Prepare the optimizer using configuration from cfg\n","model_params[\"optimizer\"] = easy_torch.preparation.prepare_optimizer(**cfg[\"model\"][\"optimizer\"])\n","\n","# Prepare the metrics using configuration from cfg\n","model_params[\"metrics\"] = easy_torch.preparation.prepare_metrics(cfg[\"model\"][\"metrics\"], easy_rec.metrics)\n","\n","# Create the model using main_module, loss, and optimizer\n","model = easy_torch.process.create_model(main_module, **model_params)"]},{"cell_type":"code","execution_count":19,"metadata":{},"outputs":[],"source":["# Prepare the emission tracker using configuration from cfg\n","tracker = easy_torch.preparation.prepare_emission_tracker(**cfg[\"model\"][\"emission_tracker\"], experiment_id=experiment_id)"]},{"cell_type":"markdown","metadata":{"id":"jttfzSb5Oezv"},"source":["### Train"]},{"cell_type":"code","execution_count":20,"metadata":{"colab":{"background_save":true},"id":"SNGT89Tbdp0K"},"outputs":[{"name":"stderr","output_type":"stream","text":["Seed set to 42\n","Missing logger folder: ../out/log/prova/i7wWDu0p6cpVK3vJ/lightning_logs\n","\n"," | Name | Type | Params\n","------------------------------------------------------------\n","0 | main_module | SASRec | 242 K \n","1 | loss | SequentialBCEWithLogitsLoss | 0 \n","2 | metrics | ModuleDict | 0 \n","------------------------------------------------------------\n","242 K Trainable params\n","0 Non-trainable params\n","242 K Total params\n","0.969 Total estimated model params size (MB)\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"693bcd1c27f741879227e081b8b2a2bc","version_major":2,"version_minor":0},"text/plain":["Sanity Checking: | | 0/? [00:00┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", - "┃ Test metric ┃ DataLoader 0 ┃\n", - "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", - "│ test_loss │ 6171157594112.0 │\n", - "└───────────────────────────┴───────────────────────────┘\n", - "\n" - ], - "text/plain": [ - "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", - "┃\u001b[1m \u001b[0m\u001b[1m Test metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", - "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", - "│\u001b[36m \u001b[0m\u001b[36m test_loss \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 6171157594112.0 \u001b[0m\u001b[35m \u001b[0m│\n", - "└───────────────────────────┴───────────────────────────┘\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "torch_utils.process.test_model(trainer, model, loaders)" + "easy_torch.process.test_model(trainer, model, loaders)" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "exp_utils.exp.save_experiment(cfg)" + "easy_exp.exp.save_experiment(cfg)" ] }, { diff --git a/rec_utils b/rec_utils deleted file mode 160000 index e86544d..0000000 --- a/rec_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e86544d1bc5f45e0d6b9bb0f61376e5fb0ab12dc diff --git a/setup.py b/setup.py index 1c25dbb..ad4c6a8 100644 --- a/setup.py +++ b/setup.py @@ -20,12 +20,12 @@ description='Easy Lightning: Simplify AI-Deep learning with PyTorch Lightning', long_description=long_description, # Use the contents of README.md as the long description long_description_content_type="text/markdown", - version='0.0.1', # Specify the version of your package - install_requires=install_requires+['data_utils @ git+https://github.com/siciliano-diag/data_utils.git', - 'exp_utils @ git+https://github.com/siciliano-diag/exp_utils.git', - 'rec_utils @ git+https://github.com/siciliano-diag/rec_utils.git', - 'torch_utils @ git+https://github.com/siciliano-diag/torch_utils.git'], # List of required dependencies - url='https://github.com/siciliano-diag/easy_lightning.git', # Replace with the URL of your GitHub repository + version='1.0.0', # Specify the version of your package + install_requires=install_requires+['easy_data @ git+https://github.com/federicosiciliano/easy_data.git', + 'easy_exp @ git+https://github.com/federicosiciliano/easy_exp.git', + 'easy_rec @ git+https://github.com/federicosiciliano/easy_rec.git', + 'easy_torch @ git+https://github.com/federicosiciliano/easy_torch.git'], # List of required dependencies + url='https://github.com/federicosiciliano/easy_lightning.git', # Replace with the URL of your GitHub repository author='Federico Siciliano, Federico Carmignani', author_email='siciliano@diag.uniroma1.it, carmignanifederico@gmail.com', keywords=['DeepLearning', 'MachineLearning', 'PyTorch', 'Lightning', 'AI'] # Keywords related to your package diff --git a/torch_utils b/torch_utils deleted file mode 160000 index 90a0eac..0000000 --- a/torch_utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 90a0eac5eb542c2e3092379ede818ca32cdb322e