From 90408215bf9bfed81b663d3ba90b40093fc4e1f9 Mon Sep 17 00:00:00 2001
From: VOC Shy <143907302+VOCdevShy@users.noreply.github.com>
Date: Sun, 14 Apr 2024 13:28:56 +0000
Subject: [PATCH] Version 2.1.0 (Check the "Patch Note.md" file in the "doc"
folder for more information)
---
.local/share/rippkgs-index.sqlite | 1 -
Doc/Patch Note.md | 13 +-
QInjection.py | 221 ++++++++++++++++++++-----
README.md | 11 +-
Tools/Q-injection.md | 23 +++
Tools/injection.md | 17 --
Tools/{map.txt => map.md} | 36 +++-
__pycache__/QInjection.cpython-310.pyc | Bin 1284 -> 1530 bytes
main.py | 19 ++-
9 files changed, 267 insertions(+), 74 deletions(-)
delete mode 120000 .local/share/rippkgs-index.sqlite
create mode 100644 Tools/Q-injection.md
delete mode 100644 Tools/injection.md
rename Tools/{map.txt => map.md} (78%)
diff --git a/.local/share/rippkgs-index.sqlite b/.local/share/rippkgs-index.sqlite
deleted file mode 120000
index 29e24ef..0000000
--- a/.local/share/rippkgs-index.sqlite
+++ /dev/null
@@ -1 +0,0 @@
-/etc/replit/rippkgs-indices/stable-23_05.sqlite
\ No newline at end of file
diff --git a/Doc/Patch Note.md b/Doc/Patch Note.md
index c4c15fe..05891b0 100644
--- a/Doc/Patch Note.md
+++ b/Doc/Patch Note.md
@@ -1,4 +1,4 @@
-## Index
+## Update Index
- Ver. 1.0.0
- [Ver. 1.0.1][1]
- [Ver. 1.1.0][2]
@@ -8,7 +8,8 @@
- [Ver. 1.2.3][6]
- [Ver. 1.2.4][7]
- [Ver. 2.0.0][8]
-
+ -[Ver. 2.1.0][9]
+
[1]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/Doc/Patch%20Note.md#version-101-minor-update-date-29022024-at-1020-am-cetutc1 "Version 1.0.1"
[2]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/Doc/Patch%20Note.md#version-110-major-update-date-01032024-at-0953-am-cetutc1 "Version 1.1.0"
[3]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/Doc/Patch%20Note.md#version-120-major-update-date-03032024-at-220-pm-cetutc1 "Version 1.2.0"
@@ -17,6 +18,7 @@
[6]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/Doc/Patch%20Note.md#version-123-minor-update-date-20032024-at-555-pm-cetutc1 "Version 1.2.3"
[7]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/Doc/Patch%20Note.md#version-124-minor-update-date-12042024-at-500-pm-cetutc1 "Version 1.2.4"
[8]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/Doc/Patch%20Note.md#version-200-major-update-date-13042024-at-653-pm-cetutc1 "Version 2.0.0"
+[9]: https "Version 2.1.0"
### Version: 1.0.1 (minor update) date: 29/02/2024 at: 10:20 a.m (CET(UTC+1)):
- Half resolved the bug n°4 (Check the `Bug List.md files` to see the explication of te fix)
@@ -91,4 +93,9 @@ These Q-Tables are in the `QInjection.py` file
(_For more information about check the_ `injection.md` _file in the_ `Tools` _folder_)
- Bug/Problem found and listed (For more informations about the Bugs/Problems checks the `Bug list.md` in the `doc` folder):
- - n°10
\ No newline at end of file
+ - n°10
+
+## Version 2.1.0 (medium update) date: 14/04/2024 at 15:20 p.m (CET(UTC+1)):
+
+- Adding the Q-Injection for the 2x2 (_Check the_ `Q-injection.md` _in the_ `Tools` _folder to find more informations about the Q-Injection_)
+- Re-layout of `maps.txt` now `maps.md` to allow a better copy -> paste of the maps (_Check the_ `maps.md` _in the_ `Tools` _folder to see the re-layout_)
\ No newline at end of file
diff --git a/QInjection.py b/QInjection.py
index c22c720..95e7b5f 100644
--- a/QInjection.py
+++ b/QInjection.py
@@ -9,45 +9,192 @@ def Q_Injection():
"FFFH",
"HFFG"
], map_name="4x4", render_mode="human", is_slippery=False)
-
+ map_name = "4x4"
# Initialiser les Q-tables
random_qtable = np.random.rand(env.observation_space.n, env.action_space.n)
- trained_qtable = np.array([
+ if map_name == "2x2":
+ trained_qtable = np.array([
+ [0.80999099, 0., 0.89999847, 0.80998402],
+ [0.80983744, 0.99999952, 0.89990816, 0.89998968],
+ [0, 0., 0., 0. ],
+ [0., 0., 0., 0. ],
+ ])
+ two_r_qtable = np.array([
+ [0., 0., 0., 0.],
+ [0., 0.5, 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ ])
+ if map_name == "4x4":
+ trained_qtable = np.array([
[0.15831711, 0.25917878, 0.11922523, 0.15359299],
- [0.15965656, 0., 0.02154766, 0.09100679],
- [0.02719321, 0., 0.00526721, 0.0120596],
- [0.00709388, 0., 0.00526721, 0.],
- [0.25900676, 0.31536319, 0., 0.18784653],
- [0., 0., 0., 0.],
- [0., 0.2278125, 0., 0.],
- [0., 0., 0., 0.],
- [0.18020325, 0., 0.37108521, 0.23696614],
- [0.17203403, 0.03075469, 0.43664063, 0.],
- [0., 0.590625, 0., 0.10251563],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0.03075469, 0.3375, 0.06834375],
- [0., 0.3375, 0.875, 0.10125],
- [0., 0., 0., 0.]
- ])
- two_r_qtable = np.array([
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0., 0.],
- [0., 0., 0.25, 0.],
- [0., 0.25, 0.75, 0.],
- [0., 0., 0., 0.]
- ])
-
+ [0.15965656, 0., 0.02154766, 0.09100679],
+ [0.02719321, 0., 0.00526721, 0.0120596 ],
+ [0.00709388, 0., 0.00526721, 0. ],
+ [0.25900676, 0.31536319, 0., 0.18784653],
+ [0., 0., 0., 0. ],
+ [0., 0.2278125, 0., 0. ],
+ [0., 0., 0., 0. ],
+ [0.18020325, 0., 0.37108521, 0.23696614],
+ [0.17203403, 0.03075469, 0.43664063, 0. ],
+ [0., 0.590625, 0., 0.10251563],
+ [0., 0., 0., 0. ],
+ [0., 0., 0., 0. ],
+ [0., 0.03075469, 0.3375, 0.06834375],
+ [0., 0.3375, 0.875, 0.10125 ],
+ [0., 0., 0., 0. ]
+ ])
+ two_r_qtable = np.array([
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0.25, 0.],
+ [0., 0.25, 0.75, 0.],
+ [0., 0., 0., 0.]
+ ])
+ if map_name == "8x8": # WIP
+ trained_qtable = np.array([
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ ])
+ two_r_qtable = np.array([
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ [0., 0., 0., 0.],
+ ])
+
# Retourner les Q-tables initialisées
return random_qtable, trained_qtable, two_r_qtable
diff --git a/README.md b/README.md
index 135063e..c68973e 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,12 @@
If you want more information about Q-Learning and the Frozen Lake game, you could read the article found on medium, he help me a lot to understand how works the Q-Learning: Q-Learning For Beginners by Maxime Labonne
+
+ Welcome on one of the most ultra-detailed version of the
+
Frozen-Lake Q-Learning project
+
Ver. 2.1.0
+
+
## Table of content
- [About][1]
- [Datas][2]
@@ -21,11 +27,6 @@
[4]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake?tab=readme-ov-file#for-those-who-are-interested-by-the-calculation-of-the-q-table-here-is-an-explication--hope-it-helps-you-to-understand-the-q-learning "formula"
[5]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake?tab=readme-ov-file#maps "maps"
[6]: https://github.com/VOCdevShy/Q-Learning_Frozen_Lake/blob/main/README.md#q-injection "Q-Injection"
-
- Welcome on one of the most ultra-detailed version of the
-
Frozen-Lake Q-Learning project
-
Ver. 2.0.0
-
## About
Like his name is telling, the project is an ultra-detailed version of the Frozen-Lake Q-Learning project.
diff --git a/Tools/Q-injection.md b/Tools/Q-injection.md
new file mode 100644
index 0000000..47f8c5b
--- /dev/null
+++ b/Tools/Q-injection.md
@@ -0,0 +1,23 @@
+# Q-Injection
+
+This project is made to inject different types of Q-Tables in the Q-Table for the Q-Learning Frozen Lake.
+
+### Here's the type of Q-Table you could inject:
+- Virgin Q-Table
+- Randomize Q-Table
+- Optimized Q-Table
+- 2 Rewards Q-Table
+
+### **⚠️WARNING!⚠️**
+1. This project doesn't work fully with the area 8x8, 16x16 and could contain bugs.
+2. To have a full access to the 2x2 and 4x4 Q-Tables to inject, you need to change the environment in the `main.py` and `QInjection.py` files.
+
+(_The maps are in the_ `maps.md` _file in the_ `Tools` _folder. You only need to copy the map you want and replace it in the code by paste it._)
+
+## Goals
+This project have three goals,
+1. Test Q-Table
+2. Upgrade them again
+3. Do it both train before test after
+
+blue
\ No newline at end of file
diff --git a/Tools/injection.md b/Tools/injection.md
deleted file mode 100644
index fd87f3c..0000000
--- a/Tools/injection.md
+++ /dev/null
@@ -1,17 +0,0 @@
-# Q-Injection
-
-This project is made to inject different types of Q-Tables in the Q-Table for the Q-Learning Frozen Lake.
-
-### Here's the type of Q-Table you could inject:
-- Virgin Q-Table
-- Randomize Q-Table
-- Optimized Q-Table
-- 2 Rewards Q-Table
-
-**WARNING!** This project doesn't work fully with the area 2x2, 8x8, 16x16 and could contain bugs.
-
-## Goals
-This project have three goals,
-1. Test Q-Table
-2. Upgrade them again
-3. Do it both train before test after
\ No newline at end of file
diff --git a/Tools/map.txt b/Tools/map.md
similarity index 78%
rename from Tools/map.txt
rename to Tools/map.md
index 6f95601..da0c8e6 100644
--- a/Tools/map.txt
+++ b/Tools/map.md
@@ -1,16 +1,24 @@
Map Generation:
2X2:
+```
env = gym.make('FrozenLake-v1', desc=[
"SF",
"HG"
],
map_name="2x2",
render_mode="human", is_slippery=False)
-Random: env = gym.make('FrozenLake-v1', desc=generate_random_map(size=2), map_name="2x2",
+map_name = "2x2"
+```
+Random 2x2:
+```
+ env = gym.make('FrozenLake-v1', desc=generate_random_map(size=2), map_name="2x2",
render_mode="human", is_slippery=False)
+map_name = "2x2"
+```
4X4:
+```
env = gym.make('FrozenLake-v1', desc=[
"SFFF",
"FHFH",
@@ -19,10 +27,17 @@ env = gym.make('FrozenLake-v1', desc=[
],
map_name="4x4",
render_mode="human", is_slippery=False)
-Random: env = gym.make('FrozenLake-v1', desc=generate_random_map(size=4), map_name="4x4",
+map_name = "4x4"
+```
+Random 4x4:
+```
+env = gym.make('FrozenLake-v1', desc=generate_random_map(size=4), map_name="4x4",
render_mode="human", is_slippery=False)
+map_name = "4x4"
+```
8X8:
+```
env = gym.make('FrozenLake-v1', desc=[
"SFFFFFFF",
"FFFFFFFF",
@@ -35,10 +50,17 @@ env = gym.make('FrozenLake-v1', desc=[
],
map_name="8x8",
render_mode="human", is_slippery=False)
-Random: env = gym.make('FrozenLake-v1', desc=generate_random_map(size=8), map_name="8x8",
+map_name = "8x8"
+```
+Random 8x8:
+```
+env = gym.make('FrozenLake-v1', desc=generate_random_map(size=8), map_name="8x8",
render_mode="human", is_slippery=False)
+map_name = "8x8"
+```
16X16:
+```
env = gym.make('FrozenLake-v1', desc=[
"SFFFFFFFFFFFFFFF",
"FFFFFFFFFFFFFFFF",
@@ -59,5 +81,11 @@ env = gym.make('FrozenLake-v1', desc=[
],
map_name="16x16",
render_mode="human", is_slippery=False)
-Random : env = gym.make('FrozenLake-v1', desc=generate_random_map(size=16), map_name="16x16",
+map_name = "16x16"
+```
+Random 16x16:
+```
+env = gym.make('FrozenLake-v1', desc=generate_random_map(size=16), map_name="16x16",
render_mode="human", is_slippery=False)
+map_name = "16x16"
+```
\ No newline at end of file
diff --git a/__pycache__/QInjection.cpython-310.pyc b/__pycache__/QInjection.cpython-310.pyc
index 59d0a87f8b06cbe3a1e5abc3830ee626fd5797aa..0f899c191766aa96b3bd65b32212064cdb27d66f 100644
GIT binary patch
delta 599
zcmZqS`o+zc&&$ij00fRlq|>apCh`UHu>yI*KjbD)j*b+$;t5|B7YM5(S(wKu8G^HljF(!vbF&kAFX|klJMK)U&y?$x`92mx6
zkltvVJpJUEH}>ftueZ)l21)@{X>$KDF?jXTK0VFX_us$&@9ony111~sfK=T$nojX7Dn04UAF#Ky|@n}tn`iGz`akqrj_u&^Q6
a9BeF;C$Pv&ZeU@X+`*#8C^~sPODF)X$DvUG
delta 340
zcmeyx-NMC}&&$ij00gt=NTnTQpU4-;#{%RD1FQ*`n*#^lKtm<-vT1AWNAFqxUT
zYVvX>@y&J2`iyccx0qA&%5L!%CFZ5%=f)S7Bqrsg-V(?!D9OytteSj|C7CY>sELP(
bjg{@U7zYdE