Virtual Camera update: Rllib and onnx support, added onnx

examples/VirtualCamera/Env.tscn

@@ -56,6 +56,7 @@ transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 63, 0, -50)
 [node name="Sync" type="Node" parent="."]
 process_priority = -1
 script = ExtResource("2")
+onnx_model_path = "VirtualCamera.onnx"
 
 [node name="Camera" type="Camera3D" parent="."]
 transform = Transform3D(1, 0, 0, 0, 0.0220418, 0.999757, 0, -0.999757, 0.0220418, 25.3538, 75.4275, -10.0795)

examples/VirtualCamera/Player.gd

@@ -113,7 +113,8 @@ func reset():
 
 
 func update_reward():
-	ai_controller.reward -= 0.01  # step penalty
+	#ai_controller.reward -= 0.01  # step penalty
+	pass
 
 
 func calculate_translation(other_pad_translation: Vector3) -> Vector3:

examples/VirtualCamera/Player.tscn

@@ -2,8 +2,8 @@
 
 [ext_resource type="Script" path="res://Player.gd" id="1"]
 [ext_resource type="PackedScene" uid="uid://b4hphc8dab5h" path="res://Robot.tscn" id="2"]
-[ext_resource type="PackedScene" uid="uid://b30vsuwotx0u2" path="res://VirtualCamera.tscn" id="3_tv0v5"]
 [ext_resource type="Script" path="res://AIController3D.gd" id="4_rq7t7"]
+[ext_resource type="PackedScene" uid="uid://baaywi3arsl2m" path="res://addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.tscn" id="4_ybcln"]
 
 [sub_resource type="CapsuleShape3D" id="1"]
 radius = 1.6
@@ -27,8 +27,11 @@ mesh = SubResource("2")
 [node name="Robot" parent="." instance=ExtResource("2")]
 transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0, -1.43952, 0.0576344)
 
-[node name="RGBCameraSensor3D" parent="." instance=ExtResource("3_tv0v5")]
-
 [node name="AIController3D" type="Node3D" parent="."]
 script = ExtResource("4_rq7t7")
 reset_after = 10000
+
+[node name="RGBCameraSensor3D" parent="." instance=ExtResource("4_ybcln")]
+training_mode = true
+render_image_resolution = Vector2(10, 10)
+displayed_image_scale_factor = Vector2(20, 20)

examples/VirtualCamera/VirtualCamera.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Godot.NET.Sdk/4.3.0-dev.5">
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework Condition=" '$(GodotTargetPlatform)' == 'android' ">net7.0</TargetFramework>
+    <TargetFramework Condition=" '$(GodotTargetPlatform)' == 'ios' ">net8.0</TargetFramework>
+    <EnableDynamicLoading>true</EnableDynamicLoading>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.15.1" />
+  </ItemGroup>
+</Project>

examples/VirtualCamera/VirtualCamera.sln

@@ -0,0 +1,19 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2012
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VirtualCamera", "VirtualCamera.csproj", "{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+	Debug|Any CPU = Debug|Any CPU
+	ExportDebug|Any CPU = ExportDebug|Any CPU
+	ExportRelease|Any CPU = ExportRelease|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}.ExportDebug|Any CPU.ActiveCfg = ExportDebug|Any CPU
+		{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}.ExportDebug|Any CPU.Build.0 = ExportDebug|Any CPU
+		{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}.ExportRelease|Any CPU.ActiveCfg = ExportRelease|Any CPU
+		{716A4AE2-20D9-4B9A-BB9F-FA26A57B6249}.ExportRelease|Any CPU.Build.0 = ExportRelease|Any CPU
+	EndGlobalSection
+EndGlobal

examples/VirtualCamera/addons/godot_rl_agents/controller/ai_controller_2d.gd

@@ -1,8 +1,29 @@
 extends Node2D
 class_name AIController2D
 
+enum ControlModes { INHERIT_FROM_SYNC, HUMAN, TRAINING, ONNX_INFERENCE, RECORD_EXPERT_DEMOS }
+@export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC
+@export var onnx_model_path := ""
 @export var reset_after := 1000
 
+@export_group("Record expert demos mode options")
+## Path where the demos will be saved. The file can later be used for imitation learning.
+@export var expert_demo_save_path: String
+## The action that erases the last recorded episode from the currently recorded data.
+@export var remove_last_episode_key: InputEvent
+## Action will be repeated for n frames. Will introduce control lag if larger than 1.
+## Can be used to ensure that action_repeat on inference and training matches
+## the recorded demonstrations.
+@export var action_repeat: int = 1
+
+@export_group("Multi-policy mode options")
+## Allows you to set certain agents to use different policies.
+## Changing has no effect with default SB3 training. Works with Rllib example.
+## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md
+@export var policy_name: String = "shared_policy"
+
+var onnx_model: ONNXModel
+
 var heuristic := "human"
 var done := false
 var reward := 0.0
@@ -11,72 +32,88 @@ var needs_reset := false
 
 var _player: Node2D
 
+
 func _ready():
 	add_to_group("AGENT")
-
+
+
 func init(player: Node2D):
 	_player = player
-
+
+
 #-- Methods that need implementing using the "extend script" option in Godot --#
 func get_obs() -> Dictionary:
-	assert(false, "the get_obs method is not implemented when extending from ai_controller") 
-	return {"obs":[]}
+	assert(false, "the get_obs method is not implemented when extending from ai_controller")
+	return {"obs": []}
 
-func get_reward() -> float:	
-	assert(false, "the get_reward method is not implemented when extending from ai_controller") 
+
+func get_reward() -> float:
+	assert(false, "the get_reward method is not implemented when extending from ai_controller")
 	return 0.0
-
+
+
 func get_action_space() -> Dictionary:
-	assert(false, "the get get_action_space method is not implemented when extending from ai_controller") 
+	assert(
+		false,
+		"the get get_action_space method is not implemented when extending from ai_controller"
+	)
 	return {
-		"example_actions_continous" : {
-			"size": 2,
-			"action_type": "continuous"
-		},
-		"example_actions_discrete" : {
-			"size": 2,
-			"action_type": "discrete"
-		},
-		}
-
-func set_action(action) -> void:	
-	assert(false, "the get set_action method is not implemented when extending from ai_controller") 	
+		"example_actions_continous": {"size": 2, "action_type": "continuous"},
+		"example_actions_discrete": {"size": 2, "action_type": "discrete"},
+	}
+
+
+func set_action(action) -> void:
+	assert(false, "the set_action method is not implemented when extending from ai_controller")
+
+
+#-----------------------------------------------------------------------------#
+
+
+#-- Methods that sometimes need implementing using the "extend script" option in Godot --#
+# Only needed if you are recording expert demos with this AIController
+func get_action() -> Array:
+	assert(false, "the get_action method is not implemented in extended AIController but demo_recorder is used")
+	return []
+
 # -----------------------------------------------------------------------------#
-	
+
 func _physics_process(delta):
 	n_steps += 1
 	if n_steps > reset_after:
 		needs_reset = true
-
+
+
 func get_obs_space():
 	# may need overriding if the obs space is complex
 	var obs = get_obs()
 	return {
-		"obs": {
-			"size": [len(obs["obs"])],
-			"space": "box"
-		},
+		"obs": {"size": [len(obs["obs"])], "space": "box"},
 	}
 
+
 func reset():
 	n_steps = 0
 	needs_reset = false
 
+
 func reset_if_done():
 	if done:
 		reset()
-
+
+
 func set_heuristic(h):
 	# sets the heuristic from "human" or "model" nothing to change here
 	heuristic = h
 
+
 func get_done():
 	return done
-
+
+
 func set_done_false():
 	done = false
 
+
 func zero_reward():
 	reward = 0.0
-
-

examples/VirtualCamera/addons/godot_rl_agents/controller/ai_controller_3d.gd

@@ -1,8 +1,29 @@
 extends Node3D
 class_name AIController3D
 
+enum ControlModes { INHERIT_FROM_SYNC, HUMAN, TRAINING, ONNX_INFERENCE, RECORD_EXPERT_DEMOS }
+@export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC
+@export var onnx_model_path := ""
 @export var reset_after := 1000
 
+@export_group("Record expert demos mode options")
+## Path where the demos will be saved. The file can later be used for imitation learning.
+@export var expert_demo_save_path: String
+## The action that erases the last recorded episode from the currently recorded data.
+@export var remove_last_episode_key: InputEvent
+## Action will be repeated for n frames. Will introduce control lag if larger than 1.
+## Can be used to ensure that action_repeat on inference and training matches
+## the recorded demonstrations.
+@export var action_repeat: int = 1
+
+@export_group("Multi-policy mode options")
+## Allows you to set certain agents to use different policies.
+## Changing has no effect with default SB3 training. Works with Rllib example.
+## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md
+@export var policy_name: String = "shared_policy"
+
+var onnx_model: ONNXModel
+
 var heuristic := "human"
 var done := false
 var reward := 0.0
@@ -11,70 +32,89 @@ var needs_reset := false
 
 var _player: Node3D
 
+
 func _ready():
 	add_to_group("AGENT")
-
+
+
 func init(player: Node3D):
 	_player = player
-
+
+
 #-- Methods that need implementing using the "extend script" option in Godot --#
 func get_obs() -> Dictionary:
-	assert(false, "the get_obs method is not implemented when extending from ai_controller") 
-	return {"obs":[]}
+	assert(false, "the get_obs method is not implemented when extending from ai_controller")
+	return {"obs": []}
 
-func get_reward() -> float:	
-	assert(false, "the get_reward method is not implemented when extending from ai_controller") 
+
+func get_reward() -> float:
+	assert(false, "the get_reward method is not implemented when extending from ai_controller")
 	return 0.0
-
+
+
 func get_action_space() -> Dictionary:
-	assert(false, "the get get_action_space method is not implemented when extending from ai_controller") 
+	assert(
+		false,
+		"the get_action_space method is not implemented when extending from ai_controller"
+	)
 	return {
-		"example_actions_continous" : {
-			"size": 2,
-			"action_type": "continuous"
-		},
-		"example_actions_discrete" : {
-			"size": 2,
-			"action_type": "discrete"
-		},
-		}
-
-func set_action(action) -> void:	
-	assert(false, "the get set_action method is not implemented when extending from ai_controller") 	
+		"example_actions_continous": {"size": 2, "action_type": "continuous"},
+		"example_actions_discrete": {"size": 2, "action_type": "discrete"},
+	}
+
+
+func set_action(action) -> void:
+	assert(false, "the set_action method is not implemented when extending from ai_controller")
+
+
+#-----------------------------------------------------------------------------#
+
+
+#-- Methods that sometimes need implementing using the "extend script" option in Godot --#
+# Only needed if you are recording expert demos with this AIController
+func get_action() -> Array:
+	assert(false, "the get_action method is not implemented in extended AIController but demo_recorder is used")
+	return []
+
 # -----------------------------------------------------------------------------#
-
+
+
 func _physics_process(delta):
 	n_steps += 1
 	if n_steps > reset_after:
 		needs_reset = true
-
+
+
 func get_obs_space():
 	# may need overriding if the obs space is complex
 	var obs = get_obs()
 	return {
-		"obs": {
-			"size": [len(obs["obs"])],
-			"space": "box"
-		},
+		"obs": {"size": [len(obs["obs"])], "space": "box"},
 	}
 
+
 func reset():
 	n_steps = 0
 	needs_reset = false
 
+
 func reset_if_done():
 	if done:
 		reset()
-
+
+
 func set_heuristic(h):
 	# sets the heuristic from "human" or "model" nothing to change here
 	heuristic = h
 
+
 func get_done():
 	return done
-
+
+
 func set_done_false():
 	done = false
 
+
 func zero_reward():
 	reward = 0.0

examples/VirtualCamera/addons/godot_rl_agents/onnx/csharp/ONNXInference.cs

@@ -19,16 +19,22 @@ public partial class ONNXInference : GodotObject
 
 		private SessionOptions SessionOpt;
 
-		//init function
-		/// <include file='docs/ONNXInference.xml' path='docs/members[@name="ONNXInference"]/Initialize/*'/>
-		public void Initialize(string Path, int BatchSize)
+        /// <summary>
+        /// init function
+        /// </summary>
+        /// <param name="Path"></param>
+        /// <param name="BatchSize"></param>
+        /// <returns>Returns the output size of the model</returns>
+        public int Initialize(string Path, int BatchSize)
 		{
 			modelPath = Path;
 			batchSize = BatchSize;
             SessionOpt = SessionConfigurator.MakeConfiguredSessionOptions();
             session = LoadModel(modelPath);
+            return session.OutputMetadata["output"].Dimensions[1];
+        }
+
 
-		}
 		/// <include file='docs/ONNXInference.xml' path='docs/members[@name="ONNXInference"]/Run/*'/>
 		public Godot.Collections.Dictionary<string, Godot.Collections.Array<float>> RunInference(Godot.Collections.Array<float> obs, int state_ins)
 		{