Can evaluate NEAT networks in a loop

#24 #23 @nazaruka I committed this to my own new branch dev_schrum, but you can feel free to checkout that branch and edit things, or merge this into your master branch. I didn't commit this to master since I though there might be uncommitted changes on your machine (don't do that in the future).

In any case, running this version of NSGAII will now show Sonic being controlled by a NEAT network. Evolution definitely doesn't work yet though. Basically, what this does is create a NEAT population, but then the network you see evaluated is just the same one over an over rather than distinct members of the population. Also, after all of the individuals are evaluated, the code crashes because the fitness objectives are still the ones that originally came with the NSGA-II code. Here is what you should do next if you think working with this would be useful (but you can mess with your own code instead of that seems more fruitful ... just do that in your own branch):

1. Add the behavior characterization tracking to this.
2. Make the evaluate method return the behavior characterization along with the fitness.

If you get that far, then we can move onto more interesting stuff.

NSGA2/NSGAII.py

@@ -9,6 +9,14 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
+# Copied from Training.py in the sonicNEAT repo
+import retro
+import numpy as np
+import cv2
+import neat
+import pickle
+
+
 #First function to optimize
 def function1(x):
     value = -x**2
@@ -100,26 +108,94 @@ def mutation(solution):
         solution = min_x+(max_x-min_x)*random.random()
     return solution
 
+# Copied from Training.py in the sonicNEAT repo
+def evaluate(env,net):
+    ob = env.reset()
+    ac = env.action_space.sample()
+    inx, iny, inc = env.observation_space.shape
+    inx = int(inx/8)
+    iny = int(iny/8)
+    current_max_fitness = 0
+    fitness_current = 0
+    frame = 0
+    counter = 0
+    xpos = 0
+    done = False
+
+    while not done:
+
+        env.render()
+        frame += 1
+        ob = cv2.resize(ob, (inx, iny))
+        ob = cv2.cvtColor(ob, cv2.COLOR_BGR2GRAY)
+        ob = np.reshape(ob, (inx,iny))
+
+        imgarray = np.ndarray.flatten(ob)
+
+        nnOutput = net.activate(imgarray)
+
+        ob, rew, done, info = env.step(nnOutput)
+
+
+        xpos = info['x']
+
+        if xpos >= 65664:
+                fitness_current += 10000000
+                done = True
+
+        fitness_current += rew
+
+        if fitness_current > current_max_fitness:
+            current_max_fitness = fitness_current
+            counter = 0
+        else:
+            counter += 1
+
+        if done or counter == 250:
+            done = True
+            print(fitness_current)
+
+    # Add code to return the behavior characterization as well.
+    return fitness_current
+
 def random_genome(n):
     # n is the number of weights
     return np.random.rand(1,n)
 
 if __name__ == '__main__':
     #Main program starts here
 
+    # Copied from Training.py in the sonicNEAT repo
+    env = retro.make(game = "SonicTheHedgehog-Genesis", state = "GreenHillZone.Act1")
+    imgarray = []
+    xpos_end = 0
+    config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
+                     neat.DefaultSpeciesSet, neat.DefaultStagnation,
+                     'config-feedforward')
+    p = neat.Population(config)
+    example_genome = p.population[1]
+
     # Schrum: Makes these small to test at first
-    pop_size = 5
-    max_gen = 5
+    pop_size = 3
+    max_gen = 3
 
     #Initialization
     # Schrum: This will need to be replaced with initialization for the network weights ... probably from -1 to 1, but how many will you need? Depends on network architecture.
     num_weights = 10 # What should this actually be?
     solution=[random_genome(num_weights) for i in range(0,pop_size)]
     gen_no=0
     while(gen_no<max_gen):
-
-        #Schrum: Add a call to a method you make that evaluates sonic and returns both the score (RL Return) and behavior characterization
-
+        # Copied/Adapted from Training.py in the sonicNEAT repo
+        for genome in solution:
+            # Replace example_genome with genome once the types are worked out.
+            # Of course, eventually we won't be using NEAT at all ... so maybe skip that step.
+            net = neat.nn.recurrent.RecurrentNetwork.create(example_genome, config)
+            fitness = evaluate(env,net)
+            # Schrum: Need to get the behavior characterization as well, and then actually do something with 
+            #         both that and the fitness
+
+        # Schrum: This code will crash because the solution variable has nothing to do with the type of fitness that is supposed to be calculated here.
+
         function1_values = [function1(solution[i])for i in range(0,pop_size)] # Schrum: Repalce with the RL returns for each individual
 
         # Schrum: Here is where you have to compare all of the behavior characterizations to get the diversity/novelty scores.

NSGA2/config-feedforward

@@ -0,0 +1,83 @@
+[NEAT]
+fitness_criterion     = max
+fitness_threshold     = 10000000
+pop_size              = 20
+reset_on_extinction   = True
+
+[DefaultGenome]
+# node activation options
+activation_default      = sigmoid
+activation_mutate_rate  = 0.05
+activation_options      = sigmoid gauss 
+#abs clamped cube exp gauss hat identity inv log relu sigmoid sin softplus square tanh
+
+# node aggregation options
+aggregation_default     = random
+aggregation_mutate_rate = 0.05
+aggregation_options     = sum product min max mean median maxabs
+
+# node bias options
+bias_init_mean          = 0.05
+bias_init_stdev         = 1.0
+bias_max_value          = 30.0
+bias_min_value          = -30.0
+bias_mutate_power       = 0.5
+bias_mutate_rate        = 0.7
+bias_replace_rate       = 0.1
+
+# genome compatibility options
+compatibility_disjoint_coefficient = 1.0
+compatibility_weight_coefficient   = 0.5
+
+# connection add/remove rates
+conn_add_prob           = 0.5
+conn_delete_prob        = 0.5
+
+# connection enable options
+enabled_default         = True
+enabled_mutate_rate     = 0.5
+
+feed_forward            = False
+#initial_connection      = unconnected
+initial_connection      = partial_nodirect 0.5
+
+# node add/remove rates
+node_add_prob           = 0.5
+node_delete_prob        = 0.2
+
+# network parameters
+num_hidden              = 0
+num_inputs              = 1120
+num_outputs             = 12
+
+# node response options
+response_init_mean      = 1.0
+response_init_stdev     = 0.05
+response_max_value      = 30.0
+response_min_value      = -30.0
+response_mutate_power   = 0.1
+response_mutate_rate    = 0.75
+response_replace_rate   = 0.1
+
+# connection weight options
+weight_init_mean        = 0.1
+weight_init_stdev       = 1.0
+weight_max_value        = 30
+weight_min_value        = -30
+weight_mutate_power     = 0.5
+weight_mutate_rate      = 0.8
+weight_replace_rate     = 0.1
+
+[DefaultSpeciesSet]
+compatibility_threshold = 2.5
+
+[DefaultStagnation]
+species_fitness_func = max
+max_stagnation       = 50
+species_elitism      = 0
+
+[DefaultReproduction]
+elitism            = 3
+survival_threshold = 0.3
+
+