XOR.ipynb

'''

인공지능 

Multi-Layer perceptron 구현

서울시립대학교 컴퓨터과학부 2014920017 박인수

AND, OR, XOR 비교 그래프 구현

XOR 구현

'''

import tensorflow as tf
import numpy as np
from numpy import *
from numpy.random import multivariate_normal
import matplotlib.pyplot as plt
import pandas as pd
from pandas import Series, DataFrame

# Hidden Layer 노드 개수
num_units = 2

x = tf.placeholder(tf.float32, [None, 2])
w1 = tf.Variable(tf.truncated_normal([2, num_units]))
b1 = tf.Variable(tf.zeros([num_units]))
hidden1 = tf.nn.sigmoid(tf.matmul(x,w1) + b1)
w0 =  tf.Variable(tf.zeros([num_units, 1]))
b0 = tf.Variable(tf.zeros([1]))
p = tf.nn.sigmoid(tf.matmul(hidden1, w0) + b0)
t = tf.placeholder(tf.float32, [None, 1])
loss = -tf.reduce_sum(t*tf.log(p)+(1-t)*tf.log(1-p))

# 학습 방법 결정
# train_step = tf.train.AdamOptimizer().minimize(loss)
# train_step = tf.train.GradientDescentOptimizer(0.05).minimize(loss)
train_step = tf.train.MomentumOptimizer(0.05, 0.05).minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

# Iteration Graph 세팅
fig1 = plt.figure(figsize=(8,8))
iter_subplot = fig1.add_subplot(1, 1, 1)
loss_valarr = []

# Learning Graph 세팅
fig2 = plt.figure(figsize=(8,8))
learning_subplot = fig2.add_subplot(1, 1, 1)
learning_subplot.set_xlim(-0.2, 1.2)
learning_subplot.set_ylim(-0.2, 1.2)
linex = np.linspace(-0.2,1.2, 400)

train_x = np.zeros([50,2])
train_t = np.zeros([50,1])
    
i = 0
for _ in range(100000):
  i+= 1
  # 난수 데이터 50개를 매번 학습시킬 때마다 mini_batch로 실행
  for row in range(0, 50):
    for col in range(0,2):
      train_x[row][col] = np.random.randint(2)
      if((train_x[row][0] == 1 and train_x[row][1] == 0) or (train_x[row][0] == 0 and train_x[row][1] == 1)):
        train_t[row][0] = 1
      else:
        train_t[row][0] = 0
  sess.run(train_step, feed_dict={x:train_x, t:train_t})
  loss_val = sess.run(loss, feed_dict={x:train_x, t:train_t})
  loss_valarr.append(loss_val)
  # 500번 실행할 때마다 그래프에 표시
  if i % 500 == 0:
    #loss가 15보다 크면 w1 초기화 후 다시 학습 시작
    if loss_val > 10:
      w1 = tf.Variable(tf.truncated_normal([2, num_units]))
      b1 = tf.Variable(tf.zeros([num_units]))
      w0 =  tf.Variable(tf.zeros([num_units, 1]))
      b0 = tf.Variable(tf.zeros([1]))
      hidden1 = tf.nn.sigmoid(tf.matmul(x,w1) + b1)
      p = tf.nn.sigmoid(tf.matmul(hidden1, w0) + b0)
      t = tf.placeholder(tf.float32, [None, 1])      
      loss = -tf.reduce_sum(t*tf.log(p)+(1-t)*tf.log(1-p))
      train_step = tf.train.MomentumOptimizer(0.05, 0.05).minimize(loss)
      sess.close()
      sess = tf.Session()
      sess.run(tf.global_variables_initializer())
      continue
      
    print('Step: %d, Loss: %f' %(i, loss_val))
    w0_val, b0_val, w1_val, b1_val = sess.run([w0, b0, w1, b1])
    w01_val, w02_val = w0_val[0][0], w0_val[1][0]
    w11_val, w12_val, w13_val, w14_val = w1_val[0][0], w1_val[1][0], w1_val[0][1], w1_val[1][1]
    b11_val, b12_val = b1_val[0], b1_val[1]
    
    liney = -(w11_val*linex/w12_val + b11_val/w12_val)
    learning_subplot.plot(linex, liney, linewidth=2, label='%d 1st' %(i))
    learning_subplot.legend(loc='upper left')
    liney = -(w13_val*linex/w14_val + b12_val/w14_val)
    learning_subplot.plot(linex, liney, linewidth=2, label='%d 2nd' %(i))
    learning_subplot.legend(loc='upper right')
    
    #loss가 0.1보다 작아지면 종료
    if loss_val < 0.5:
      break
    
# loss 배열을  그래프로 표시
iter_subplot.plot(range(len(loss_valarr)), loss_valarr,linewidth=2, label='Error')
iter_subplot.legend(loc='upper left')

# X1, X2 좌표에 대해 데이터를 점으로 표시 - (0,0), (0,1), (1,0), (1,1)
train_set = hstack((train_x, train_t))
df = DataFrame(train_set)
df.columns = ["x1", "x2", "t"]          
train_set0 = df[df['t']==0]
train_set1 = df[df['t']==1] 
learning_subplot.scatter(train_set1.x1, train_set1.x2, marker='o')
learning_subplot.scatter(train_set0.x1, train_set0.x2, marker='x')

# w를 파일로 저장
w_set = vstack((w11_val, w12_val, w13_val, w14_val, b11_val, b12_val, w01_val, w02_val, b0_val))
w_set.tofile('test.txt', sep = '\t')

'''
i = 0
count = 0
correct = 0
learning_rate = 0.05
momentum = 0.05

while True:
  while True:
    for _ in range(50000):    
      for row in range(0, 50):
        for col in range(0,2):
          train_x[row][col] = np.random.randint(2)
          if((train_x[row][0] == 1 and train_x[row][1] == 0) or (train_x[row][0] == 0 and train_x[row][1] == 1)):
            train_t[row][0] = 1
          else:
            train_t[row][0] = 0

      i += 1
      sess.run(train_step, feed_dict={x:train_x, t:train_t})
      loss_val = sess.run(loss, feed_dict={x:train_x, t:train_t})

      if i % 500 == 0:
        if loss_val < 5:
          correct = correct + 1
        w1 = tf.Variable(tf.truncated_normal([2, num_units]))
        b1 = tf.Variable(tf.zeros([num_units]))
        w0 =  tf.Variable(tf.zeros([num_units, 1]))
        b0 = tf.Variable(tf.zeros([1]))
        hidden1 = tf.nn.sigmoid(tf.matmul(x,w1) + b1)

        p = tf.nn.sigmoid(tf.matmul(hidden1, w0) + b0)
        t = tf.placeholder(tf.float32, [None, 1])
        loss = -tf.reduce_sum(t*tf.log(p)+(1-t)*tf.log(1-p))
        train_step = tf.train.MomentumOptimizer(0.05, 0.05).minimize(loss)
        #train_step = tf.train.GradientDescentOptimizer(0.05).minimize(loss)
        sess.close()
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())
        count = count + 1
    print('Learning Rate : %f, Momentum : %f, Count : %d, Correct : %f' %(learning_rate, momentum, count, correct/float(count)))
    momentum += 0.01
    count = 0
    correct = 0    
    if(momentum > 0.09):
      break
  learning_rate += 0.01
  momentum = 0.01
  if(learning_rate > 0.09):
    break
    
'''