1-9 Circle Decision Boundary Classification: SVC vs TensorFlow

Sklearn의 datasets에서 지원하는 make_circle 루틴을 사용하여 원형의 hyperplane 즉 Decision Boundary 가 원형인 비선형 예제를 다루어 보자. 헤더 영역에서 sklearn.datasets를 불러들이면 make_circles 루틴 사용이 가능하다.

Make_circles 루틴에서 noise 는 내부 원에 해당하는 데이터와 외곽 원에 속한 데이터를 서로 반대 영역에 섞어 넣는 효과를 준다. 그 값이 0.0 이라면 그러한 섞임이 전혀 없음을 뜻한다. factor 는 내부 원과 외부 원의 축척 비율로서 default 가 0.8 이다.

XOR 코드의 입력 데이터 준비 부분에서 이정도만 수정하면 Circle 문제를 실행할 수 있다. 다음 결과를 관찰해 보면 타원형으로 보이지만 좌표축을 보면 원임을 즉각 알 수 있다. Accuracy 는 비슷해 보인다.

 

#Circle_SVM_softmax_01.py

from sklearn import __version__ as sklearn_version
from sklearn.svm import SVC
from sklearn.datasets import make_circles
import numpy as np
from matplotlib.colors import ListedColormap
import matplotlib.pyplot as plt

def plot_decision_regions(X, y, classifier, test_idx=None, resolution=0.01):

    # setup marker generator and color map
    markers = ('s', 'x', 'o', '^', 'v')
    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
    cmap = ListedColormap(colors[:len(np.unique(y))])

    # plot the decision surface
    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    #print(x1_min.shape)
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),
                           np.arange(x2_min, x2_max, resolution))
    #print('xx1,xx2=',xx1,xx2)
    P=np.array([xx1.ravel(), xx2.ravel()])
    #print('P=',P.T)
    #print(classifier)
    Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)
    #print('Z.shape=',Z.shape)
    #print('Z=',Z)
    Z = Z.reshape(xx1.shape)
    #print('Z.reshape',Z)
    plt.contourf(xx1, xx2, Z, alpha=0.3, cmap=cmap)
    plt.grid(True)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())

    for idx, cl in enumerate(np.unique(y)):
        plt.scatter(x=X[y == cl, 0],
                    y=X[y == cl, 1],
                    alpha=0.8,
                    c=colors[idx],
                    marker=markers[idx],
                    label=cl,
                    edgecolor='black')

    # highlight test samples
    if test_idx:
        # plot all samples
        X_test, y_test = X[test_idx, :], y[test_idx]

        plt.scatter(X_test[:, 0],
                    X_test[:, 1],
                    c='',
                    edgecolor='black',
                    alpha=1.0,
                    linewidth=1,
                    marker='o',
                    s=100,
                    label='test set')

np.random.seed(1)
pts = 200
X_xor, y_xor = make_circles(n_samples=pts, noise=0.2, factor=0.5,
                            random_state=1)

plt.scatter(X_xor[:,0], X_xor[:, 1])
plt.show()

#Support Vector Machine "RBF"
svm = SVC(kernel='rbf', random_state=0, gamma=2.0, C=1.0)
svm.fit(X_xor, y_xor)
plot_decision_regions(X_xor, y_xor,classifier=svm)

plt.legend(loc='upper left')
plt.tight_layout()
plt.show()

#Following is method to convert numpy array to tensor
import tensorflow as tf
import time

start_time = time.time()

def fn(X,W1,b1,W2,b2):
    #print(X.shape)
    hypothesis =  tf.nn.softmax((tf.matmul(X, W2) + b2)*(tf.matmul(X, W1) + b1))
    return hypothesis

def tf_plot_decision_regions(X_xor, y, hypothesis, predicted, test_idx=None, resolution=0.01):
   
    # setup marker generator and color map
    markers = ('s', 'x', 'o', '^', 'v')
    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
    cmap = ListedColormap(colors[:len(np.unique(y))])
    # plot the decision surface
    x1_min, x1_max = X_xor[:, 0].min() - 1, X_xor[:, 0].max() + 1
    x2_min, x2_max = X_xor[:, 1].min() - 1, X_xor[:, 1].max() + 1
    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),
                           np.arange(x2_min, x2_max, resolution))
    print(xx1.shape)
    XX = np.array([xx1.ravel(), xx2.ravel()]).T
    #print(XX.shape)
    h, p = sess.run([hypothesis, predicted], feed_dict={X: XX })

    #print(p)
    p = p.reshape(xx1.shape)
    plt.contourf(xx1, xx2, p, alpha=0.3, cmap=cmap)
    plt.grid(True)
    plt.xlim(xx1.min(), xx1.max())
    plt.ylim(xx2.min(), xx2.max())

    for idx, cl in enumerate(np.unique(y)):
        plt.scatter(x=X_xor[y == cl, 0],
                    y=X_xor[y == cl, 1],
                    alpha=0.8,
                    c=colors[idx],
                    marker=markers[idx],
                    label=cl,
                    edgecolor='black')

    # highlight test samples
    if test_idx:
        # plot all samples
        X_test, y_test = X_xor[test_idx, :], y[test_idx]

        plt.scatter(X_test[:, 0],
                    X_test[:, 1],
                    c='',
                    edgecolor='black',
                    alpha=1.0,
                    linewidth=1,
                    marker='o',
                    s=100,
                    label='test set')

#Training Data
Y_xor = np.zeros([pts,2])
Y_xor[y_xor == 0] = [1., 0.]
Y_xor[y_xor == 1] = [0., 1.]
#print(Y_xor)

X_xor = np.float32(X_xor)
Y_xor = np.float32(Y_xor)
#print(Y_xor)

#print(X_xor.shape)
#print(Y_xor.shape)

#hyperparameter
learning_rate = 0.0001
training_epochs = 40000
display_steps = 10000

#Network parameters
n_input = 2
dof1 = 2
#Graph Nodes
X = tf.placeholder("float", [None, n_input])
Y = tf.placeholder("float", [None, dof1])

#Weights and Biases, model, loss and optimizer
W0 = tf.Variable(tf.random_normal([n_input, dof1], stddev=0.01))
b0 = tf.Variable(tf.random_normal([dof1], stddev=0.01))
W1 = tf.Variable(tf.random_normal([n_input, dof1], stddev=0.01))
b1 = tf.Variable(tf.random_normal([dof1], stddev=0.01))
W2 = tf.Variable(tf.random_normal([n_input, dof1], stddev=0.01))
b2 = tf.Variable(tf.random_normal([dof1], stddev=0.01))

#hypothesis
hypothesis = fn(X,W1,b1,W2,b2)

# cost/loss function
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits
                      (logits=hypothesis, labels=Y))

#optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(cost)
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cost)

predicted = tf.cast(hypothesis[:, 0] < 0.5, dtype=tf.float32)
print(predicted.shape)

#predicted =  tf.cast(hypothesis > 0.5, dtype=tf.float32)

accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(hypothesis,1), tf.argmax(Y,1)), dtype=tf.float32))

#Initializing global variables
init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init)
   
    for epoch in range(training_epochs):
        _, c, w1, B1, w2, B2 = sess.run([optimizer, cost, W1, b1, W2, b2], feed_dict={X: X_xor, Y: Y_xor})
        #if(epoch + 1) % display_steps == 0:
            #print( "Epoch: ", (epoch+1), "Cost: ", c, w1, B1, w2, B2 )
    print("Optimization Finished!")

    # Accuracy report
    h, p, a = sess.run([hypothesis, predicted, accuracy],feed_dict={X: X_xor, Y: Y_xor})

    #print("\nHypothesis:\n", h, "\nCorrect:\n", p, "\nAccuracy: ", a)
    #print(p.shape)
    p = p.reshape(pts,1)
    #print(p)
    tf_plot_decision_regions(X_xor, y_xor, hypothesis, predicted)
    plt.legend(loc='upper left')
    plt.tight_layout()
    plt.show()

sess.close()

end_time = time.time()
print( "\nCompleted in ", end_time - start_time , " seconds")