lab-10-2-2-mnist_nn_deep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Xavier Deep network version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager mode on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size).\\\n",
    "    repeat()\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x)).\\\n",
    "    repeat()\n",
    "\n",
    "train_iterator = train_dataset.make_one_shot_iterator()\n",
    "test_iterator = test_dataset.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_deep'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.4442, train_loss: 2.05249906, train_accuracy: 0.6328, test_Accuracy: 0.3818\n",
      "Epoch: [ 0] [    1/  468] time: 0.6793, train_loss: 1.93041635, train_accuracy: 0.6172, test_Accuracy: 0.5202\n",
      "Epoch: [ 0] [    2/  468] time: 0.9141, train_loss: 1.68725586, train_accuracy: 0.7500, test_Accuracy: 0.6194\n",
      "Epoch: [ 0] [    3/  468] time: 1.1409, train_loss: 1.41266954, train_accuracy: 0.7266, test_Accuracy: 0.6493\n",
      "Epoch: [ 0] [    4/  468] time: 1.3676, train_loss: 1.18582010, train_accuracy: 0.6953, test_Accuracy: 0.7176\n",
      "Epoch: [ 0] [    5/  468] time: 1.5918, train_loss: 0.95347941, train_accuracy: 0.7891, test_Accuracy: 0.7844\n",
      "Epoch: [ 0] [    6/  468] time: 1.8217, train_loss: 0.74819553, train_accuracy: 0.8125, test_Accuracy: 0.7582\n",
      "Epoch: [ 0] [    7/  468] time: 2.0467, train_loss: 0.61154401, train_accuracy: 0.8281, test_Accuracy: 0.7833\n",
      "Epoch: [ 0] [    8/  468] time: 2.2675, train_loss: 0.61696053, train_accuracy: 0.8047, test_Accuracy: 0.7329\n",
      "Epoch: [ 0] [    9/  468] time: 2.4988, train_loss: 0.74705696, train_accuracy: 0.7656, test_Accuracy: 0.7527\n",
      "Epoch: [ 0] [   10/  468] time: 2.7324, train_loss: 0.64232647, train_accuracy: 0.8125, test_Accuracy: 0.7914\n",
      "Epoch: [ 0] [   11/  468] time: 2.9686, train_loss: 0.56650764, train_accuracy: 0.7578, test_Accuracy: 0.8086\n",
      "Epoch: [ 0] [   12/  468] time: 3.1897, train_loss: 0.57275212, train_accuracy: 0.8359, test_Accuracy: 0.8008\n",
      "Epoch: [ 0] [   13/  468] time: 3.4057, train_loss: 0.54453456, train_accuracy: 0.8359, test_Accuracy: 0.8181\n",
      "Epoch: [ 0] [   14/  468] time: 3.6364, train_loss: 0.37354526, train_accuracy: 0.8672, test_Accuracy: 0.8370\n",
      "Epoch: [ 0] [   15/  468] time: 3.8720, train_loss: 0.44905001, train_accuracy: 0.8281, test_Accuracy: 0.8413\n",
      "Epoch: [ 0] [   16/  468] time: 4.0986, train_loss: 0.49822363, train_accuracy: 0.8281, test_Accuracy: 0.8243\n",
      "Epoch: [ 0] [   17/  468] time: 4.3163, train_loss: 0.61060381, train_accuracy: 0.8203, test_Accuracy: 0.8732\n",
      "Epoch: [ 0] [   18/  468] time: 4.5398, train_loss: 0.39486212, train_accuracy: 0.8828, test_Accuracy: 0.8723\n",
      "Epoch: [ 0] [   19/  468] time: 4.7634, train_loss: 0.38840914, train_accuracy: 0.9062, test_Accuracy: 0.8529\n",
      "Epoch: [ 0] [   20/  468] time: 4.9937, train_loss: 0.35018367, train_accuracy: 0.8906, test_Accuracy: 0.8296\n",
      "Epoch: [ 0] [   21/  468] time: 5.2264, train_loss: 0.51245594, train_accuracy: 0.8125, test_Accuracy: 0.8700\n",
      "Epoch: [ 0] [   22/  468] time: 5.4408, train_loss: 0.24256152, train_accuracy: 0.9297, test_Accuracy: 0.8828\n",
      "Epoch: [ 0] [   23/  468] time: 5.6733, train_loss: 0.43990484, train_accuracy: 0.8672, test_Accuracy: 0.8828\n",
      "Epoch: [ 0] [   24/  468] time: 5.8834, train_loss: 0.43246940, train_accuracy: 0.8438, test_Accuracy: 0.8843\n",
      "Epoch: [ 0] [   25/  468] time: 6.1099, train_loss: 0.38432562, train_accuracy: 0.8906, test_Accuracy: 0.8771\n",
      "Epoch: [ 0] [   26/  468] time: 6.3215, train_loss: 0.34956402, train_accuracy: 0.9219, test_Accuracy: 0.8736\n",
      "Epoch: [ 0] [   27/  468] time: 6.5331, train_loss: 0.31082648, train_accuracy: 0.9219, test_Accuracy: 0.8811\n",
      "Epoch: [ 0] [   28/  468] time: 6.7457, train_loss: 0.33460373, train_accuracy: 0.9141, test_Accuracy: 0.8912\n",
      "Epoch: [ 0] [   29/  468] time: 6.9821, train_loss: 0.30613306, train_accuracy: 0.9375, test_Accuracy: 0.8913\n",
      "Epoch: [ 0] [   30/  468] time: 7.2076, train_loss: 0.40066603, train_accuracy: 0.8828, test_Accuracy: 0.8845\n",
      "Epoch: [ 0] [   31/  468] time: 7.4318, train_loss: 0.26313818, train_accuracy: 0.9141, test_Accuracy: 0.8793\n",
      "Epoch: [ 0] [   32/  468] time: 7.6650, train_loss: 0.31208426, train_accuracy: 0.8984, test_Accuracy: 0.8830\n",
      "Epoch: [ 0] [   33/  468] time: 7.8854, train_loss: 0.37260449, train_accuracy: 0.8828, test_Accuracy: 0.8925\n",
      "Epoch: [ 0] [   34/  468] time: 8.1095, train_loss: 0.33495778, train_accuracy: 0.8672, test_Accuracy: 0.8934\n",
      "Epoch: [ 0] [   35/  468] time: 8.3423, train_loss: 0.27398932, train_accuracy: 0.8828, test_Accuracy: 0.9057\n",
      "Epoch: [ 0] [   36/  468] time: 8.5727, train_loss: 0.20400712, train_accuracy: 0.9453, test_Accuracy: 0.9096\n",
      "Epoch: [ 0] [   37/  468] time: 8.8006, train_loss: 0.28922883, train_accuracy: 0.9141, test_Accuracy: 0.9043\n",
      "Epoch: [ 0] [   38/  468] time: 9.0133, train_loss: 0.28440210, train_accuracy: 0.9141, test_Accuracy: 0.9017\n",
      "Epoch: [ 0] [   39/  468] time: 9.2462, train_loss: 0.24891603, train_accuracy: 0.9141, test_Accuracy: 0.9041\n",
      "Epoch: [ 0] [   40/  468] time: 9.4668, train_loss: 0.32510191, train_accuracy: 0.9297, test_Accuracy: 0.9097\n",
      "Epoch: [ 0] [   41/  468] time: 9.6704, train_loss: 0.22598577, train_accuracy: 0.9141, test_Accuracy: 0.9116\n",
      "Epoch: [ 0] [   42/  468] time: 9.9011, train_loss: 0.27010673, train_accuracy: 0.8984, test_Accuracy: 0.9166\n",
      "Epoch: [ 0] [   43/  468] time: 10.1325, train_loss: 0.27177966, train_accuracy: 0.9141, test_Accuracy: 0.9059\n",
      "Epoch: [ 0] [   44/  468] time: 10.3609, train_loss: 0.28677177, train_accuracy: 0.8828, test_Accuracy: 0.8892\n",
      "Epoch: [ 0] [   45/  468] time: 10.5883, train_loss: 0.26920494, train_accuracy: 0.9062, test_Accuracy: 0.8896\n",
      "Epoch: [ 0] [   46/  468] time: 10.8140, train_loss: 0.26334357, train_accuracy: 0.9062, test_Accuracy: 0.8960\n",
      "Epoch: [ 0] [   47/  468] time: 11.0460, train_loss: 0.23434947, train_accuracy: 0.9297, test_Accuracy: 0.9042\n",
      "Epoch: [ 0] [   48/  468] time: 11.2596, train_loss: 0.18511131, train_accuracy: 0.9375, test_Accuracy: 0.9066\n",
      "Epoch: [ 0] [   49/  468] time: 11.4831, train_loss: 0.50069070, train_accuracy: 0.8594, test_Accuracy: 0.9097\n",
      "Epoch: [ 0] [   50/  468] time: 11.6949, train_loss: 0.26590931, train_accuracy: 0.8984, test_Accuracy: 0.9134\n",
      "Epoch: [ 0] [   51/  468] time: 11.9211, train_loss: 0.23455450, train_accuracy: 0.9297, test_Accuracy: 0.9161\n",
      "Epoch: [ 0] [   52/  468] time: 12.1539, train_loss: 0.25061911, train_accuracy: 0.9297, test_Accuracy: 0.9089\n",
      "Epoch: [ 0] [   53/  468] time: 12.3665, train_loss: 0.24591726, train_accuracy: 0.9375, test_Accuracy: 0.9005\n",
      "Epoch: [ 0] [   54/  468] time: 12.5804, train_loss: 0.36767107, train_accuracy: 0.9141, test_Accuracy: 0.9146\n",
      "Epoch: [ 0] [   55/  468] time: 12.8019, train_loss: 0.25725555, train_accuracy: 0.9141, test_Accuracy: 0.9288\n",
      "Epoch: [ 0] [   56/  468] time: 13.0263, train_loss: 0.23370585, train_accuracy: 0.9297, test_Accuracy: 0.9276\n",
      "Epoch: [ 0] [   57/  468] time: 13.2515, train_loss: 0.31216016, train_accuracy: 0.9297, test_Accuracy: 0.9169\n",
      "Epoch: [ 0] [   58/  468] time: 13.4635, train_loss: 0.30888891, train_accuracy: 0.9219, test_Accuracy: 0.9144\n",
      "Epoch: [ 0] [   59/  468] time: 13.6778, train_loss: 0.26934111, train_accuracy: 0.9141, test_Accuracy: 0.9195\n",
      "Epoch: [ 0] [   60/  468] time: 13.9027, train_loss: 0.17624927, train_accuracy: 0.9453, test_Accuracy: 0.9236\n",
      "Epoch: [ 0] [   61/  468] time: 14.1367, train_loss: 0.22730497, train_accuracy: 0.9297, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [   62/  468] time: 14.3624, train_loss: 0.19185452, train_accuracy: 0.9453, test_Accuracy: 0.9233\n",
      "Epoch: [ 0] [   63/  468] time: 14.5812, train_loss: 0.19475916, train_accuracy: 0.9297, test_Accuracy: 0.9286\n",
      "Epoch: [ 0] [   64/  468] time: 14.8194, train_loss: 0.17102247, train_accuracy: 0.9609, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [   65/  468] time: 15.0356, train_loss: 0.31814590, train_accuracy: 0.9141, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [   66/  468] time: 15.2629, train_loss: 0.24114597, train_accuracy: 0.9375, test_Accuracy: 0.9296\n",
      "Epoch: [ 0] [   67/  468] time: 15.4839, train_loss: 0.25597858, train_accuracy: 0.9297, test_Accuracy: 0.9313\n",
      "Epoch: [ 0] [   68/  468] time: 15.7025, train_loss: 0.15937175, train_accuracy: 0.9609, test_Accuracy: 0.9288\n",
      "Epoch: [ 0] [   69/  468] time: 15.9239, train_loss: 0.15169677, train_accuracy: 0.9609, test_Accuracy: 0.9282\n",
      "Epoch: [ 0] [   70/  468] time: 16.1487, train_loss: 0.24358395, train_accuracy: 0.9375, test_Accuracy: 0.9293\n",
      "Epoch: [ 0] [   71/  468] time: 16.3621, train_loss: 0.25459892, train_accuracy: 0.9297, test_Accuracy: 0.9275\n",
      "Epoch: [ 0] [   72/  468] time: 16.5908, train_loss: 0.18833514, train_accuracy: 0.9297, test_Accuracy: 0.9262\n",
      "Epoch: [ 0] [   73/  468] time: 16.8140, train_loss: 0.17040908, train_accuracy: 0.9531, test_Accuracy: 0.9173\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 17.0395, train_loss: 0.21832043, train_accuracy: 0.9219, test_Accuracy: 0.9035\n",
      "Epoch: [ 0] [   75/  468] time: 17.2529, train_loss: 0.32004800, train_accuracy: 0.8672, test_Accuracy: 0.9204\n",
      "Epoch: [ 0] [   76/  468] time: 17.4781, train_loss: 0.21903375, train_accuracy: 0.9297, test_Accuracy: 0.9135\n",
      "Epoch: [ 0] [   77/  468] time: 17.7076, train_loss: 0.26172560, train_accuracy: 0.9062, test_Accuracy: 0.9051\n",
      "Epoch: [ 0] [   78/  468] time: 17.9462, train_loss: 0.21773836, train_accuracy: 0.9375, test_Accuracy: 0.8978\n",
      "Epoch: [ 0] [   79/  468] time: 18.1716, train_loss: 0.34637338, train_accuracy: 0.8984, test_Accuracy: 0.9044\n",
      "Epoch: [ 0] [   80/  468] time: 18.3915, train_loss: 0.24700768, train_accuracy: 0.9453, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [   81/  468] time: 18.6069, train_loss: 0.23192921, train_accuracy: 0.9297, test_Accuracy: 0.9335\n",
      "Epoch: [ 0] [   82/  468] time: 18.8295, train_loss: 0.29736102, train_accuracy: 0.8984, test_Accuracy: 0.9339\n",
      "Epoch: [ 0] [   83/  468] time: 19.0607, train_loss: 0.31178871, train_accuracy: 0.9141, test_Accuracy: 0.9224\n",
      "Epoch: [ 0] [   84/  468] time: 19.2869, train_loss: 0.23283041, train_accuracy: 0.9141, test_Accuracy: 0.9100\n",
      "Epoch: [ 0] [   85/  468] time: 19.5087, train_loss: 0.27160999, train_accuracy: 0.9219, test_Accuracy: 0.9193\n",
      "Epoch: [ 0] [   86/  468] time: 19.7342, train_loss: 0.21747437, train_accuracy: 0.9375, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [   87/  468] time: 19.9602, train_loss: 0.21956950, train_accuracy: 0.9609, test_Accuracy: 0.9362\n",
      "Epoch: [ 0] [   88/  468] time: 20.1792, train_loss: 0.14242157, train_accuracy: 0.9609, test_Accuracy: 0.9345\n",
      "Epoch: [ 0] [   89/  468] time: 20.3990, train_loss: 0.19534807, train_accuracy: 0.9219, test_Accuracy: 0.9298\n",
      "Epoch: [ 0] [   90/  468] time: 20.6152, train_loss: 0.21854465, train_accuracy: 0.9219, test_Accuracy: 0.9278\n",
      "Epoch: [ 0] [   91/  468] time: 20.8388, train_loss: 0.15652683, train_accuracy: 0.9688, test_Accuracy: 0.9269\n",
      "Epoch: [ 0] [   92/  468] time: 21.0622, train_loss: 0.13013768, train_accuracy: 0.9453, test_Accuracy: 0.9296\n",
      "Epoch: [ 0] [   93/  468] time: 21.2907, train_loss: 0.22439453, train_accuracy: 0.9219, test_Accuracy: 0.9323\n",
      "Epoch: [ 0] [   94/  468] time: 21.5135, train_loss: 0.21071047, train_accuracy: 0.9453, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [   95/  468] time: 21.7325, train_loss: 0.14633003, train_accuracy: 0.9688, test_Accuracy: 0.9423\n",
      "Epoch: [ 0] [   96/  468] time: 21.9600, train_loss: 0.08634470, train_accuracy: 0.9766, test_Accuracy: 0.9421\n",
      "Epoch: [ 0] [   97/  468] time: 22.1835, train_loss: 0.14013897, train_accuracy: 0.9688, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [   98/  468] time: 22.4105, train_loss: 0.13980487, train_accuracy: 0.9531, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [   99/  468] time: 22.6277, train_loss: 0.21910344, train_accuracy: 0.9297, test_Accuracy: 0.9293\n",
      "Epoch: [ 0] [  100/  468] time: 22.8503, train_loss: 0.19582853, train_accuracy: 0.9375, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [  101/  468] time: 23.0663, train_loss: 0.19232342, train_accuracy: 0.9297, test_Accuracy: 0.9344\n",
      "Epoch: [ 0] [  102/  468] time: 23.2862, train_loss: 0.09006768, train_accuracy: 0.9766, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [  103/  468] time: 23.5056, train_loss: 0.20535573, train_accuracy: 0.9375, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [  104/  468] time: 23.7253, train_loss: 0.17537981, train_accuracy: 0.9609, test_Accuracy: 0.9416\n",
      "Epoch: [ 0] [  105/  468] time: 23.9463, train_loss: 0.11805163, train_accuracy: 0.9688, test_Accuracy: 0.9428\n",
      "Epoch: [ 0] [  106/  468] time: 24.1641, train_loss: 0.07446890, train_accuracy: 0.9688, test_Accuracy: 0.9419\n",
      "Epoch: [ 0] [  107/  468] time: 24.3820, train_loss: 0.25199711, train_accuracy: 0.9297, test_Accuracy: 0.9452\n",
      "Epoch: [ 0] [  108/  468] time: 24.5970, train_loss: 0.18659122, train_accuracy: 0.9297, test_Accuracy: 0.9407\n",
      "Epoch: [ 0] [  109/  468] time: 24.8184, train_loss: 0.19178995, train_accuracy: 0.9375, test_Accuracy: 0.9358\n",
      "Epoch: [ 0] [  110/  468] time: 25.0330, train_loss: 0.12825243, train_accuracy: 0.9688, test_Accuracy: 0.9273\n",
      "Epoch: [ 0] [  111/  468] time: 25.2557, train_loss: 0.23180917, train_accuracy: 0.9297, test_Accuracy: 0.9290\n",
      "Epoch: [ 0] [  112/  468] time: 25.4798, train_loss: 0.18464771, train_accuracy: 0.9453, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  113/  468] time: 25.6997, train_loss: 0.21853626, train_accuracy: 0.9609, test_Accuracy: 0.9393\n",
      "Epoch: [ 0] [  114/  468] time: 25.9185, train_loss: 0.29174900, train_accuracy: 0.9219, test_Accuracy: 0.9427\n",
      "Epoch: [ 0] [  115/  468] time: 26.1424, train_loss: 0.12358031, train_accuracy: 0.9766, test_Accuracy: 0.9437\n",
      "Epoch: [ 0] [  116/  468] time: 26.3630, train_loss: 0.23147163, train_accuracy: 0.9375, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  117/  468] time: 26.5801, train_loss: 0.20433819, train_accuracy: 0.9531, test_Accuracy: 0.9363\n",
      "Epoch: [ 0] [  118/  468] time: 26.8028, train_loss: 0.27793548, train_accuracy: 0.9375, test_Accuracy: 0.9395\n",
      "Epoch: [ 0] [  119/  468] time: 27.0253, train_loss: 0.07795866, train_accuracy: 0.9922, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  120/  468] time: 27.2424, train_loss: 0.15232459, train_accuracy: 0.9688, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  121/  468] time: 27.4585, train_loss: 0.07568970, train_accuracy: 0.9922, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  122/  468] time: 27.6753, train_loss: 0.13192144, train_accuracy: 0.9609, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  123/  468] time: 27.8972, train_loss: 0.20791322, train_accuracy: 0.9453, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  124/  468] time: 28.1308, train_loss: 0.12790596, train_accuracy: 0.9609, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  125/  468] time: 28.3491, train_loss: 0.12340897, train_accuracy: 0.9609, test_Accuracy: 0.9414\n",
      "Epoch: [ 0] [  126/  468] time: 28.5786, train_loss: 0.14853404, train_accuracy: 0.9453, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [  127/  468] time: 28.8005, train_loss: 0.12986916, train_accuracy: 0.9609, test_Accuracy: 0.9438\n",
      "Epoch: [ 0] [  128/  468] time: 29.0239, train_loss: 0.15917617, train_accuracy: 0.9766, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  129/  468] time: 29.2460, train_loss: 0.13025479, train_accuracy: 0.9531, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  130/  468] time: 29.4672, train_loss: 0.17654946, train_accuracy: 0.9531, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  131/  468] time: 29.6915, train_loss: 0.25160718, train_accuracy: 0.9219, test_Accuracy: 0.9432\n",
      "Epoch: [ 0] [  132/  468] time: 29.9223, train_loss: 0.12202396, train_accuracy: 0.9688, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  133/  468] time: 30.1591, train_loss: 0.25998685, train_accuracy: 0.9297, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  134/  468] time: 30.3819, train_loss: 0.15155441, train_accuracy: 0.9453, test_Accuracy: 0.9429\n",
      "Epoch: [ 0] [  135/  468] time: 30.5992, train_loss: 0.32089245, train_accuracy: 0.8906, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  136/  468] time: 30.8267, train_loss: 0.17301516, train_accuracy: 0.9453, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  137/  468] time: 31.0469, train_loss: 0.18742171, train_accuracy: 0.9609, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  138/  468] time: 31.2633, train_loss: 0.10434747, train_accuracy: 0.9531, test_Accuracy: 0.9423\n",
      "Epoch: [ 0] [  139/  468] time: 31.4884, train_loss: 0.11721679, train_accuracy: 0.9688, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  140/  468] time: 31.7055, train_loss: 0.12302766, train_accuracy: 0.9531, test_Accuracy: 0.9412\n",
      "Epoch: [ 0] [  141/  468] time: 31.9211, train_loss: 0.10339820, train_accuracy: 0.9766, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [  142/  468] time: 32.1368, train_loss: 0.25868577, train_accuracy: 0.9219, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [  143/  468] time: 32.3553, train_loss: 0.19348469, train_accuracy: 0.9453, test_Accuracy: 0.9360\n",
      "Epoch: [ 0] [  144/  468] time: 32.5718, train_loss: 0.18039174, train_accuracy: 0.9531, test_Accuracy: 0.9406\n",
      "Epoch: [ 0] [  145/  468] time: 32.7896, train_loss: 0.11074802, train_accuracy: 0.9766, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  146/  468] time: 33.0138, train_loss: 0.07491134, train_accuracy: 0.9844, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  147/  468] time: 33.2330, train_loss: 0.17060331, train_accuracy: 0.9375, test_Accuracy: 0.9486\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 33.4555, train_loss: 0.05204679, train_accuracy: 0.9922, test_Accuracy: 0.9438\n",
      "Epoch: [ 0] [  149/  468] time: 33.6690, train_loss: 0.25343269, train_accuracy: 0.9219, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  150/  468] time: 33.8874, train_loss: 0.10684416, train_accuracy: 0.9766, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  151/  468] time: 34.1057, train_loss: 0.12944846, train_accuracy: 0.9375, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  152/  468] time: 34.3239, train_loss: 0.17856355, train_accuracy: 0.9531, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  153/  468] time: 34.5476, train_loss: 0.07587519, train_accuracy: 0.9844, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  154/  468] time: 34.7711, train_loss: 0.21152380, train_accuracy: 0.9219, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  155/  468] time: 35.0009, train_loss: 0.15661627, train_accuracy: 0.9609, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  156/  468] time: 35.2231, train_loss: 0.13233268, train_accuracy: 0.9766, test_Accuracy: 0.9475\n",
      "Epoch: [ 0] [  157/  468] time: 35.4445, train_loss: 0.15522540, train_accuracy: 0.9609, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  158/  468] time: 35.6659, train_loss: 0.12406418, train_accuracy: 0.9766, test_Accuracy: 0.9482\n",
      "Epoch: [ 0] [  159/  468] time: 35.8929, train_loss: 0.14089745, train_accuracy: 0.9688, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  160/  468] time: 36.1122, train_loss: 0.10556926, train_accuracy: 0.9688, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  161/  468] time: 36.3319, train_loss: 0.04414634, train_accuracy: 0.9922, test_Accuracy: 0.9480\n",
      "Epoch: [ 0] [  162/  468] time: 36.5483, train_loss: 0.08590610, train_accuracy: 0.9688, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  163/  468] time: 36.7623, train_loss: 0.09487806, train_accuracy: 0.9844, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  164/  468] time: 36.9806, train_loss: 0.04851328, train_accuracy: 0.9922, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  165/  468] time: 37.1991, train_loss: 0.22721031, train_accuracy: 0.9297, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  166/  468] time: 37.4225, train_loss: 0.08967899, train_accuracy: 0.9688, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  167/  468] time: 37.6410, train_loss: 0.15103535, train_accuracy: 0.9609, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [  168/  468] time: 37.8626, train_loss: 0.09375620, train_accuracy: 0.9844, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [  169/  468] time: 38.0815, train_loss: 0.15148434, train_accuracy: 0.9609, test_Accuracy: 0.9426\n",
      "Epoch: [ 0] [  170/  468] time: 38.3000, train_loss: 0.20036358, train_accuracy: 0.9453, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [  171/  468] time: 38.5184, train_loss: 0.11125729, train_accuracy: 0.9531, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [  172/  468] time: 38.7377, train_loss: 0.09178940, train_accuracy: 0.9844, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  173/  468] time: 38.9537, train_loss: 0.11081444, train_accuracy: 0.9609, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  174/  468] time: 39.1764, train_loss: 0.14787403, train_accuracy: 0.9609, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  175/  468] time: 39.3966, train_loss: 0.10098834, train_accuracy: 0.9766, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  176/  468] time: 39.6186, train_loss: 0.12706584, train_accuracy: 0.9609, test_Accuracy: 0.9428\n",
      "Epoch: [ 0] [  177/  468] time: 39.8497, train_loss: 0.12193981, train_accuracy: 0.9609, test_Accuracy: 0.9419\n",
      "Epoch: [ 0] [  178/  468] time: 40.0738, train_loss: 0.10513002, train_accuracy: 0.9766, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  179/  468] time: 40.2929, train_loss: 0.18155472, train_accuracy: 0.9531, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  180/  468] time: 40.5197, train_loss: 0.10247546, train_accuracy: 0.9688, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  181/  468] time: 40.7411, train_loss: 0.16947046, train_accuracy: 0.9453, test_Accuracy: 0.9369\n",
      "Epoch: [ 0] [  182/  468] time: 40.9733, train_loss: 0.17461011, train_accuracy: 0.9609, test_Accuracy: 0.9325\n",
      "Epoch: [ 0] [  183/  468] time: 41.2201, train_loss: 0.10469441, train_accuracy: 0.9609, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [  184/  468] time: 41.4555, train_loss: 0.13815516, train_accuracy: 0.9531, test_Accuracy: 0.9435\n",
      "Epoch: [ 0] [  185/  468] time: 41.6838, train_loss: 0.10331583, train_accuracy: 0.9766, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  186/  468] time: 41.9454, train_loss: 0.17102686, train_accuracy: 0.9453, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  187/  468] time: 42.2034, train_loss: 0.14054735, train_accuracy: 0.9531, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  188/  468] time: 42.4429, train_loss: 0.21361439, train_accuracy: 0.9219, test_Accuracy: 0.9423\n",
      "Epoch: [ 0] [  189/  468] time: 42.6709, train_loss: 0.10862835, train_accuracy: 0.9609, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [  190/  468] time: 42.8948, train_loss: 0.09968789, train_accuracy: 0.9609, test_Accuracy: 0.9293\n",
      "Epoch: [ 0] [  191/  468] time: 43.1172, train_loss: 0.17431805, train_accuracy: 0.9297, test_Accuracy: 0.9329\n",
      "Epoch: [ 0] [  192/  468] time: 43.3382, train_loss: 0.16025613, train_accuracy: 0.9453, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  193/  468] time: 43.5558, train_loss: 0.17302710, train_accuracy: 0.9453, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  194/  468] time: 43.7674, train_loss: 0.10152128, train_accuracy: 0.9766, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  195/  468] time: 43.9964, train_loss: 0.11760332, train_accuracy: 0.9844, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  196/  468] time: 44.2169, train_loss: 0.11783737, train_accuracy: 0.9844, test_Accuracy: 0.9429\n",
      "Epoch: [ 0] [  197/  468] time: 44.4402, train_loss: 0.14087684, train_accuracy: 0.9375, test_Accuracy: 0.9400\n",
      "Epoch: [ 0] [  198/  468] time: 44.6615, train_loss: 0.20499174, train_accuracy: 0.9531, test_Accuracy: 0.9423\n",
      "Epoch: [ 0] [  199/  468] time: 44.8889, train_loss: 0.07206976, train_accuracy: 0.9844, test_Accuracy: 0.9475\n",
      "Epoch: [ 0] [  200/  468] time: 45.1113, train_loss: 0.13187441, train_accuracy: 0.9688, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  201/  468] time: 45.3417, train_loss: 0.09232134, train_accuracy: 0.9609, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  202/  468] time: 45.5686, train_loss: 0.18207854, train_accuracy: 0.9609, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  203/  468] time: 45.7985, train_loss: 0.20481008, train_accuracy: 0.9375, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  204/  468] time: 46.0222, train_loss: 0.10538368, train_accuracy: 0.9766, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [  205/  468] time: 46.2449, train_loss: 0.16770089, train_accuracy: 0.9688, test_Accuracy: 0.9435\n",
      "Epoch: [ 0] [  206/  468] time: 46.4676, train_loss: 0.08991968, train_accuracy: 0.9844, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  207/  468] time: 46.6877, train_loss: 0.19492511, train_accuracy: 0.9375, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  208/  468] time: 46.9128, train_loss: 0.09657705, train_accuracy: 0.9766, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  209/  468] time: 47.1313, train_loss: 0.09809151, train_accuracy: 0.9688, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  210/  468] time: 47.3523, train_loss: 0.14572793, train_accuracy: 0.9609, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  211/  468] time: 47.5733, train_loss: 0.08416952, train_accuracy: 0.9766, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  212/  468] time: 47.7961, train_loss: 0.11137106, train_accuracy: 0.9531, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  213/  468] time: 48.0195, train_loss: 0.19413829, train_accuracy: 0.9453, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  214/  468] time: 48.2409, train_loss: 0.09614500, train_accuracy: 0.9844, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  215/  468] time: 48.4595, train_loss: 0.11850545, train_accuracy: 0.9688, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  216/  468] time: 48.6753, train_loss: 0.06317431, train_accuracy: 0.9922, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  217/  468] time: 48.8991, train_loss: 0.14677571, train_accuracy: 0.9531, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  218/  468] time: 49.1114, train_loss: 0.08659574, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  219/  468] time: 49.3287, train_loss: 0.04500069, train_accuracy: 0.9844, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  220/  468] time: 49.5554, train_loss: 0.08015017, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  221/  468] time: 49.7762, train_loss: 0.08388603, train_accuracy: 0.9688, test_Accuracy: 0.9576\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 49.9938, train_loss: 0.05645273, train_accuracy: 0.9922, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  223/  468] time: 50.2111, train_loss: 0.19242182, train_accuracy: 0.9531, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  224/  468] time: 50.4353, train_loss: 0.10268214, train_accuracy: 0.9688, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  225/  468] time: 50.6588, train_loss: 0.03280109, train_accuracy: 1.0000, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  226/  468] time: 50.8787, train_loss: 0.15803194, train_accuracy: 0.9453, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  227/  468] time: 51.1099, train_loss: 0.15623832, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  228/  468] time: 51.3344, train_loss: 0.09782127, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  229/  468] time: 51.5648, train_loss: 0.05671175, train_accuracy: 0.9844, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  230/  468] time: 51.8023, train_loss: 0.22575526, train_accuracy: 0.9375, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  231/  468] time: 52.0417, train_loss: 0.07971391, train_accuracy: 0.9844, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  232/  468] time: 52.2600, train_loss: 0.17949715, train_accuracy: 0.9375, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  233/  468] time: 52.4816, train_loss: 0.08679979, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  234/  468] time: 52.7032, train_loss: 0.12286955, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  235/  468] time: 52.9251, train_loss: 0.11548427, train_accuracy: 0.9531, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  236/  468] time: 53.1465, train_loss: 0.10228884, train_accuracy: 0.9766, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  237/  468] time: 53.3758, train_loss: 0.10917015, train_accuracy: 0.9609, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  238/  468] time: 53.5972, train_loss: 0.08201773, train_accuracy: 0.9688, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  239/  468] time: 53.8149, train_loss: 0.06834941, train_accuracy: 0.9844, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  240/  468] time: 54.0339, train_loss: 0.15222064, train_accuracy: 0.9531, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  241/  468] time: 54.2521, train_loss: 0.16299321, train_accuracy: 0.9531, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  242/  468] time: 54.4873, train_loss: 0.10151526, train_accuracy: 0.9531, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  243/  468] time: 54.7088, train_loss: 0.08330199, train_accuracy: 0.9766, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  244/  468] time: 54.9324, train_loss: 0.06423654, train_accuracy: 0.9844, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  245/  468] time: 55.1559, train_loss: 0.18082823, train_accuracy: 0.9297, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  246/  468] time: 55.3808, train_loss: 0.15966590, train_accuracy: 0.9453, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  247/  468] time: 55.6040, train_loss: 0.15541971, train_accuracy: 0.9531, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  248/  468] time: 55.8262, train_loss: 0.09221433, train_accuracy: 0.9688, test_Accuracy: 0.9456\n",
      "Epoch: [ 0] [  249/  468] time: 56.0501, train_loss: 0.10700148, train_accuracy: 0.9688, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [  250/  468] time: 56.2712, train_loss: 0.11513714, train_accuracy: 0.9609, test_Accuracy: 0.9399\n",
      "Epoch: [ 0] [  251/  468] time: 56.4884, train_loss: 0.04553677, train_accuracy: 1.0000, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  252/  468] time: 56.7111, train_loss: 0.15880007, train_accuracy: 0.9688, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  253/  468] time: 56.9302, train_loss: 0.12904881, train_accuracy: 0.9609, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  254/  468] time: 57.1434, train_loss: 0.16023284, train_accuracy: 0.9375, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  255/  468] time: 57.3728, train_loss: 0.18649831, train_accuracy: 0.9219, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  256/  468] time: 57.5890, train_loss: 0.10219241, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  257/  468] time: 57.8158, train_loss: 0.13054225, train_accuracy: 0.9766, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  258/  468] time: 58.0433, train_loss: 0.12964928, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  259/  468] time: 58.2678, train_loss: 0.10476327, train_accuracy: 0.9688, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  260/  468] time: 58.4973, train_loss: 0.06114588, train_accuracy: 0.9922, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  261/  468] time: 58.7298, train_loss: 0.14213482, train_accuracy: 0.9609, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  262/  468] time: 58.9558, train_loss: 0.06457325, train_accuracy: 0.9922, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  263/  468] time: 59.1795, train_loss: 0.12144940, train_accuracy: 0.9609, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  264/  468] time: 59.4017, train_loss: 0.13074547, train_accuracy: 0.9766, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  265/  468] time: 59.6267, train_loss: 0.06598435, train_accuracy: 0.9766, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  266/  468] time: 59.8470, train_loss: 0.09211612, train_accuracy: 0.9766, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  267/  468] time: 60.0709, train_loss: 0.17384762, train_accuracy: 0.9375, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  268/  468] time: 60.2888, train_loss: 0.06192871, train_accuracy: 0.9844, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  269/  468] time: 60.5100, train_loss: 0.19513141, train_accuracy: 0.9531, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  270/  468] time: 60.7259, train_loss: 0.15693241, train_accuracy: 0.9453, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  271/  468] time: 60.9533, train_loss: 0.09964094, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  272/  468] time: 61.1795, train_loss: 0.06922150, train_accuracy: 0.9766, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  273/  468] time: 61.3977, train_loss: 0.09726828, train_accuracy: 0.9609, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  274/  468] time: 61.6191, train_loss: 0.11630234, train_accuracy: 0.9609, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  275/  468] time: 61.8448, train_loss: 0.07582889, train_accuracy: 0.9766, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  276/  468] time: 62.0668, train_loss: 0.03599039, train_accuracy: 1.0000, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  277/  468] time: 62.2896, train_loss: 0.16239099, train_accuracy: 0.9688, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  278/  468] time: 62.5128, train_loss: 0.11686817, train_accuracy: 0.9766, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  279/  468] time: 62.7344, train_loss: 0.16080754, train_accuracy: 0.9453, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  280/  468] time: 62.9581, train_loss: 0.19008411, train_accuracy: 0.9453, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  281/  468] time: 63.1825, train_loss: 0.08350077, train_accuracy: 0.9688, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  282/  468] time: 63.4034, train_loss: 0.13735735, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  283/  468] time: 63.6211, train_loss: 0.08635369, train_accuracy: 0.9766, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  284/  468] time: 63.8458, train_loss: 0.14765412, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  285/  468] time: 64.0619, train_loss: 0.04556937, train_accuracy: 0.9922, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  286/  468] time: 64.2795, train_loss: 0.12438185, train_accuracy: 0.9609, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  287/  468] time: 64.4998, train_loss: 0.22795361, train_accuracy: 0.9453, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  288/  468] time: 64.7195, train_loss: 0.06494239, train_accuracy: 0.9844, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  289/  468] time: 64.9427, train_loss: 0.08689237, train_accuracy: 0.9844, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  290/  468] time: 65.1728, train_loss: 0.10259469, train_accuracy: 0.9688, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  291/  468] time: 65.3977, train_loss: 0.10242089, train_accuracy: 0.9609, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  292/  468] time: 65.6143, train_loss: 0.08222520, train_accuracy: 0.9844, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  293/  468] time: 65.8316, train_loss: 0.03827330, train_accuracy: 0.9844, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  294/  468] time: 66.0560, train_loss: 0.17006183, train_accuracy: 0.9688, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  295/  468] time: 66.2751, train_loss: 0.04047388, train_accuracy: 0.9922, test_Accuracy: 0.9643\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 66.4923, train_loss: 0.06944358, train_accuracy: 0.9766, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  297/  468] time: 66.7184, train_loss: 0.08888323, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  298/  468] time: 66.9477, train_loss: 0.02686856, train_accuracy: 0.9922, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  299/  468] time: 67.1723, train_loss: 0.06676940, train_accuracy: 0.9688, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  300/  468] time: 67.3905, train_loss: 0.04729158, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  301/  468] time: 67.6134, train_loss: 0.12113629, train_accuracy: 0.9531, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  302/  468] time: 67.8350, train_loss: 0.10192113, train_accuracy: 0.9688, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  303/  468] time: 68.0565, train_loss: 0.08748785, train_accuracy: 0.9688, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  304/  468] time: 68.2804, train_loss: 0.12438609, train_accuracy: 0.9609, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  305/  468] time: 68.5037, train_loss: 0.11369878, train_accuracy: 0.9688, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  306/  468] time: 68.7264, train_loss: 0.08413526, train_accuracy: 0.9688, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  307/  468] time: 68.9417, train_loss: 0.10368341, train_accuracy: 0.9688, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  308/  468] time: 69.1622, train_loss: 0.20816699, train_accuracy: 0.9609, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  309/  468] time: 69.3840, train_loss: 0.05722831, train_accuracy: 0.9844, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  310/  468] time: 69.6005, train_loss: 0.11295467, train_accuracy: 0.9453, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  311/  468] time: 69.8305, train_loss: 0.08825050, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  312/  468] time: 70.0487, train_loss: 0.11602065, train_accuracy: 0.9375, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  313/  468] time: 70.2735, train_loss: 0.09447019, train_accuracy: 0.9609, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  314/  468] time: 70.5069, train_loss: 0.08242031, train_accuracy: 0.9766, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  315/  468] time: 70.7291, train_loss: 0.13633721, train_accuracy: 0.9688, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  316/  468] time: 70.9492, train_loss: 0.05442894, train_accuracy: 0.9766, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  317/  468] time: 71.1742, train_loss: 0.07123268, train_accuracy: 0.9844, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  318/  468] time: 71.3976, train_loss: 0.13967001, train_accuracy: 0.9609, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  319/  468] time: 71.6186, train_loss: 0.05222353, train_accuracy: 0.9922, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  320/  468] time: 71.8407, train_loss: 0.09359077, train_accuracy: 0.9688, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  321/  468] time: 72.0631, train_loss: 0.10172203, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  322/  468] time: 72.2854, train_loss: 0.08923075, train_accuracy: 0.9688, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  323/  468] time: 72.5155, train_loss: 0.16383581, train_accuracy: 0.9531, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  324/  468] time: 72.7399, train_loss: 0.03084872, train_accuracy: 0.9922, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  325/  468] time: 72.9618, train_loss: 0.11500735, train_accuracy: 0.9688, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  326/  468] time: 73.1819, train_loss: 0.12150934, train_accuracy: 0.9688, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  327/  468] time: 73.4028, train_loss: 0.10589678, train_accuracy: 0.9688, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  328/  468] time: 73.6242, train_loss: 0.13343823, train_accuracy: 0.9531, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  329/  468] time: 73.8481, train_loss: 0.09898008, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  330/  468] time: 74.0727, train_loss: 0.19493148, train_accuracy: 0.9531, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  331/  468] time: 74.3024, train_loss: 0.15902084, train_accuracy: 0.9375, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  332/  468] time: 74.5212, train_loss: 0.13532467, train_accuracy: 0.9375, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  333/  468] time: 74.7453, train_loss: 0.07767354, train_accuracy: 0.9688, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  334/  468] time: 74.9677, train_loss: 0.13052180, train_accuracy: 0.9688, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  335/  468] time: 75.1880, train_loss: 0.03620407, train_accuracy: 1.0000, test_Accuracy: 0.9685\n",
      "Epoch: [ 0] [  336/  468] time: 75.4244, train_loss: 0.09051219, train_accuracy: 0.9766, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  337/  468] time: 75.6478, train_loss: 0.09203047, train_accuracy: 0.9922, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  338/  468] time: 75.8647, train_loss: 0.05987356, train_accuracy: 0.9844, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  339/  468] time: 76.0898, train_loss: 0.20791925, train_accuracy: 0.9375, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  340/  468] time: 76.3193, train_loss: 0.06104165, train_accuracy: 0.9844, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  341/  468] time: 76.5483, train_loss: 0.05976996, train_accuracy: 0.9922, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  342/  468] time: 76.7754, train_loss: 0.06364177, train_accuracy: 0.9766, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  343/  468] time: 77.0076, train_loss: 0.11789159, train_accuracy: 0.9688, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  344/  468] time: 77.2324, train_loss: 0.05007035, train_accuracy: 0.9922, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  345/  468] time: 77.4666, train_loss: 0.04262977, train_accuracy: 0.9922, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  346/  468] time: 77.6920, train_loss: 0.13269757, train_accuracy: 0.9766, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  347/  468] time: 77.9158, train_loss: 0.06122722, train_accuracy: 0.9844, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  348/  468] time: 78.1403, train_loss: 0.07860287, train_accuracy: 0.9844, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  349/  468] time: 78.3606, train_loss: 0.10147815, train_accuracy: 0.9766, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  350/  468] time: 78.5820, train_loss: 0.13160375, train_accuracy: 0.9609, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  351/  468] time: 78.8023, train_loss: 0.09214284, train_accuracy: 0.9844, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  352/  468] time: 79.0266, train_loss: 0.07721861, train_accuracy: 0.9766, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  353/  468] time: 79.2466, train_loss: 0.08883487, train_accuracy: 0.9844, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  354/  468] time: 79.4679, train_loss: 0.11101037, train_accuracy: 0.9688, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  355/  468] time: 79.6906, train_loss: 0.07116283, train_accuracy: 0.9766, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  356/  468] time: 79.9092, train_loss: 0.06422583, train_accuracy: 0.9688, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  357/  468] time: 80.1305, train_loss: 0.06041586, train_accuracy: 0.9844, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  358/  468] time: 80.3528, train_loss: 0.21180823, train_accuracy: 0.9531, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  359/  468] time: 80.5744, train_loss: 0.10699323, train_accuracy: 0.9688, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  360/  468] time: 80.7986, train_loss: 0.07824484, train_accuracy: 0.9922, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  361/  468] time: 81.0200, train_loss: 0.10430089, train_accuracy: 0.9766, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  362/  468] time: 81.2505, train_loss: 0.14544863, train_accuracy: 0.9453, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  363/  468] time: 81.4800, train_loss: 0.12353817, train_accuracy: 0.9688, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  364/  468] time: 81.6990, train_loss: 0.06776362, train_accuracy: 0.9766, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  365/  468] time: 81.9251, train_loss: 0.20098504, train_accuracy: 0.9453, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  366/  468] time: 82.1490, train_loss: 0.08183049, train_accuracy: 0.9766, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  367/  468] time: 82.3756, train_loss: 0.06659944, train_accuracy: 0.9844, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  368/  468] time: 82.6048, train_loss: 0.27160242, train_accuracy: 0.9297, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  369/  468] time: 82.8341, train_loss: 0.07418948, train_accuracy: 0.9766, test_Accuracy: 0.9592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 83.0609, train_loss: 0.02590164, train_accuracy: 1.0000, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  371/  468] time: 83.2803, train_loss: 0.06930669, train_accuracy: 0.9766, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  372/  468] time: 83.4997, train_loss: 0.16387059, train_accuracy: 0.9688, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  373/  468] time: 83.7243, train_loss: 0.09610395, train_accuracy: 0.9844, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  374/  468] time: 83.9452, train_loss: 0.06315024, train_accuracy: 0.9766, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  375/  468] time: 84.1671, train_loss: 0.05466352, train_accuracy: 0.9922, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  376/  468] time: 84.3862, train_loss: 0.08280216, train_accuracy: 0.9688, test_Accuracy: 0.9659\n",
      "Epoch: [ 0] [  377/  468] time: 84.6062, train_loss: 0.15085006, train_accuracy: 0.9688, test_Accuracy: 0.9680\n",
      "Epoch: [ 0] [  378/  468] time: 84.8266, train_loss: 0.15551591, train_accuracy: 0.9609, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  379/  468] time: 85.0474, train_loss: 0.11936281, train_accuracy: 0.9688, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  380/  468] time: 85.2661, train_loss: 0.08943550, train_accuracy: 0.9688, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  381/  468] time: 85.4883, train_loss: 0.08589156, train_accuracy: 0.9688, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  382/  468] time: 85.7095, train_loss: 0.10101581, train_accuracy: 0.9766, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  383/  468] time: 85.9306, train_loss: 0.11590950, train_accuracy: 0.9688, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  384/  468] time: 86.1552, train_loss: 0.17087382, train_accuracy: 0.9453, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  385/  468] time: 86.3751, train_loss: 0.24602936, train_accuracy: 0.9375, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  386/  468] time: 86.5969, train_loss: 0.07255183, train_accuracy: 0.9922, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  387/  468] time: 86.8220, train_loss: 0.10175449, train_accuracy: 0.9766, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  388/  468] time: 87.0453, train_loss: 0.15176840, train_accuracy: 0.9531, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  389/  468] time: 87.2607, train_loss: 0.14575019, train_accuracy: 0.9766, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  390/  468] time: 87.4773, train_loss: 0.05310081, train_accuracy: 1.0000, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  391/  468] time: 87.7006, train_loss: 0.09580595, train_accuracy: 0.9688, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  392/  468] time: 87.9214, train_loss: 0.12148112, train_accuracy: 0.9531, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  393/  468] time: 88.1434, train_loss: 0.16580305, train_accuracy: 0.9453, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  394/  468] time: 88.3596, train_loss: 0.19454074, train_accuracy: 0.9375, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  395/  468] time: 88.5778, train_loss: 0.13558117, train_accuracy: 0.9609, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  396/  468] time: 88.7991, train_loss: 0.09638456, train_accuracy: 0.9531, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  397/  468] time: 89.0238, train_loss: 0.11307450, train_accuracy: 0.9766, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  398/  468] time: 89.2482, train_loss: 0.12231653, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  399/  468] time: 89.4733, train_loss: 0.08816850, train_accuracy: 0.9766, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  400/  468] time: 89.7072, train_loss: 0.13915405, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  401/  468] time: 89.9273, train_loss: 0.05707996, train_accuracy: 0.9844, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  402/  468] time: 90.1488, train_loss: 0.08032688, train_accuracy: 0.9766, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  403/  468] time: 90.3686, train_loss: 0.12671511, train_accuracy: 0.9531, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  404/  468] time: 90.5885, train_loss: 0.06478880, train_accuracy: 0.9766, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  405/  468] time: 90.8090, train_loss: 0.03266035, train_accuracy: 1.0000, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  406/  468] time: 91.0322, train_loss: 0.08129542, train_accuracy: 0.9922, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  407/  468] time: 91.2582, train_loss: 0.06894413, train_accuracy: 0.9844, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  408/  468] time: 91.4753, train_loss: 0.13175097, train_accuracy: 0.9609, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  409/  468] time: 91.7010, train_loss: 0.13322562, train_accuracy: 0.9766, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  410/  468] time: 91.9254, train_loss: 0.10461392, train_accuracy: 0.9766, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  411/  468] time: 92.1488, train_loss: 0.06242160, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  412/  468] time: 92.3725, train_loss: 0.11338450, train_accuracy: 0.9609, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  413/  468] time: 92.5908, train_loss: 0.07021929, train_accuracy: 0.9766, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  414/  468] time: 92.8097, train_loss: 0.02615711, train_accuracy: 0.9922, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  415/  468] time: 93.0310, train_loss: 0.07157885, train_accuracy: 0.9688, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  416/  468] time: 93.2518, train_loss: 0.09795089, train_accuracy: 0.9688, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  417/  468] time: 93.4747, train_loss: 0.05448652, train_accuracy: 0.9922, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  418/  468] time: 93.6929, train_loss: 0.14940572, train_accuracy: 0.9609, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  419/  468] time: 93.9131, train_loss: 0.08773511, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  420/  468] time: 94.1323, train_loss: 0.12641677, train_accuracy: 0.9609, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  421/  468] time: 94.3528, train_loss: 0.09091647, train_accuracy: 0.9766, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  422/  468] time: 94.5699, train_loss: 0.07491738, train_accuracy: 0.9766, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  423/  468] time: 94.7877, train_loss: 0.05580225, train_accuracy: 0.9922, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  424/  468] time: 95.0067, train_loss: 0.09975530, train_accuracy: 0.9688, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  425/  468] time: 95.2342, train_loss: 0.10752380, train_accuracy: 0.9609, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  426/  468] time: 95.4527, train_loss: 0.07769112, train_accuracy: 0.9688, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  427/  468] time: 95.6725, train_loss: 0.10640617, train_accuracy: 0.9844, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  428/  468] time: 95.8904, train_loss: 0.08986091, train_accuracy: 0.9766, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  429/  468] time: 96.1134, train_loss: 0.05996444, train_accuracy: 0.9922, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  430/  468] time: 96.3346, train_loss: 0.13503578, train_accuracy: 0.9531, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  431/  468] time: 96.5575, train_loss: 0.09268419, train_accuracy: 0.9766, test_Accuracy: 0.9671\n",
      "Epoch: [ 0] [  432/  468] time: 96.7841, train_loss: 0.18160275, train_accuracy: 0.9609, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  433/  468] time: 97.0090, train_loss: 0.10024484, train_accuracy: 0.9453, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  434/  468] time: 97.2346, train_loss: 0.12074058, train_accuracy: 0.9688, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  435/  468] time: 97.4559, train_loss: 0.09740203, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  436/  468] time: 97.6740, train_loss: 0.04638202, train_accuracy: 0.9844, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  437/  468] time: 97.8929, train_loss: 0.03607454, train_accuracy: 0.9922, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  438/  468] time: 98.1121, train_loss: 0.11826365, train_accuracy: 0.9688, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  439/  468] time: 98.3304, train_loss: 0.08088779, train_accuracy: 0.9766, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  440/  468] time: 98.5513, train_loss: 0.05849773, train_accuracy: 0.9922, test_Accuracy: 0.9669\n",
      "Epoch: [ 0] [  441/  468] time: 98.7723, train_loss: 0.07616133, train_accuracy: 0.9844, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  442/  468] time: 98.9886, train_loss: 0.08917651, train_accuracy: 0.9766, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  443/  468] time: 99.2108, train_loss: 0.07266602, train_accuracy: 0.9844, test_Accuracy: 0.9666\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 99.4326, train_loss: 0.20337307, train_accuracy: 0.9375, test_Accuracy: 0.9675\n",
      "Epoch: [ 0] [  445/  468] time: 99.6537, train_loss: 0.06113275, train_accuracy: 0.9922, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  446/  468] time: 99.8762, train_loss: 0.04254772, train_accuracy: 1.0000, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  447/  468] time: 100.0923, train_loss: 0.09401506, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  448/  468] time: 100.3119, train_loss: 0.06421863, train_accuracy: 0.9844, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  449/  468] time: 100.5365, train_loss: 0.07544518, train_accuracy: 0.9766, test_Accuracy: 0.9659\n",
      "Epoch: [ 0] [  450/  468] time: 100.7660, train_loss: 0.10679516, train_accuracy: 0.9688, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  451/  468] time: 100.9965, train_loss: 0.09860048, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  452/  468] time: 101.2231, train_loss: 0.02897664, train_accuracy: 0.9922, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  453/  468] time: 101.4421, train_loss: 0.07724638, train_accuracy: 0.9766, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  454/  468] time: 101.6663, train_loss: 0.12798882, train_accuracy: 0.9609, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  455/  468] time: 101.8966, train_loss: 0.04202047, train_accuracy: 0.9922, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  456/  468] time: 102.1158, train_loss: 0.08251347, train_accuracy: 0.9844, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  457/  468] time: 102.3283, train_loss: 0.11222225, train_accuracy: 0.9609, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  458/  468] time: 102.5566, train_loss: 0.04215899, train_accuracy: 1.0000, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  459/  468] time: 102.7831, train_loss: 0.10023360, train_accuracy: 0.9766, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  460/  468] time: 103.0044, train_loss: 0.11636223, train_accuracy: 0.9609, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  461/  468] time: 103.2255, train_loss: 0.03003471, train_accuracy: 0.9922, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  462/  468] time: 103.4526, train_loss: 0.04125806, train_accuracy: 0.9922, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  463/  468] time: 103.6750, train_loss: 0.11145420, train_accuracy: 0.9688, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  464/  468] time: 103.8912, train_loss: 0.09315600, train_accuracy: 0.9688, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  465/  468] time: 104.1096, train_loss: 0.13561453, train_accuracy: 0.9688, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  466/  468] time: 104.3242, train_loss: 0.11915449, train_accuracy: 0.9609, test_Accuracy: 0.9695\n",
      "Epoch: [ 0] [  467/  468] time: 104.5450, train_loss: 0.17887419, train_accuracy: 0.9609, test_Accuracy: 0.9688\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.contrib.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)\n",
    "    global_step = tf.train.create_global_step()\n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)\n",
    "        start_iteration = checkpoint_counter - start_epoch * training_iterations\n",
    "        counter = checkpoint_counter\n",
    "        global_step.assign(checkpoint_counter)\n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx in range(start_iteration, training_iterations):\n",
    "                train_input, train_label = train_iterator.get_next()\n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables), global_step=global_step)\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                test_input, test_label = test_iterator.get_next()\n",
    "                test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.contrib.summary.scalar(name='train_loss', tensor=train_loss)\n",
    "                tf.contrib.summary.scalar(name='train_accuracy', tensor=train_accuracy)\n",
    "                tf.contrib.summary.scalar(name='test_accuracy', tensor=test_accuracy)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    test_input, test_label = test_iterator.get_next()\n",
    "    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 96.88 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}