lab-10-1-2-mnist_nn_relu.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Eager mode on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.RandomNormal()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(2):\n",
    "            self.model.add(dense(256, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.RandomNormal()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(2) :\n",
    "        model.add(dense(256, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size).\\\n",
    "    repeat()\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x)).\\\n",
    "    repeat()\n",
    "\n",
    "train_iterator = train_dataset.make_one_shot_iterator()\n",
    "test_iterator = test_dataset.make_one_shot_iterator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_relu'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.3115, train_loss: 2.14351797, train_accuracy: 0.4922, test_Accuracy: 0.2823\n",
      "Epoch: [ 0] [    1/  468] time: 0.4366, train_loss: 2.10850906, train_accuracy: 0.4922, test_Accuracy: 0.4468\n",
      "Epoch: [ 0] [    2/  468] time: 0.5389, train_loss: 2.01465869, train_accuracy: 0.6250, test_Accuracy: 0.5340\n",
      "Epoch: [ 0] [    3/  468] time: 0.6307, train_loss: 2.00693560, train_accuracy: 0.5469, test_Accuracy: 0.5937\n",
      "Epoch: [ 0] [    4/  468] time: 0.7357, train_loss: 1.88391960, train_accuracy: 0.6172, test_Accuracy: 0.6357\n",
      "Epoch: [ 0] [    5/  468] time: 0.8431, train_loss: 1.78965557, train_accuracy: 0.6406, test_Accuracy: 0.6713\n",
      "Epoch: [ 0] [    6/  468] time: 0.9512, train_loss: 1.70328987, train_accuracy: 0.6953, test_Accuracy: 0.6985\n",
      "Epoch: [ 0] [    7/  468] time: 1.0521, train_loss: 1.52768254, train_accuracy: 0.6875, test_Accuracy: 0.7223\n",
      "Epoch: [ 0] [    8/  468] time: 1.1572, train_loss: 1.51776791, train_accuracy: 0.6797, test_Accuracy: 0.7400\n",
      "Epoch: [ 0] [    9/  468] time: 1.2505, train_loss: 1.30525339, train_accuracy: 0.8203, test_Accuracy: 0.7461\n",
      "Epoch: [ 0] [   10/  468] time: 1.3459, train_loss: 1.26801586, train_accuracy: 0.7500, test_Accuracy: 0.7447\n",
      "Epoch: [ 0] [   11/  468] time: 1.4534, train_loss: 1.20718193, train_accuracy: 0.7031, test_Accuracy: 0.7414\n",
      "Epoch: [ 0] [   12/  468] time: 1.5417, train_loss: 1.05175114, train_accuracy: 0.7969, test_Accuracy: 0.7510\n",
      "Epoch: [ 0] [   13/  468] time: 1.6518, train_loss: 0.94791597, train_accuracy: 0.7812, test_Accuracy: 0.7652\n",
      "Epoch: [ 0] [   14/  468] time: 1.7373, train_loss: 0.89156866, train_accuracy: 0.8281, test_Accuracy: 0.7846\n",
      "Epoch: [ 0] [   15/  468] time: 1.8490, train_loss: 0.88374597, train_accuracy: 0.7734, test_Accuracy: 0.7948\n",
      "Epoch: [ 0] [   16/  468] time: 1.9549, train_loss: 0.80194950, train_accuracy: 0.7891, test_Accuracy: 0.7924\n",
      "Epoch: [ 0] [   17/  468] time: 2.0528, train_loss: 0.61217487, train_accuracy: 0.8438, test_Accuracy: 0.7856\n",
      "Epoch: [ 0] [   18/  468] time: 2.1587, train_loss: 0.67157137, train_accuracy: 0.8438, test_Accuracy: 0.8030\n",
      "Epoch: [ 0] [   19/  468] time: 2.2614, train_loss: 0.54838407, train_accuracy: 0.8672, test_Accuracy: 0.8115\n",
      "Epoch: [ 0] [   20/  468] time: 2.3635, train_loss: 0.62177503, train_accuracy: 0.8125, test_Accuracy: 0.8231\n",
      "Epoch: [ 0] [   21/  468] time: 2.4627, train_loss: 0.70921552, train_accuracy: 0.7656, test_Accuracy: 0.8309\n",
      "Epoch: [ 0] [   22/  468] time: 2.5666, train_loss: 0.60276926, train_accuracy: 0.8047, test_Accuracy: 0.8418\n",
      "Epoch: [ 0] [   23/  468] time: 2.6692, train_loss: 0.64741707, train_accuracy: 0.8203, test_Accuracy: 0.8477\n",
      "Epoch: [ 0] [   24/  468] time: 2.7704, train_loss: 0.56265235, train_accuracy: 0.8203, test_Accuracy: 0.8428\n",
      "Epoch: [ 0] [   25/  468] time: 2.8685, train_loss: 0.54725909, train_accuracy: 0.8125, test_Accuracy: 0.8396\n",
      "Epoch: [ 0] [   26/  468] time: 2.9619, train_loss: 0.52226597, train_accuracy: 0.8438, test_Accuracy: 0.8463\n",
      "Epoch: [ 0] [   27/  468] time: 3.0520, train_loss: 0.45085788, train_accuracy: 0.8672, test_Accuracy: 0.8526\n",
      "Epoch: [ 0] [   28/  468] time: 3.1523, train_loss: 0.48955667, train_accuracy: 0.8281, test_Accuracy: 0.8520\n",
      "Epoch: [ 0] [   29/  468] time: 3.2391, train_loss: 0.44008225, train_accuracy: 0.8828, test_Accuracy: 0.8472\n",
      "Epoch: [ 0] [   30/  468] time: 3.3409, train_loss: 0.49190298, train_accuracy: 0.8438, test_Accuracy: 0.8425\n",
      "Epoch: [ 0] [   31/  468] time: 3.4478, train_loss: 0.41276670, train_accuracy: 0.8438, test_Accuracy: 0.8470\n",
      "Epoch: [ 0] [   32/  468] time: 3.5471, train_loss: 0.29355663, train_accuracy: 0.9062, test_Accuracy: 0.8602\n",
      "Epoch: [ 0] [   33/  468] time: 3.6391, train_loss: 0.54430795, train_accuracy: 0.8438, test_Accuracy: 0.8718\n",
      "Epoch: [ 0] [   34/  468] time: 3.7294, train_loss: 0.48485044, train_accuracy: 0.8594, test_Accuracy: 0.8759\n",
      "Epoch: [ 0] [   35/  468] time: 3.8163, train_loss: 0.42300725, train_accuracy: 0.8594, test_Accuracy: 0.8796\n",
      "Epoch: [ 0] [   36/  468] time: 3.9037, train_loss: 0.65334672, train_accuracy: 0.8516, test_Accuracy: 0.8798\n",
      "Epoch: [ 0] [   37/  468] time: 3.9904, train_loss: 0.37491623, train_accuracy: 0.8984, test_Accuracy: 0.8818\n",
      "Epoch: [ 0] [   38/  468] time: 4.0774, train_loss: 0.40804064, train_accuracy: 0.8750, test_Accuracy: 0.8742\n",
      "Epoch: [ 0] [   39/  468] time: 4.1642, train_loss: 0.35693562, train_accuracy: 0.8828, test_Accuracy: 0.8672\n",
      "Epoch: [ 0] [   40/  468] time: 4.2650, train_loss: 0.38754064, train_accuracy: 0.8438, test_Accuracy: 0.8610\n",
      "Epoch: [ 0] [   41/  468] time: 4.3638, train_loss: 0.48385713, train_accuracy: 0.8594, test_Accuracy: 0.8659\n",
      "Epoch: [ 0] [   42/  468] time: 4.4606, train_loss: 0.52156305, train_accuracy: 0.8359, test_Accuracy: 0.8804\n",
      "Epoch: [ 0] [   43/  468] time: 4.5562, train_loss: 0.29416278, train_accuracy: 0.9453, test_Accuracy: 0.8884\n",
      "Epoch: [ 0] [   44/  468] time: 4.6576, train_loss: 0.38783616, train_accuracy: 0.9141, test_Accuracy: 0.8856\n",
      "Epoch: [ 0] [   45/  468] time: 4.7531, train_loss: 0.29984835, train_accuracy: 0.9219, test_Accuracy: 0.8730\n",
      "Epoch: [ 0] [   46/  468] time: 4.8430, train_loss: 0.43648627, train_accuracy: 0.8281, test_Accuracy: 0.8704\n",
      "Epoch: [ 0] [   47/  468] time: 4.9308, train_loss: 0.25685996, train_accuracy: 0.9219, test_Accuracy: 0.8701\n",
      "Epoch: [ 0] [   48/  468] time: 5.0196, train_loss: 0.43898284, train_accuracy: 0.8516, test_Accuracy: 0.8760\n",
      "Epoch: [ 0] [   49/  468] time: 5.1223, train_loss: 0.64768296, train_accuracy: 0.8672, test_Accuracy: 0.8859\n",
      "Epoch: [ 0] [   50/  468] time: 5.2135, train_loss: 0.30812627, train_accuracy: 0.8984, test_Accuracy: 0.8872\n",
      "Epoch: [ 0] [   51/  468] time: 5.3052, train_loss: 0.40071401, train_accuracy: 0.8594, test_Accuracy: 0.8891\n",
      "Epoch: [ 0] [   52/  468] time: 5.3963, train_loss: 0.38357860, train_accuracy: 0.8906, test_Accuracy: 0.8895\n",
      "Epoch: [ 0] [   53/  468] time: 5.4897, train_loss: 0.42938173, train_accuracy: 0.8906, test_Accuracy: 0.8843\n",
      "Epoch: [ 0] [   54/  468] time: 5.5781, train_loss: 0.41269195, train_accuracy: 0.8672, test_Accuracy: 0.8812\n",
      "Epoch: [ 0] [   55/  468] time: 5.6718, train_loss: 0.42416659, train_accuracy: 0.8672, test_Accuracy: 0.8834\n",
      "Epoch: [ 0] [   56/  468] time: 5.7594, train_loss: 0.32260150, train_accuracy: 0.8906, test_Accuracy: 0.8864\n",
      "Epoch: [ 0] [   57/  468] time: 5.8463, train_loss: 0.35592049, train_accuracy: 0.9062, test_Accuracy: 0.8909\n",
      "Epoch: [ 0] [   58/  468] time: 5.9353, train_loss: 0.27471745, train_accuracy: 0.9141, test_Accuracy: 0.8900\n",
      "Epoch: [ 0] [   59/  468] time: 6.0225, train_loss: 0.23434584, train_accuracy: 0.9297, test_Accuracy: 0.8869\n",
      "Epoch: [ 0] [   60/  468] time: 6.1172, train_loss: 0.39671481, train_accuracy: 0.8828, test_Accuracy: 0.8918\n",
      "Epoch: [ 0] [   61/  468] time: 6.2081, train_loss: 0.30821875, train_accuracy: 0.8984, test_Accuracy: 0.8956\n",
      "Epoch: [ 0] [   62/  468] time: 6.2958, train_loss: 0.32624808, train_accuracy: 0.8984, test_Accuracy: 0.8893\n",
      "Epoch: [ 0] [   63/  468] time: 6.3838, train_loss: 0.32803768, train_accuracy: 0.8984, test_Accuracy: 0.8857\n",
      "Epoch: [ 0] [   64/  468] time: 6.4758, train_loss: 0.62339813, train_accuracy: 0.7969, test_Accuracy: 0.8894\n",
      "Epoch: [ 0] [   65/  468] time: 6.5734, train_loss: 0.40417135, train_accuracy: 0.8594, test_Accuracy: 0.8951\n",
      "Epoch: [ 0] [   66/  468] time: 6.6676, train_loss: 0.45245939, train_accuracy: 0.8359, test_Accuracy: 0.8995\n",
      "Epoch: [ 0] [   67/  468] time: 6.7604, train_loss: 0.31140167, train_accuracy: 0.9062, test_Accuracy: 0.9029\n",
      "Epoch: [ 0] [   68/  468] time: 6.8488, train_loss: 0.27538085, train_accuracy: 0.9219, test_Accuracy: 0.9020\n",
      "Epoch: [ 0] [   69/  468] time: 6.9382, train_loss: 0.35155973, train_accuracy: 0.9219, test_Accuracy: 0.8990\n",
      "Epoch: [ 0] [   70/  468] time: 7.0344, train_loss: 0.30051386, train_accuracy: 0.9219, test_Accuracy: 0.8981\n",
      "Epoch: [ 0] [   71/  468] time: 7.1291, train_loss: 0.22398081, train_accuracy: 0.9375, test_Accuracy: 0.8947\n",
      "Epoch: [ 0] [   72/  468] time: 7.2238, train_loss: 0.31053323, train_accuracy: 0.9297, test_Accuracy: 0.8952\n",
      "Epoch: [ 0] [   73/  468] time: 7.3152, train_loss: 0.31106350, train_accuracy: 0.9219, test_Accuracy: 0.8972\n",
      "Epoch: [ 0] [   74/  468] time: 7.4059, train_loss: 0.28837320, train_accuracy: 0.9062, test_Accuracy: 0.9002\n",
      "Epoch: [ 0] [   75/  468] time: 7.4910, train_loss: 0.23635522, train_accuracy: 0.9453, test_Accuracy: 0.9057\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   76/  468] time: 7.5749, train_loss: 0.25014365, train_accuracy: 0.9141, test_Accuracy: 0.9081\n",
      "Epoch: [ 0] [   77/  468] time: 7.6634, train_loss: 0.29067466, train_accuracy: 0.8984, test_Accuracy: 0.9093\n",
      "Epoch: [ 0] [   78/  468] time: 7.7502, train_loss: 0.40782946, train_accuracy: 0.8828, test_Accuracy: 0.9081\n",
      "Epoch: [ 0] [   79/  468] time: 7.8430, train_loss: 0.29894257, train_accuracy: 0.8750, test_Accuracy: 0.9082\n",
      "Epoch: [ 0] [   80/  468] time: 7.9286, train_loss: 0.29954958, train_accuracy: 0.9141, test_Accuracy: 0.9065\n",
      "Epoch: [ 0] [   81/  468] time: 8.0173, train_loss: 0.48582801, train_accuracy: 0.8281, test_Accuracy: 0.9082\n",
      "Epoch: [ 0] [   82/  468] time: 8.1165, train_loss: 0.28034693, train_accuracy: 0.9219, test_Accuracy: 0.9093\n",
      "Epoch: [ 0] [   83/  468] time: 8.2144, train_loss: 0.39410660, train_accuracy: 0.8828, test_Accuracy: 0.9134\n",
      "Epoch: [ 0] [   84/  468] time: 8.3135, train_loss: 0.19375709, train_accuracy: 0.9375, test_Accuracy: 0.9151\n",
      "Epoch: [ 0] [   85/  468] time: 8.4193, train_loss: 0.33734703, train_accuracy: 0.9141, test_Accuracy: 0.9140\n",
      "Epoch: [ 0] [   86/  468] time: 8.5128, train_loss: 0.38094884, train_accuracy: 0.8906, test_Accuracy: 0.9112\n",
      "Epoch: [ 0] [   87/  468] time: 8.6108, train_loss: 0.24214005, train_accuracy: 0.9141, test_Accuracy: 0.9097\n",
      "Epoch: [ 0] [   88/  468] time: 8.6999, train_loss: 0.31520736, train_accuracy: 0.9062, test_Accuracy: 0.9105\n",
      "Epoch: [ 0] [   89/  468] time: 8.7993, train_loss: 0.29912055, train_accuracy: 0.9062, test_Accuracy: 0.9101\n",
      "Epoch: [ 0] [   90/  468] time: 8.8887, train_loss: 0.34666103, train_accuracy: 0.8828, test_Accuracy: 0.9089\n",
      "Epoch: [ 0] [   91/  468] time: 8.9915, train_loss: 0.27899745, train_accuracy: 0.9062, test_Accuracy: 0.9098\n",
      "Epoch: [ 0] [   92/  468] time: 9.0941, train_loss: 0.29691660, train_accuracy: 0.9297, test_Accuracy: 0.9137\n",
      "Epoch: [ 0] [   93/  468] time: 9.1929, train_loss: 0.30360764, train_accuracy: 0.9141, test_Accuracy: 0.9173\n",
      "Epoch: [ 0] [   94/  468] time: 9.2851, train_loss: 0.28626585, train_accuracy: 0.9141, test_Accuracy: 0.9193\n",
      "Epoch: [ 0] [   95/  468] time: 9.3814, train_loss: 0.28891778, train_accuracy: 0.8906, test_Accuracy: 0.9194\n",
      "Epoch: [ 0] [   96/  468] time: 9.4791, train_loss: 0.17140040, train_accuracy: 0.9453, test_Accuracy: 0.9182\n",
      "Epoch: [ 0] [   97/  468] time: 9.5686, train_loss: 0.18048185, train_accuracy: 0.9375, test_Accuracy: 0.9147\n",
      "Epoch: [ 0] [   98/  468] time: 9.6635, train_loss: 0.30283302, train_accuracy: 0.9062, test_Accuracy: 0.9096\n",
      "Epoch: [ 0] [   99/  468] time: 9.7523, train_loss: 0.32144594, train_accuracy: 0.8828, test_Accuracy: 0.9098\n",
      "Epoch: [ 0] [  100/  468] time: 9.8407, train_loss: 0.34320685, train_accuracy: 0.8828, test_Accuracy: 0.9116\n",
      "Epoch: [ 0] [  101/  468] time: 9.9340, train_loss: 0.26874208, train_accuracy: 0.9297, test_Accuracy: 0.9156\n",
      "Epoch: [ 0] [  102/  468] time: 10.0282, train_loss: 0.33579716, train_accuracy: 0.8984, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [  103/  468] time: 10.1237, train_loss: 0.32672656, train_accuracy: 0.8828, test_Accuracy: 0.9220\n",
      "Epoch: [ 0] [  104/  468] time: 10.2177, train_loss: 0.22283921, train_accuracy: 0.9531, test_Accuracy: 0.9225\n",
      "Epoch: [ 0] [  105/  468] time: 10.3139, train_loss: 0.19848382, train_accuracy: 0.9453, test_Accuracy: 0.9233\n",
      "Epoch: [ 0] [  106/  468] time: 10.4073, train_loss: 0.31137434, train_accuracy: 0.9141, test_Accuracy: 0.9209\n",
      "Epoch: [ 0] [  107/  468] time: 10.5010, train_loss: 0.20895717, train_accuracy: 0.9219, test_Accuracy: 0.9196\n",
      "Epoch: [ 0] [  108/  468] time: 10.5924, train_loss: 0.31701007, train_accuracy: 0.9219, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [  109/  468] time: 10.6827, train_loss: 0.28934950, train_accuracy: 0.9375, test_Accuracy: 0.9231\n",
      "Epoch: [ 0] [  110/  468] time: 10.7771, train_loss: 0.20560099, train_accuracy: 0.9297, test_Accuracy: 0.9208\n",
      "Epoch: [ 0] [  111/  468] time: 10.8682, train_loss: 0.32409489, train_accuracy: 0.9141, test_Accuracy: 0.9205\n",
      "Epoch: [ 0] [  112/  468] time: 10.9634, train_loss: 0.35497305, train_accuracy: 0.9219, test_Accuracy: 0.9188\n",
      "Epoch: [ 0] [  113/  468] time: 11.0586, train_loss: 0.19972242, train_accuracy: 0.9453, test_Accuracy: 0.9142\n",
      "Epoch: [ 0] [  114/  468] time: 11.1558, train_loss: 0.29834574, train_accuracy: 0.8828, test_Accuracy: 0.9124\n",
      "Epoch: [ 0] [  115/  468] time: 11.2475, train_loss: 0.31999692, train_accuracy: 0.8906, test_Accuracy: 0.9133\n",
      "Epoch: [ 0] [  116/  468] time: 11.3435, train_loss: 0.34748182, train_accuracy: 0.8984, test_Accuracy: 0.9163\n",
      "Epoch: [ 0] [  117/  468] time: 11.4371, train_loss: 0.20719773, train_accuracy: 0.9609, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [  118/  468] time: 11.5313, train_loss: 0.32928419, train_accuracy: 0.9062, test_Accuracy: 0.9212\n",
      "Epoch: [ 0] [  119/  468] time: 11.6248, train_loss: 0.22185788, train_accuracy: 0.9297, test_Accuracy: 0.9230\n",
      "Epoch: [ 0] [  120/  468] time: 11.7178, train_loss: 0.26977032, train_accuracy: 0.9062, test_Accuracy: 0.9244\n",
      "Epoch: [ 0] [  121/  468] time: 11.8104, train_loss: 0.33186316, train_accuracy: 0.9375, test_Accuracy: 0.9227\n",
      "Epoch: [ 0] [  122/  468] time: 11.9062, train_loss: 0.29057434, train_accuracy: 0.9062, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [  123/  468] time: 11.9988, train_loss: 0.27394873, train_accuracy: 0.9297, test_Accuracy: 0.9188\n",
      "Epoch: [ 0] [  124/  468] time: 12.0882, train_loss: 0.32516417, train_accuracy: 0.8984, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  125/  468] time: 12.1821, train_loss: 0.29021800, train_accuracy: 0.9219, test_Accuracy: 0.9244\n",
      "Epoch: [ 0] [  126/  468] time: 12.2797, train_loss: 0.13805328, train_accuracy: 0.9688, test_Accuracy: 0.9280\n",
      "Epoch: [ 0] [  127/  468] time: 12.3753, train_loss: 0.20344947, train_accuracy: 0.9297, test_Accuracy: 0.9304\n",
      "Epoch: [ 0] [  128/  468] time: 12.4685, train_loss: 0.24062711, train_accuracy: 0.9297, test_Accuracy: 0.9299\n",
      "Epoch: [ 0] [  129/  468] time: 12.5572, train_loss: 0.19126292, train_accuracy: 0.9609, test_Accuracy: 0.9264\n",
      "Epoch: [ 0] [  130/  468] time: 12.6529, train_loss: 0.17487241, train_accuracy: 0.9375, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  131/  468] time: 12.7456, train_loss: 0.31802899, train_accuracy: 0.9062, test_Accuracy: 0.9223\n",
      "Epoch: [ 0] [  132/  468] time: 12.8387, train_loss: 0.31214172, train_accuracy: 0.9219, test_Accuracy: 0.9225\n",
      "Epoch: [ 0] [  133/  468] time: 12.9338, train_loss: 0.30553478, train_accuracy: 0.8906, test_Accuracy: 0.9251\n",
      "Epoch: [ 0] [  134/  468] time: 13.0244, train_loss: 0.19862334, train_accuracy: 0.9375, test_Accuracy: 0.9259\n",
      "Epoch: [ 0] [  135/  468] time: 13.1135, train_loss: 0.31315443, train_accuracy: 0.9062, test_Accuracy: 0.9221\n",
      "Epoch: [ 0] [  136/  468] time: 13.2038, train_loss: 0.21155649, train_accuracy: 0.8984, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [  137/  468] time: 13.2945, train_loss: 0.24998631, train_accuracy: 0.9375, test_Accuracy: 0.9213\n",
      "Epoch: [ 0] [  138/  468] time: 13.3827, train_loss: 0.31040365, train_accuracy: 0.9219, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  139/  468] time: 13.4727, train_loss: 0.25496653, train_accuracy: 0.9219, test_Accuracy: 0.9238\n",
      "Epoch: [ 0] [  140/  468] time: 13.5583, train_loss: 0.32866406, train_accuracy: 0.8984, test_Accuracy: 0.9223\n",
      "Epoch: [ 0] [  141/  468] time: 13.6556, train_loss: 0.27787554, train_accuracy: 0.9219, test_Accuracy: 0.9224\n",
      "Epoch: [ 0] [  142/  468] time: 13.7498, train_loss: 0.20905341, train_accuracy: 0.9375, test_Accuracy: 0.9259\n",
      "Epoch: [ 0] [  143/  468] time: 13.8387, train_loss: 0.20167378, train_accuracy: 0.9609, test_Accuracy: 0.9278\n",
      "Epoch: [ 0] [  144/  468] time: 13.9365, train_loss: 0.40608838, train_accuracy: 0.8906, test_Accuracy: 0.9298\n",
      "Epoch: [ 0] [  145/  468] time: 14.0218, train_loss: 0.19985238, train_accuracy: 0.9453, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [  146/  468] time: 14.1117, train_loss: 0.19789751, train_accuracy: 0.9375, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [  147/  468] time: 14.2080, train_loss: 0.18663096, train_accuracy: 0.9297, test_Accuracy: 0.9279\n",
      "Epoch: [ 0] [  148/  468] time: 14.2992, train_loss: 0.22030172, train_accuracy: 0.9062, test_Accuracy: 0.9278\n",
      "Epoch: [ 0] [  149/  468] time: 14.3958, train_loss: 0.18182477, train_accuracy: 0.9297, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  150/  468] time: 14.4906, train_loss: 0.27799180, train_accuracy: 0.9375, test_Accuracy: 0.9232\n",
      "Epoch: [ 0] [  151/  468] time: 14.5816, train_loss: 0.15075676, train_accuracy: 0.9453, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  152/  468] time: 14.6701, train_loss: 0.27510926, train_accuracy: 0.9375, test_Accuracy: 0.9262\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  153/  468] time: 14.7648, train_loss: 0.22450532, train_accuracy: 0.9219, test_Accuracy: 0.9282\n",
      "Epoch: [ 0] [  154/  468] time: 14.8588, train_loss: 0.21915072, train_accuracy: 0.9375, test_Accuracy: 0.9320\n",
      "Epoch: [ 0] [  155/  468] time: 14.9522, train_loss: 0.18834199, train_accuracy: 0.9375, test_Accuracy: 0.9321\n",
      "Epoch: [ 0] [  156/  468] time: 15.0483, train_loss: 0.18547265, train_accuracy: 0.9453, test_Accuracy: 0.9305\n",
      "Epoch: [ 0] [  157/  468] time: 15.1411, train_loss: 0.28770554, train_accuracy: 0.9375, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [  158/  468] time: 15.2347, train_loss: 0.29708147, train_accuracy: 0.9297, test_Accuracy: 0.9297\n",
      "Epoch: [ 0] [  159/  468] time: 15.3289, train_loss: 0.21299247, train_accuracy: 0.9531, test_Accuracy: 0.9288\n",
      "Epoch: [ 0] [  160/  468] time: 15.4235, train_loss: 0.15039845, train_accuracy: 0.9688, test_Accuracy: 0.9295\n",
      "Epoch: [ 0] [  161/  468] time: 15.5116, train_loss: 0.16528589, train_accuracy: 0.9453, test_Accuracy: 0.9300\n",
      "Epoch: [ 0] [  162/  468] time: 15.6003, train_loss: 0.31142923, train_accuracy: 0.9375, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [  163/  468] time: 15.6914, train_loss: 0.22813430, train_accuracy: 0.9453, test_Accuracy: 0.9315\n",
      "Epoch: [ 0] [  164/  468] time: 15.7852, train_loss: 0.19194105, train_accuracy: 0.9531, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [  165/  468] time: 15.8788, train_loss: 0.31482512, train_accuracy: 0.9062, test_Accuracy: 0.9331\n",
      "Epoch: [ 0] [  166/  468] time: 15.9729, train_loss: 0.25547159, train_accuracy: 0.9297, test_Accuracy: 0.9338\n",
      "Epoch: [ 0] [  167/  468] time: 16.0641, train_loss: 0.22950724, train_accuracy: 0.9297, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  168/  468] time: 16.1588, train_loss: 0.28938416, train_accuracy: 0.9062, test_Accuracy: 0.9327\n",
      "Epoch: [ 0] [  169/  468] time: 16.2562, train_loss: 0.16666691, train_accuracy: 0.9375, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [  170/  468] time: 16.3488, train_loss: 0.19896898, train_accuracy: 0.9531, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  171/  468] time: 16.4413, train_loss: 0.36126739, train_accuracy: 0.9219, test_Accuracy: 0.9351\n",
      "Epoch: [ 0] [  172/  468] time: 16.5296, train_loss: 0.23079449, train_accuracy: 0.9531, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [  173/  468] time: 16.6225, train_loss: 0.19739439, train_accuracy: 0.9531, test_Accuracy: 0.9363\n",
      "Epoch: [ 0] [  174/  468] time: 16.7163, train_loss: 0.12105645, train_accuracy: 0.9688, test_Accuracy: 0.9360\n",
      "Epoch: [ 0] [  175/  468] time: 16.8066, train_loss: 0.17808026, train_accuracy: 0.9297, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [  176/  468] time: 16.9008, train_loss: 0.19897430, train_accuracy: 0.9453, test_Accuracy: 0.9322\n",
      "Epoch: [ 0] [  177/  468] time: 16.9933, train_loss: 0.16771479, train_accuracy: 0.9375, test_Accuracy: 0.9296\n",
      "Epoch: [ 0] [  178/  468] time: 17.0856, train_loss: 0.18626100, train_accuracy: 0.9219, test_Accuracy: 0.9267\n",
      "Epoch: [ 0] [  179/  468] time: 17.1815, train_loss: 0.27088019, train_accuracy: 0.9062, test_Accuracy: 0.9272\n",
      "Epoch: [ 0] [  180/  468] time: 17.2749, train_loss: 0.29559952, train_accuracy: 0.9141, test_Accuracy: 0.9309\n",
      "Epoch: [ 0] [  181/  468] time: 17.3683, train_loss: 0.30717582, train_accuracy: 0.8984, test_Accuracy: 0.9348\n",
      "Epoch: [ 0] [  182/  468] time: 17.4609, train_loss: 0.09658732, train_accuracy: 0.9844, test_Accuracy: 0.9344\n",
      "Epoch: [ 0] [  183/  468] time: 17.5583, train_loss: 0.23194218, train_accuracy: 0.9062, test_Accuracy: 0.9321\n",
      "Epoch: [ 0] [  184/  468] time: 17.6465, train_loss: 0.23788801, train_accuracy: 0.9375, test_Accuracy: 0.9283\n",
      "Epoch: [ 0] [  185/  468] time: 17.7405, train_loss: 0.35483155, train_accuracy: 0.9141, test_Accuracy: 0.9265\n",
      "Epoch: [ 0] [  186/  468] time: 17.8276, train_loss: 0.16385269, train_accuracy: 0.9531, test_Accuracy: 0.9298\n",
      "Epoch: [ 0] [  187/  468] time: 17.9168, train_loss: 0.17862706, train_accuracy: 0.9453, test_Accuracy: 0.9313\n",
      "Epoch: [ 0] [  188/  468] time: 18.0085, train_loss: 0.26114824, train_accuracy: 0.9297, test_Accuracy: 0.9339\n",
      "Epoch: [ 0] [  189/  468] time: 18.1031, train_loss: 0.31486869, train_accuracy: 0.9531, test_Accuracy: 0.9363\n",
      "Epoch: [ 0] [  190/  468] time: 18.1960, train_loss: 0.23443881, train_accuracy: 0.8984, test_Accuracy: 0.9392\n",
      "Epoch: [ 0] [  191/  468] time: 18.2848, train_loss: 0.37614799, train_accuracy: 0.9062, test_Accuracy: 0.9385\n",
      "Epoch: [ 0] [  192/  468] time: 18.3778, train_loss: 0.19408935, train_accuracy: 0.9531, test_Accuracy: 0.9387\n",
      "Epoch: [ 0] [  193/  468] time: 18.4708, train_loss: 0.18849581, train_accuracy: 0.9297, test_Accuracy: 0.9392\n",
      "Epoch: [ 0] [  194/  468] time: 18.5611, train_loss: 0.15006760, train_accuracy: 0.9531, test_Accuracy: 0.9391\n",
      "Epoch: [ 0] [  195/  468] time: 18.6538, train_loss: 0.14547570, train_accuracy: 0.9453, test_Accuracy: 0.9399\n",
      "Epoch: [ 0] [  196/  468] time: 18.7448, train_loss: 0.17078212, train_accuracy: 0.9609, test_Accuracy: 0.9409\n",
      "Epoch: [ 0] [  197/  468] time: 18.8377, train_loss: 0.17586815, train_accuracy: 0.9375, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  198/  468] time: 18.9319, train_loss: 0.22049290, train_accuracy: 0.9375, test_Accuracy: 0.9386\n",
      "Epoch: [ 0] [  199/  468] time: 19.0274, train_loss: 0.22802082, train_accuracy: 0.9375, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [  200/  468] time: 19.1187, train_loss: 0.17821144, train_accuracy: 0.9609, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [  201/  468] time: 19.2081, train_loss: 0.19715571, train_accuracy: 0.9297, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  202/  468] time: 19.3016, train_loss: 0.20144027, train_accuracy: 0.9297, test_Accuracy: 0.9434\n",
      "Epoch: [ 0] [  203/  468] time: 19.3940, train_loss: 0.18203083, train_accuracy: 0.9609, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  204/  468] time: 19.4841, train_loss: 0.11385176, train_accuracy: 0.9766, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  205/  468] time: 19.5764, train_loss: 0.12362109, train_accuracy: 0.9688, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  206/  468] time: 19.6662, train_loss: 0.12244247, train_accuracy: 0.9609, test_Accuracy: 0.9438\n",
      "Epoch: [ 0] [  207/  468] time: 19.7623, train_loss: 0.14495848, train_accuracy: 0.9609, test_Accuracy: 0.9408\n",
      "Epoch: [ 0] [  208/  468] time: 19.8529, train_loss: 0.16843583, train_accuracy: 0.9453, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  209/  468] time: 19.9437, train_loss: 0.13343801, train_accuracy: 0.9688, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [  210/  468] time: 20.0365, train_loss: 0.12092042, train_accuracy: 0.9688, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  211/  468] time: 20.1290, train_loss: 0.20164937, train_accuracy: 0.9219, test_Accuracy: 0.9386\n",
      "Epoch: [ 0] [  212/  468] time: 20.2209, train_loss: 0.12445455, train_accuracy: 0.9609, test_Accuracy: 0.9402\n",
      "Epoch: [ 0] [  213/  468] time: 20.3138, train_loss: 0.21859488, train_accuracy: 0.9141, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  214/  468] time: 20.4070, train_loss: 0.14586233, train_accuracy: 0.9453, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  215/  468] time: 20.4986, train_loss: 0.19166367, train_accuracy: 0.9453, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  216/  468] time: 20.5939, train_loss: 0.10050358, train_accuracy: 0.9766, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  217/  468] time: 20.6854, train_loss: 0.16802615, train_accuracy: 0.9531, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  218/  468] time: 20.7772, train_loss: 0.16531524, train_accuracy: 0.9531, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  219/  468] time: 20.8718, train_loss: 0.17353405, train_accuracy: 0.9297, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  220/  468] time: 20.9662, train_loss: 0.14923529, train_accuracy: 0.9375, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [  221/  468] time: 21.0608, train_loss: 0.30725348, train_accuracy: 0.8672, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  222/  468] time: 21.1542, train_loss: 0.16151872, train_accuracy: 0.9375, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [  223/  468] time: 21.2454, train_loss: 0.08902934, train_accuracy: 0.9922, test_Accuracy: 0.9408\n",
      "Epoch: [ 0] [  224/  468] time: 21.3381, train_loss: 0.28473637, train_accuracy: 0.9375, test_Accuracy: 0.9365\n",
      "Epoch: [ 0] [  225/  468] time: 21.4344, train_loss: 0.22524267, train_accuracy: 0.9219, test_Accuracy: 0.9396\n",
      "Epoch: [ 0] [  226/  468] time: 21.5266, train_loss: 0.19467917, train_accuracy: 0.9609, test_Accuracy: 0.9408\n",
      "Epoch: [ 0] [  227/  468] time: 21.6207, train_loss: 0.12155532, train_accuracy: 0.9531, test_Accuracy: 0.9423\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  228/  468] time: 21.7107, train_loss: 0.10918133, train_accuracy: 0.9609, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  229/  468] time: 21.8019, train_loss: 0.16011849, train_accuracy: 0.9375, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [  230/  468] time: 21.8965, train_loss: 0.25482100, train_accuracy: 0.9375, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [  231/  468] time: 21.9840, train_loss: 0.29531652, train_accuracy: 0.9219, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  232/  468] time: 22.0775, train_loss: 0.16539118, train_accuracy: 0.9453, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  233/  468] time: 22.1711, train_loss: 0.22030769, train_accuracy: 0.9453, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  234/  468] time: 22.2616, train_loss: 0.20346518, train_accuracy: 0.9375, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [  235/  468] time: 22.3556, train_loss: 0.19502532, train_accuracy: 0.9375, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  236/  468] time: 22.4494, train_loss: 0.19169007, train_accuracy: 0.9297, test_Accuracy: 0.9459\n",
      "Epoch: [ 0] [  237/  468] time: 22.5434, train_loss: 0.19518711, train_accuracy: 0.9531, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  238/  468] time: 22.6371, train_loss: 0.13793960, train_accuracy: 0.9609, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  239/  468] time: 22.7297, train_loss: 0.22838797, train_accuracy: 0.9375, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  240/  468] time: 22.8222, train_loss: 0.11011869, train_accuracy: 0.9609, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  241/  468] time: 22.9157, train_loss: 0.19365633, train_accuracy: 0.9297, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  242/  468] time: 23.0094, train_loss: 0.34286979, train_accuracy: 0.8906, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  243/  468] time: 23.1019, train_loss: 0.16678268, train_accuracy: 0.9297, test_Accuracy: 0.9438\n",
      "Epoch: [ 0] [  244/  468] time: 23.1942, train_loss: 0.10796768, train_accuracy: 0.9609, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  245/  468] time: 23.2863, train_loss: 0.10097244, train_accuracy: 0.9609, test_Accuracy: 0.9435\n",
      "Epoch: [ 0] [  246/  468] time: 23.3765, train_loss: 0.21792078, train_accuracy: 0.9219, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  247/  468] time: 23.4657, train_loss: 0.20434815, train_accuracy: 0.9375, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  248/  468] time: 23.5576, train_loss: 0.15008843, train_accuracy: 0.9688, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  249/  468] time: 23.6469, train_loss: 0.19350617, train_accuracy: 0.9609, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  250/  468] time: 23.7489, train_loss: 0.14848307, train_accuracy: 0.9453, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  251/  468] time: 23.8427, train_loss: 0.18365359, train_accuracy: 0.9375, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  252/  468] time: 23.9343, train_loss: 0.23636797, train_accuracy: 0.9141, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [  253/  468] time: 24.0283, train_loss: 0.16182432, train_accuracy: 0.9531, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  254/  468] time: 24.1228, train_loss: 0.17795119, train_accuracy: 0.9531, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [  255/  468] time: 24.2189, train_loss: 0.20471475, train_accuracy: 0.9453, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [  256/  468] time: 24.3090, train_loss: 0.13279235, train_accuracy: 0.9609, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  257/  468] time: 24.4029, train_loss: 0.18378225, train_accuracy: 0.9453, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  258/  468] time: 24.4968, train_loss: 0.18550469, train_accuracy: 0.9453, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  259/  468] time: 24.5885, train_loss: 0.22784905, train_accuracy: 0.9375, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  260/  468] time: 24.6796, train_loss: 0.11382699, train_accuracy: 0.9609, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  261/  468] time: 24.7724, train_loss: 0.15936178, train_accuracy: 0.9609, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  262/  468] time: 24.8646, train_loss: 0.18245919, train_accuracy: 0.9688, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  263/  468] time: 24.9574, train_loss: 0.19684342, train_accuracy: 0.9453, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  264/  468] time: 25.0511, train_loss: 0.17210098, train_accuracy: 0.9219, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  265/  468] time: 25.1444, train_loss: 0.24408373, train_accuracy: 0.9297, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  266/  468] time: 25.2366, train_loss: 0.23860192, train_accuracy: 0.9219, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  267/  468] time: 25.3333, train_loss: 0.10311227, train_accuracy: 0.9766, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  268/  468] time: 25.4240, train_loss: 0.24750853, train_accuracy: 0.9297, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  269/  468] time: 25.5169, train_loss: 0.21255845, train_accuracy: 0.9297, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  270/  468] time: 25.6111, train_loss: 0.13305621, train_accuracy: 0.9766, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  271/  468] time: 25.7089, train_loss: 0.18233192, train_accuracy: 0.9609, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  272/  468] time: 25.8035, train_loss: 0.19636065, train_accuracy: 0.9297, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  273/  468] time: 25.8916, train_loss: 0.13935848, train_accuracy: 0.9609, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  274/  468] time: 25.9834, train_loss: 0.13924435, train_accuracy: 0.9688, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  275/  468] time: 26.0762, train_loss: 0.18764479, train_accuracy: 0.9531, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  276/  468] time: 26.1710, train_loss: 0.13322043, train_accuracy: 0.9531, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  277/  468] time: 26.2657, train_loss: 0.28026262, train_accuracy: 0.9375, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  278/  468] time: 26.3562, train_loss: 0.16867258, train_accuracy: 0.9453, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  279/  468] time: 26.4505, train_loss: 0.26346612, train_accuracy: 0.9141, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  280/  468] time: 26.5467, train_loss: 0.13476571, train_accuracy: 0.9531, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  281/  468] time: 26.6419, train_loss: 0.13907048, train_accuracy: 0.9531, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  282/  468] time: 26.7398, train_loss: 0.18574470, train_accuracy: 0.9609, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  283/  468] time: 26.8357, train_loss: 0.11681106, train_accuracy: 0.9688, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  284/  468] time: 26.9297, train_loss: 0.22090968, train_accuracy: 0.9297, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  285/  468] time: 27.0195, train_loss: 0.08431752, train_accuracy: 0.9766, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  286/  468] time: 27.1082, train_loss: 0.15431583, train_accuracy: 0.9531, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  287/  468] time: 27.2042, train_loss: 0.17277515, train_accuracy: 0.9453, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  288/  468] time: 27.2957, train_loss: 0.16563293, train_accuracy: 0.9609, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  289/  468] time: 27.3887, train_loss: 0.14710826, train_accuracy: 0.9531, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  290/  468] time: 27.4855, train_loss: 0.23141113, train_accuracy: 0.8984, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  291/  468] time: 27.5833, train_loss: 0.18191066, train_accuracy: 0.9375, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  292/  468] time: 27.6719, train_loss: 0.18958582, train_accuracy: 0.9453, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  293/  468] time: 27.7667, train_loss: 0.14865693, train_accuracy: 0.9531, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  294/  468] time: 27.8573, train_loss: 0.23204303, train_accuracy: 0.9297, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  295/  468] time: 27.9510, train_loss: 0.22735515, train_accuracy: 0.9453, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  296/  468] time: 28.0440, train_loss: 0.08710016, train_accuracy: 0.9922, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  297/  468] time: 28.1378, train_loss: 0.17789470, train_accuracy: 0.9219, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  298/  468] time: 28.2276, train_loss: 0.10725912, train_accuracy: 0.9688, test_Accuracy: 0.9480\n",
      "Epoch: [ 0] [  299/  468] time: 28.3217, train_loss: 0.21306345, train_accuracy: 0.9375, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  300/  468] time: 28.4159, train_loss: 0.21139650, train_accuracy: 0.9375, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  301/  468] time: 28.5104, train_loss: 0.13717096, train_accuracy: 0.9375, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  302/  468] time: 28.6076, train_loss: 0.19963351, train_accuracy: 0.9609, test_Accuracy: 0.9511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  303/  468] time: 28.6971, train_loss: 0.17693420, train_accuracy: 0.9375, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  304/  468] time: 28.7939, train_loss: 0.19185643, train_accuracy: 0.9375, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  305/  468] time: 28.8880, train_loss: 0.22893812, train_accuracy: 0.9453, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  306/  468] time: 28.9874, train_loss: 0.10493398, train_accuracy: 0.9688, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  307/  468] time: 29.0775, train_loss: 0.17147568, train_accuracy: 0.9609, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  308/  468] time: 29.1705, train_loss: 0.16677657, train_accuracy: 0.9531, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  309/  468] time: 29.2631, train_loss: 0.12627694, train_accuracy: 0.9766, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  310/  468] time: 29.3583, train_loss: 0.19799855, train_accuracy: 0.9453, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  311/  468] time: 29.4523, train_loss: 0.11391820, train_accuracy: 0.9766, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  312/  468] time: 29.5493, train_loss: 0.15013006, train_accuracy: 0.9609, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  313/  468] time: 29.6444, train_loss: 0.24815667, train_accuracy: 0.9297, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  314/  468] time: 29.7377, train_loss: 0.10693701, train_accuracy: 0.9844, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  315/  468] time: 29.8280, train_loss: 0.15381438, train_accuracy: 0.9453, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  316/  468] time: 29.9222, train_loss: 0.15967625, train_accuracy: 0.9453, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  317/  468] time: 30.0178, train_loss: 0.16643420, train_accuracy: 0.9531, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  318/  468] time: 30.1119, train_loss: 0.13255224, train_accuracy: 0.9688, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  319/  468] time: 30.2062, train_loss: 0.21250977, train_accuracy: 0.9297, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  320/  468] time: 30.3004, train_loss: 0.22517279, train_accuracy: 0.9375, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  321/  468] time: 30.3949, train_loss: 0.12167999, train_accuracy: 0.9766, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  322/  468] time: 30.4890, train_loss: 0.10953020, train_accuracy: 0.9688, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  323/  468] time: 30.5846, train_loss: 0.20239654, train_accuracy: 0.9375, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  324/  468] time: 30.6751, train_loss: 0.15275079, train_accuracy: 0.9531, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  325/  468] time: 30.7692, train_loss: 0.10046491, train_accuracy: 0.9766, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  326/  468] time: 30.8622, train_loss: 0.19786739, train_accuracy: 0.9375, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  327/  468] time: 30.9561, train_loss: 0.12886080, train_accuracy: 0.9688, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  328/  468] time: 31.0478, train_loss: 0.14623518, train_accuracy: 0.9531, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  329/  468] time: 31.1374, train_loss: 0.12393960, train_accuracy: 0.9688, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  330/  468] time: 31.2282, train_loss: 0.21061862, train_accuracy: 0.9141, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  331/  468] time: 31.3181, train_loss: 0.12639229, train_accuracy: 0.9688, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  332/  468] time: 31.4128, train_loss: 0.12419917, train_accuracy: 0.9609, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  333/  468] time: 31.5064, train_loss: 0.14071141, train_accuracy: 0.9688, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  334/  468] time: 31.5966, train_loss: 0.10167953, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  335/  468] time: 31.6891, train_loss: 0.18107405, train_accuracy: 0.9531, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  336/  468] time: 31.7783, train_loss: 0.16963013, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  337/  468] time: 31.8722, train_loss: 0.15360509, train_accuracy: 0.9375, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  338/  468] time: 31.9676, train_loss: 0.19270116, train_accuracy: 0.9453, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  339/  468] time: 32.0595, train_loss: 0.14556682, train_accuracy: 0.9531, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  340/  468] time: 32.1532, train_loss: 0.21152388, train_accuracy: 0.9297, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  341/  468] time: 32.2463, train_loss: 0.15967637, train_accuracy: 0.9688, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  342/  468] time: 32.3431, train_loss: 0.17132829, train_accuracy: 0.9375, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  343/  468] time: 32.4398, train_loss: 0.12416410, train_accuracy: 0.9531, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  344/  468] time: 32.5326, train_loss: 0.25265795, train_accuracy: 0.9141, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  345/  468] time: 32.6279, train_loss: 0.07808805, train_accuracy: 0.9844, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  346/  468] time: 32.7223, train_loss: 0.19045606, train_accuracy: 0.9375, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  347/  468] time: 32.8191, train_loss: 0.16811639, train_accuracy: 0.9531, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  348/  468] time: 32.9116, train_loss: 0.06125484, train_accuracy: 1.0000, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  349/  468] time: 33.0086, train_loss: 0.24751085, train_accuracy: 0.9375, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  350/  468] time: 33.1056, train_loss: 0.19033515, train_accuracy: 0.9531, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  351/  468] time: 33.1956, train_loss: 0.22441137, train_accuracy: 0.9375, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  352/  468] time: 33.2903, train_loss: 0.13363117, train_accuracy: 0.9609, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  353/  468] time: 33.3824, train_loss: 0.13034469, train_accuracy: 0.9375, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  354/  468] time: 33.4739, train_loss: 0.24553636, train_accuracy: 0.9219, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  355/  468] time: 33.5637, train_loss: 0.10840752, train_accuracy: 0.9609, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  356/  468] time: 33.6597, train_loss: 0.13518280, train_accuracy: 0.9688, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  357/  468] time: 33.7554, train_loss: 0.18411824, train_accuracy: 0.9297, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  358/  468] time: 33.8521, train_loss: 0.13022524, train_accuracy: 0.9609, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  359/  468] time: 33.9449, train_loss: 0.19780341, train_accuracy: 0.9609, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  360/  468] time: 34.0363, train_loss: 0.08195938, train_accuracy: 0.9688, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  361/  468] time: 34.1270, train_loss: 0.25139996, train_accuracy: 0.9297, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  362/  468] time: 34.2211, train_loss: 0.14191064, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  363/  468] time: 34.3109, train_loss: 0.14259803, train_accuracy: 0.9609, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  364/  468] time: 34.4021, train_loss: 0.10352658, train_accuracy: 0.9766, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  365/  468] time: 34.4990, train_loss: 0.09481404, train_accuracy: 0.9609, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  366/  468] time: 34.5943, train_loss: 0.07564222, train_accuracy: 0.9844, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  367/  468] time: 34.6866, train_loss: 0.12001048, train_accuracy: 0.9609, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  368/  468] time: 34.7796, train_loss: 0.07473579, train_accuracy: 0.9688, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  369/  468] time: 34.8742, train_loss: 0.15396728, train_accuracy: 0.9297, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  370/  468] time: 34.9683, train_loss: 0.07639180, train_accuracy: 0.9766, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  371/  468] time: 35.0611, train_loss: 0.14727776, train_accuracy: 0.9609, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  372/  468] time: 35.1557, train_loss: 0.12532800, train_accuracy: 0.9766, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  373/  468] time: 35.2507, train_loss: 0.23404920, train_accuracy: 0.9531, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  374/  468] time: 35.3464, train_loss: 0.15596867, train_accuracy: 0.9375, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  375/  468] time: 35.4404, train_loss: 0.22370099, train_accuracy: 0.9453, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  376/  468] time: 35.5345, train_loss: 0.13621882, train_accuracy: 0.9297, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  377/  468] time: 35.6290, train_loss: 0.08610551, train_accuracy: 0.9844, test_Accuracy: 0.9568\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  378/  468] time: 35.7245, train_loss: 0.13656595, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  379/  468] time: 35.8200, train_loss: 0.11856002, train_accuracy: 0.9609, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  380/  468] time: 35.9142, train_loss: 0.16988833, train_accuracy: 0.9453, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  381/  468] time: 36.0067, train_loss: 0.14984861, train_accuracy: 0.9453, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  382/  468] time: 36.1010, train_loss: 0.19075733, train_accuracy: 0.9453, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  383/  468] time: 36.1958, train_loss: 0.13548537, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  384/  468] time: 36.2906, train_loss: 0.33538210, train_accuracy: 0.9062, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  385/  468] time: 36.3815, train_loss: 0.15151492, train_accuracy: 0.9531, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  386/  468] time: 36.4774, train_loss: 0.24722907, train_accuracy: 0.9609, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  387/  468] time: 36.5732, train_loss: 0.09027052, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  388/  468] time: 36.6683, train_loss: 0.13362905, train_accuracy: 0.9609, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  389/  468] time: 36.7584, train_loss: 0.13509861, train_accuracy: 0.9609, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  390/  468] time: 36.8548, train_loss: 0.12166695, train_accuracy: 0.9688, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  391/  468] time: 36.9510, train_loss: 0.11986200, train_accuracy: 0.9531, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  392/  468] time: 37.0464, train_loss: 0.23029834, train_accuracy: 0.9219, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  393/  468] time: 37.1422, train_loss: 0.17843014, train_accuracy: 0.9609, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  394/  468] time: 37.2386, train_loss: 0.16706124, train_accuracy: 0.9609, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  395/  468] time: 37.3323, train_loss: 0.13961460, train_accuracy: 0.9609, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  396/  468] time: 37.4281, train_loss: 0.11614539, train_accuracy: 0.9531, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  397/  468] time: 37.5221, train_loss: 0.14433379, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  398/  468] time: 37.6148, train_loss: 0.09826241, train_accuracy: 0.9844, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  399/  468] time: 37.7108, train_loss: 0.04017778, train_accuracy: 0.9922, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  400/  468] time: 37.8054, train_loss: 0.13946170, train_accuracy: 0.9688, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  401/  468] time: 37.8968, train_loss: 0.19891578, train_accuracy: 0.9375, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  402/  468] time: 37.9902, train_loss: 0.07024239, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  403/  468] time: 38.0835, train_loss: 0.13314302, train_accuracy: 0.9609, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  404/  468] time: 38.1776, train_loss: 0.16047665, train_accuracy: 0.9609, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  405/  468] time: 38.2723, train_loss: 0.04645206, train_accuracy: 0.9922, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  406/  468] time: 38.3701, train_loss: 0.10772878, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  407/  468] time: 38.4593, train_loss: 0.19658870, train_accuracy: 0.9219, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  408/  468] time: 38.5480, train_loss: 0.20781910, train_accuracy: 0.9375, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  409/  468] time: 38.6413, train_loss: 0.18207651, train_accuracy: 0.9531, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  410/  468] time: 38.7358, train_loss: 0.23913187, train_accuracy: 0.9297, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  411/  468] time: 38.8309, train_loss: 0.14752543, train_accuracy: 0.9531, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  412/  468] time: 38.9282, train_loss: 0.10294898, train_accuracy: 0.9766, test_Accuracy: 0.9596\n",
      "Epoch: [ 0] [  413/  468] time: 39.0229, train_loss: 0.17505163, train_accuracy: 0.9375, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  414/  468] time: 39.1170, train_loss: 0.11045585, train_accuracy: 0.9766, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  415/  468] time: 39.2065, train_loss: 0.19455895, train_accuracy: 0.9297, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  416/  468] time: 39.3015, train_loss: 0.19989589, train_accuracy: 0.9453, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  417/  468] time: 39.3914, train_loss: 0.14106813, train_accuracy: 0.9531, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  418/  468] time: 39.4866, train_loss: 0.07584270, train_accuracy: 0.9844, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  419/  468] time: 39.5841, train_loss: 0.11940459, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  420/  468] time: 39.6760, train_loss: 0.10793650, train_accuracy: 0.9688, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  421/  468] time: 39.7689, train_loss: 0.17814448, train_accuracy: 0.9531, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  422/  468] time: 39.8596, train_loss: 0.10230882, train_accuracy: 0.9766, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  423/  468] time: 39.9568, train_loss: 0.13292009, train_accuracy: 0.9609, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  424/  468] time: 40.0518, train_loss: 0.10862310, train_accuracy: 0.9844, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  425/  468] time: 40.1507, train_loss: 0.08237278, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  426/  468] time: 40.2454, train_loss: 0.15492743, train_accuracy: 0.9453, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  427/  468] time: 40.3412, train_loss: 0.11073257, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  428/  468] time: 40.4365, train_loss: 0.06649192, train_accuracy: 0.9688, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  429/  468] time: 40.5319, train_loss: 0.19939914, train_accuracy: 0.9531, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  430/  468] time: 40.6272, train_loss: 0.13127482, train_accuracy: 0.9609, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  431/  468] time: 40.7278, train_loss: 0.13603687, train_accuracy: 0.9609, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  432/  468] time: 40.8240, train_loss: 0.09179524, train_accuracy: 0.9766, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  433/  468] time: 40.9152, train_loss: 0.07507274, train_accuracy: 0.9844, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  434/  468] time: 41.0042, train_loss: 0.10633321, train_accuracy: 0.9766, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  435/  468] time: 41.0949, train_loss: 0.04940066, train_accuracy: 0.9844, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  436/  468] time: 41.1957, train_loss: 0.08432339, train_accuracy: 0.9766, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  437/  468] time: 41.2858, train_loss: 0.17946392, train_accuracy: 0.9375, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  438/  468] time: 41.3763, train_loss: 0.08256523, train_accuracy: 0.9766, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  439/  468] time: 41.4725, train_loss: 0.11657157, train_accuracy: 0.9688, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  440/  468] time: 41.5633, train_loss: 0.12953103, train_accuracy: 0.9531, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  441/  468] time: 41.6552, train_loss: 0.14964452, train_accuracy: 0.9453, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  442/  468] time: 41.7462, train_loss: 0.13115615, train_accuracy: 0.9688, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  443/  468] time: 41.8368, train_loss: 0.07020365, train_accuracy: 0.9766, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  444/  468] time: 41.9302, train_loss: 0.14535904, train_accuracy: 0.9688, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  445/  468] time: 42.0256, train_loss: 0.15729341, train_accuracy: 0.9453, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  446/  468] time: 42.1205, train_loss: 0.15772304, train_accuracy: 0.9688, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  447/  468] time: 42.2191, train_loss: 0.13609615, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  448/  468] time: 42.3145, train_loss: 0.16543728, train_accuracy: 0.9609, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  449/  468] time: 42.4087, train_loss: 0.13686855, train_accuracy: 0.9375, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  450/  468] time: 42.5080, train_loss: 0.12511337, train_accuracy: 0.9609, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  451/  468] time: 42.6049, train_loss: 0.14103651, train_accuracy: 0.9609, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  452/  468] time: 42.7004, train_loss: 0.13820377, train_accuracy: 0.9531, test_Accuracy: 0.9616\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  453/  468] time: 42.7963, train_loss: 0.14623877, train_accuracy: 0.9531, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  454/  468] time: 42.8869, train_loss: 0.28326234, train_accuracy: 0.8984, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  455/  468] time: 42.9810, train_loss: 0.09695946, train_accuracy: 0.9688, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  456/  468] time: 43.0790, train_loss: 0.09337467, train_accuracy: 0.9844, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  457/  468] time: 43.1805, train_loss: 0.12247287, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  458/  468] time: 43.2767, train_loss: 0.13066933, train_accuracy: 0.9609, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  459/  468] time: 43.3719, train_loss: 0.09534849, train_accuracy: 0.9688, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  460/  468] time: 43.4639, train_loss: 0.06053156, train_accuracy: 0.9922, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  461/  468] time: 43.5593, train_loss: 0.10711938, train_accuracy: 0.9531, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  462/  468] time: 43.6547, train_loss: 0.05261420, train_accuracy: 0.9844, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  463/  468] time: 43.7492, train_loss: 0.09892239, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  464/  468] time: 43.8439, train_loss: 0.12821475, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  465/  468] time: 43.9392, train_loss: 0.13188425, train_accuracy: 0.9453, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  466/  468] time: 44.0343, train_loss: 0.11950347, train_accuracy: 0.9688, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  467/  468] time: 44.1255, train_loss: 0.06610233, train_accuracy: 0.9844, test_Accuracy: 0.9599\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.contrib.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)\n",
    "    global_step = tf.train.create_global_step()\n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)\n",
    "        start_iteration = checkpoint_counter - start_epoch * training_iterations\n",
    "        counter = checkpoint_counter\n",
    "        global_step.assign(checkpoint_counter)\n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx in range(start_iteration, training_iterations):\n",
    "                train_input, train_label = train_iterator.get_next()\n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables), global_step=global_step)\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                test_input, test_label = test_iterator.get_next()\n",
    "                test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.contrib.summary.scalar(name='train_loss', tensor=train_loss)\n",
    "                tf.contrib.summary.scalar(name='train_accuracy', tensor=train_accuracy)\n",
    "                tf.contrib.summary.scalar(name='test_accuracy', tensor=test_accuracy)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    test_input, test_label = test_iterator.get_next()\n",
    "    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 96.22 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}