05-CNN-For-Image-Classification/02-cifar-cnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.2 신경망 깊게 쌓아 컬러 데이터셋에 적용하기\n",
    "Convolutional Neural Network (CNN) 을 쌓아올려 딥한 러닝을 해봅시다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torchvision import transforms, datasets, models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(42)\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "DEVICE = torch.device(\"cuda\" if USE_CUDA else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 하이퍼파라미터 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS     = 300\n",
    "BATCH_SIZE = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 데이터셋 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.CIFAR10('./.data',\n",
    "                   train=True,\n",
    "                   download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.RandomCrop(32, padding=4),\n",
    "                       transforms.RandomHorizontalFlip(),\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.5, 0.5, 0.5),\n",
    "                                            (0.5, 0.5, 0.5))])),\n",
    "    batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.CIFAR10('./.data',\n",
    "                   train=False, \n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.5, 0.5, 0.5),\n",
    "                                            (0.5, 0.5, 0.5))])),\n",
    "    batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet 모델 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicBlock(nn.Module):\n",
    "    def __init__(self, in_planes, planes, stride=1):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3,\n",
    "                               stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(planes)\n",
    "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,\n",
    "                               stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(planes)\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_planes != planes:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_planes, planes,\n",
    "                          kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(planes)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        out += self.shortcut(x)\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_planes = 16\n",
    "\n",
    "        self.conv1 = nn.Conv2d(3, 16, kernel_size=3,\n",
    "                               stride=1, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(16)\n",
    "        self.layer1 = self._make_layer(16, 2, stride=1)\n",
    "        self.layer2 = self._make_layer(32, 2, stride=2)\n",
    "        self.layer3 = self._make_layer(64, 2, stride=2)\n",
    "        self.linear = nn.Linear(64, num_classes)\n",
    "\n",
    "    def _make_layer(self, planes, num_blocks, stride):\n",
    "        strides = [stride] + [1]*(num_blocks-1)\n",
    "        layers = []\n",
    "        for stride in strides:\n",
    "            layers.append(BasicBlock(self.in_planes, planes, stride))\n",
    "            self.in_planes = planes\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = F.avg_pool2d(out, 8)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.linear(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 준비"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ResNet().to(DEVICE)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1,\n",
    "                      momentum=0.9, weight_decay=0.0005)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet(\n",
      "  (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (layer1): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (layer2): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential(\n",
      "        (0): Conv2d(16, 32, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (layer3): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential(\n",
      "        (0): Conv2d(32, 64, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (shortcut): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (linear): Linear(in_features=64, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 훈련하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(DEVICE), target.to(DEVICE)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 테스트하기\n",
    "\n",
    "아무리 훈련이 잘 되었다고 해도 실제 데이터를 만났을때 성능이 낮다면 쓸모 없는 모델일 것입니다. 우리가 진정 원하는 것은 훈련 데이터에 최적화한 모델이 아니라 모든 데이터에서 높은 성능을 보이는 모델이기 때문입니다. 세상에 존재하는 모든 데이터에 최적화 하는 것을 \"일반화\"라고 부르고 모델이 얼마나 실제 데이터에 적응하는지를 수치로 나타낸 것을 \"일반화 오류\"(Generalization Error) 라고 합니다. \n",
    "\n",
    "우리가 만든 모델이 얼마나 일반화를 잘 하는지 알아보기 위해, 그리고 언제 훈련을 멈추어야 할지 알기 위해 매 이포크가 끝날때 마다 테스트셋으로 모델의 성능을 측정해보겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(DEVICE), target.to(DEVICE)\n",
    "            output = model(data)\n",
    "\n",
    "            # sum up batch loss\n",
    "            test_loss += F.cross_entropy(output, target,\n",
    "                                         size_average=False).item()\n",
    "\n",
    "            # get the index of the max log-probability\n",
    "            pred = output.max(1, keepdim=True)[1]\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    test_accuracy = 100. * correct / len(test_loader.dataset)\n",
    "    return test_loss, test_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 코드 돌려보기\n",
    "\n",
    "자, 이제 모든 준비가 끝났습니다. 코드를 돌려서 실제로 훈련이 되는지 확인해봅시다!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] Test Loss: 1.4414, Accuracy: 46.96%\n",
      "[2] Test Loss: 1.6129, Accuracy: 47.86%\n",
      "[3] Test Loss: 1.5045, Accuracy: 52.38%\n",
      "[4] Test Loss: 1.2362, Accuracy: 60.62%\n",
      "[5] Test Loss: 0.8269, Accuracy: 71.20%\n",
      "[6] Test Loss: 0.8386, Accuracy: 71.09%\n",
      "[7] Test Loss: 1.0024, Accuracy: 68.15%\n",
      "[8] Test Loss: 0.7147, Accuracy: 76.10%\n",
      "[9] Test Loss: 0.8604, Accuracy: 70.36%\n",
      "[10] Test Loss: 0.8569, Accuracy: 71.94%\n",
      "[11] Test Loss: 1.1698, Accuracy: 64.97%\n",
      "[12] Test Loss: 1.5368, Accuracy: 61.46%\n",
      "[13] Test Loss: 1.0091, Accuracy: 68.85%\n",
      "[14] Test Loss: 0.7794, Accuracy: 74.58%\n",
      "[15] Test Loss: 0.7072, Accuracy: 76.37%\n",
      "[16] Test Loss: 0.8953, Accuracy: 70.61%\n",
      "[17] Test Loss: 0.7582, Accuracy: 75.88%\n",
      "[18] Test Loss: 0.7556, Accuracy: 75.05%\n",
      "[19] Test Loss: 0.6844, Accuracy: 76.41%\n",
      "[20] Test Loss: 0.6182, Accuracy: 78.93%\n",
      "[21] Test Loss: 0.6343, Accuracy: 78.74%\n",
      "[22] Test Loss: 0.7613, Accuracy: 75.33%\n",
      "[23] Test Loss: 0.8388, Accuracy: 72.89%\n",
      "[24] Test Loss: 0.8615, Accuracy: 73.75%\n",
      "[25] Test Loss: 0.8367, Accuracy: 73.52%\n",
      "[26] Test Loss: 0.5433, Accuracy: 81.42%\n",
      "[27] Test Loss: 0.7660, Accuracy: 75.04%\n",
      "[28] Test Loss: 0.6100, Accuracy: 80.00%\n",
      "[29] Test Loss: 0.6244, Accuracy: 79.77%\n",
      "[30] Test Loss: 0.6506, Accuracy: 78.32%\n",
      "[31] Test Loss: 0.6200, Accuracy: 79.06%\n",
      "[32] Test Loss: 0.6625, Accuracy: 78.24%\n",
      "[33] Test Loss: 0.7980, Accuracy: 74.98%\n",
      "[34] Test Loss: 0.6674, Accuracy: 77.96%\n",
      "[35] Test Loss: 0.6683, Accuracy: 77.90%\n",
      "[36] Test Loss: 0.6715, Accuracy: 77.20%\n",
      "[37] Test Loss: 0.6475, Accuracy: 78.53%\n",
      "[38] Test Loss: 0.7152, Accuracy: 77.04%\n",
      "[39] Test Loss: 0.6273, Accuracy: 78.77%\n",
      "[40] Test Loss: 0.6973, Accuracy: 77.39%\n",
      "[41] Test Loss: 0.6422, Accuracy: 78.81%\n",
      "[42] Test Loss: 0.6126, Accuracy: 79.78%\n",
      "[43] Test Loss: 1.0110, Accuracy: 71.57%\n",
      "[44] Test Loss: 0.6521, Accuracy: 79.08%\n",
      "[45] Test Loss: 0.7520, Accuracy: 75.71%\n",
      "[46] Test Loss: 0.7605, Accuracy: 76.57%\n",
      "[47] Test Loss: 0.7896, Accuracy: 74.72%\n",
      "[48] Test Loss: 0.6081, Accuracy: 80.14%\n",
      "[49] Test Loss: 0.6339, Accuracy: 79.06%\n",
      "[50] Test Loss: 0.6187, Accuracy: 80.16%\n",
      "[51] Test Loss: 0.3445, Accuracy: 88.19%\n",
      "[52] Test Loss: 0.3309, Accuracy: 88.69%\n",
      "[53] Test Loss: 0.3238, Accuracy: 88.90%\n",
      "[54] Test Loss: 0.3180, Accuracy: 89.16%\n",
      "[55] Test Loss: 0.3243, Accuracy: 89.10%\n",
      "[56] Test Loss: 0.3208, Accuracy: 89.50%\n",
      "[57] Test Loss: 0.3218, Accuracy: 89.34%\n",
      "[58] Test Loss: 0.3220, Accuracy: 89.25%\n",
      "[59] Test Loss: 0.3182, Accuracy: 89.55%\n",
      "[60] Test Loss: 0.3151, Accuracy: 89.48%\n",
      "[61] Test Loss: 0.3091, Accuracy: 89.61%\n",
      "[62] Test Loss: 0.3248, Accuracy: 89.30%\n",
      "[63] Test Loss: 0.3254, Accuracy: 89.61%\n",
      "[64] Test Loss: 0.3344, Accuracy: 89.14%\n",
      "[65] Test Loss: 0.3521, Accuracy: 88.64%\n",
      "[66] Test Loss: 0.3323, Accuracy: 88.93%\n",
      "[67] Test Loss: 0.3274, Accuracy: 89.43%\n",
      "[68] Test Loss: 0.3278, Accuracy: 89.64%\n",
      "[69] Test Loss: 0.3439, Accuracy: 88.86%\n",
      "[70] Test Loss: 0.3302, Accuracy: 89.44%\n",
      "[71] Test Loss: 0.3370, Accuracy: 89.09%\n",
      "[72] Test Loss: 0.3269, Accuracy: 89.68%\n",
      "[73] Test Loss: 0.3447, Accuracy: 89.06%\n",
      "[74] Test Loss: 0.3705, Accuracy: 87.62%\n",
      "[75] Test Loss: 0.3557, Accuracy: 88.52%\n",
      "[76] Test Loss: 0.3810, Accuracy: 87.94%\n",
      "[77] Test Loss: 0.3673, Accuracy: 88.49%\n",
      "[78] Test Loss: 0.3582, Accuracy: 88.28%\n",
      "[79] Test Loss: 0.3832, Accuracy: 87.99%\n",
      "[80] Test Loss: 0.3742, Accuracy: 87.81%\n",
      "[81] Test Loss: 0.3475, Accuracy: 88.94%\n",
      "[82] Test Loss: 0.3652, Accuracy: 88.56%\n",
      "[83] Test Loss: 0.3416, Accuracy: 89.07%\n",
      "[84] Test Loss: 0.3624, Accuracy: 88.05%\n",
      "[85] Test Loss: 0.3525, Accuracy: 88.37%\n",
      "[86] Test Loss: 0.3638, Accuracy: 88.35%\n",
      "[87] Test Loss: 0.4042, Accuracy: 87.29%\n",
      "[88] Test Loss: 0.3676, Accuracy: 88.05%\n",
      "[89] Test Loss: 0.3552, Accuracy: 88.21%\n",
      "[90] Test Loss: 0.4019, Accuracy: 87.09%\n",
      "[91] Test Loss: 0.4045, Accuracy: 87.24%\n",
      "[92] Test Loss: 0.3736, Accuracy: 87.78%\n",
      "[93] Test Loss: 0.3753, Accuracy: 88.08%\n",
      "[94] Test Loss: 0.3634, Accuracy: 88.42%\n",
      "[95] Test Loss: 0.3889, Accuracy: 87.89%\n",
      "[96] Test Loss: 0.4017, Accuracy: 87.24%\n",
      "[97] Test Loss: 0.3709, Accuracy: 87.89%\n",
      "[98] Test Loss: 0.3554, Accuracy: 88.50%\n",
      "[99] Test Loss: 0.3531, Accuracy: 88.83%\n",
      "[100] Test Loss: 0.3678, Accuracy: 88.14%\n",
      "[101] Test Loss: 0.2922, Accuracy: 90.22%\n",
      "[102] Test Loss: 0.2874, Accuracy: 90.52%\n",
      "[103] Test Loss: 0.2834, Accuracy: 90.66%\n",
      "[104] Test Loss: 0.2865, Accuracy: 90.62%\n",
      "[105] Test Loss: 0.2843, Accuracy: 90.54%\n",
      "[106] Test Loss: 0.2844, Accuracy: 90.80%\n",
      "[107] Test Loss: 0.2867, Accuracy: 90.74%\n",
      "[108] Test Loss: 0.2829, Accuracy: 90.81%\n",
      "[109] Test Loss: 0.2857, Accuracy: 90.75%\n",
      "[110] Test Loss: 0.2884, Accuracy: 90.90%\n",
      "[111] Test Loss: 0.2855, Accuracy: 90.86%\n",
      "[112] Test Loss: 0.2882, Accuracy: 90.84%\n",
      "[113] Test Loss: 0.2876, Accuracy: 90.79%\n",
      "[114] Test Loss: 0.2861, Accuracy: 90.96%\n",
      "[115] Test Loss: 0.2875, Accuracy: 90.92%\n",
      "[116] Test Loss: 0.2918, Accuracy: 90.85%\n",
      "[117] Test Loss: 0.2879, Accuracy: 91.11%\n",
      "[118] Test Loss: 0.2885, Accuracy: 91.04%\n",
      "[119] Test Loss: 0.2918, Accuracy: 90.86%\n",
      "[120] Test Loss: 0.2913, Accuracy: 90.93%\n",
      "[121] Test Loss: 0.2960, Accuracy: 90.93%\n",
      "[122] Test Loss: 0.2917, Accuracy: 90.91%\n",
      "[123] Test Loss: 0.2940, Accuracy: 90.85%\n",
      "[124] Test Loss: 0.2971, Accuracy: 91.05%\n",
      "[125] Test Loss: 0.2990, Accuracy: 90.86%\n",
      "[126] Test Loss: 0.2970, Accuracy: 90.79%\n",
      "[127] Test Loss: 0.2964, Accuracy: 90.79%\n",
      "[128] Test Loss: 0.2964, Accuracy: 90.94%\n",
      "[129] Test Loss: 0.2952, Accuracy: 90.85%\n",
      "[130] Test Loss: 0.2974, Accuracy: 91.03%\n",
      "[131] Test Loss: 0.2980, Accuracy: 90.74%\n",
      "[132] Test Loss: 0.2979, Accuracy: 91.01%\n",
      "[133] Test Loss: 0.2997, Accuracy: 90.90%\n",
      "[134] Test Loss: 0.2991, Accuracy: 91.11%\n",
      "[135] Test Loss: 0.3004, Accuracy: 91.08%\n",
      "[136] Test Loss: 0.2992, Accuracy: 91.01%\n",
      "[137] Test Loss: 0.3000, Accuracy: 91.06%\n",
      "[138] Test Loss: 0.3035, Accuracy: 90.70%\n",
      "[139] Test Loss: 0.3011, Accuracy: 90.91%\n",
      "[140] Test Loss: 0.3007, Accuracy: 90.91%\n",
      "[141] Test Loss: 0.3042, Accuracy: 90.96%\n",
      "[142] Test Loss: 0.3026, Accuracy: 90.78%\n",
      "[143] Test Loss: 0.3036, Accuracy: 90.85%\n",
      "[144] Test Loss: 0.3075, Accuracy: 90.98%\n",
      "[145] Test Loss: 0.3027, Accuracy: 90.96%\n",
      "[146] Test Loss: 0.3026, Accuracy: 90.98%\n",
      "[147] Test Loss: 0.3022, Accuracy: 90.96%\n",
      "[148] Test Loss: 0.3083, Accuracy: 90.87%\n",
      "[149] Test Loss: 0.3042, Accuracy: 90.93%\n",
      "[150] Test Loss: 0.3062, Accuracy: 90.94%\n",
      "[151] Test Loss: 0.3042, Accuracy: 90.80%\n",
      "[152] Test Loss: 0.3056, Accuracy: 90.83%\n",
      "[153] Test Loss: 0.3046, Accuracy: 90.84%\n",
      "[154] Test Loss: 0.3033, Accuracy: 90.85%\n",
      "[155] Test Loss: 0.3037, Accuracy: 90.89%\n",
      "[156] Test Loss: 0.3018, Accuracy: 90.91%\n",
      "[157] Test Loss: 0.3013, Accuracy: 90.92%\n",
      "[158] Test Loss: 0.3040, Accuracy: 91.02%\n",
      "[159] Test Loss: 0.3038, Accuracy: 91.04%\n",
      "[160] Test Loss: 0.3025, Accuracy: 91.07%\n",
      "[161] Test Loss: 0.3031, Accuracy: 91.08%\n",
      "[162] Test Loss: 0.3049, Accuracy: 91.00%\n",
      "[163] Test Loss: 0.3055, Accuracy: 90.95%\n",
      "[164] Test Loss: 0.3034, Accuracy: 90.98%\n",
      "[165] Test Loss: 0.3027, Accuracy: 90.95%\n",
      "[166] Test Loss: 0.3038, Accuracy: 91.02%\n",
      "[167] Test Loss: 0.3050, Accuracy: 91.06%\n",
      "[168] Test Loss: 0.3032, Accuracy: 91.07%\n",
      "[169] Test Loss: 0.3048, Accuracy: 91.03%\n",
      "[170] Test Loss: 0.3048, Accuracy: 91.10%\n",
      "[171] Test Loss: 0.3019, Accuracy: 91.05%\n",
      "[172] Test Loss: 0.3037, Accuracy: 91.13%\n",
      "[173] Test Loss: 0.3050, Accuracy: 91.09%\n",
      "[174] Test Loss: 0.3034, Accuracy: 90.92%\n",
      "[175] Test Loss: 0.3059, Accuracy: 90.95%\n",
      "[176] Test Loss: 0.3063, Accuracy: 90.90%\n",
      "[177] Test Loss: 0.3052, Accuracy: 90.94%\n",
      "[178] Test Loss: 0.3056, Accuracy: 90.96%\n",
      "[179] Test Loss: 0.3039, Accuracy: 91.10%\n",
      "[180] Test Loss: 0.3041, Accuracy: 90.99%\n",
      "[181] Test Loss: 0.3043, Accuracy: 91.12%\n",
      "[182] Test Loss: 0.3032, Accuracy: 91.05%\n",
      "[183] Test Loss: 0.3030, Accuracy: 91.16%\n",
      "[184] Test Loss: 0.3059, Accuracy: 91.02%\n",
      "[185] Test Loss: 0.3046, Accuracy: 91.03%\n",
      "[186] Test Loss: 0.3062, Accuracy: 90.88%\n",
      "[187] Test Loss: 0.3060, Accuracy: 91.02%\n",
      "[188] Test Loss: 0.3045, Accuracy: 91.04%\n",
      "[189] Test Loss: 0.3058, Accuracy: 91.11%\n",
      "[190] Test Loss: 0.3053, Accuracy: 90.99%\n",
      "[191] Test Loss: 0.3069, Accuracy: 91.07%\n",
      "[192] Test Loss: 0.3077, Accuracy: 91.07%\n",
      "[193] Test Loss: 0.3056, Accuracy: 90.95%\n",
      "[194] Test Loss: 0.3043, Accuracy: 91.04%\n",
      "[195] Test Loss: 0.3056, Accuracy: 91.05%\n",
      "[196] Test Loss: 0.3063, Accuracy: 90.97%\n",
      "[197] Test Loss: 0.3054, Accuracy: 90.95%\n",
      "[198] Test Loss: 0.3051, Accuracy: 91.17%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[199] Test Loss: 0.3071, Accuracy: 91.11%\n",
      "[200] Test Loss: 0.3058, Accuracy: 91.04%\n",
      "[201] Test Loss: 0.3044, Accuracy: 91.02%\n",
      "[202] Test Loss: 0.3075, Accuracy: 91.02%\n",
      "[203] Test Loss: 0.3071, Accuracy: 90.97%\n",
      "[204] Test Loss: 0.3071, Accuracy: 90.99%\n",
      "[205] Test Loss: 0.3064, Accuracy: 91.04%\n",
      "[206] Test Loss: 0.3072, Accuracy: 91.02%\n",
      "[207] Test Loss: 0.3032, Accuracy: 91.03%\n",
      "[208] Test Loss: 0.3071, Accuracy: 91.03%\n",
      "[209] Test Loss: 0.3022, Accuracy: 91.18%\n",
      "[210] Test Loss: 0.3029, Accuracy: 91.14%\n",
      "[211] Test Loss: 0.3062, Accuracy: 91.21%\n",
      "[212] Test Loss: 0.3042, Accuracy: 91.05%\n",
      "[213] Test Loss: 0.3073, Accuracy: 91.00%\n",
      "[214] Test Loss: 0.3039, Accuracy: 91.13%\n",
      "[215] Test Loss: 0.3068, Accuracy: 91.05%\n",
      "[216] Test Loss: 0.3042, Accuracy: 91.05%\n",
      "[217] Test Loss: 0.3063, Accuracy: 91.00%\n",
      "[218] Test Loss: 0.3066, Accuracy: 90.97%\n",
      "[219] Test Loss: 0.3053, Accuracy: 91.07%\n",
      "[220] Test Loss: 0.3068, Accuracy: 91.12%\n",
      "[221] Test Loss: 0.3055, Accuracy: 91.13%\n",
      "[222] Test Loss: 0.3032, Accuracy: 91.12%\n",
      "[223] Test Loss: 0.3070, Accuracy: 91.14%\n",
      "[224] Test Loss: 0.3051, Accuracy: 91.05%\n",
      "[225] Test Loss: 0.3049, Accuracy: 91.17%\n",
      "[226] Test Loss: 0.3053, Accuracy: 91.07%\n",
      "[227] Test Loss: 0.3049, Accuracy: 91.19%\n",
      "[228] Test Loss: 0.3061, Accuracy: 91.01%\n",
      "[229] Test Loss: 0.3071, Accuracy: 90.93%\n",
      "[230] Test Loss: 0.3073, Accuracy: 91.02%\n",
      "[231] Test Loss: 0.3045, Accuracy: 91.11%\n",
      "[232] Test Loss: 0.3078, Accuracy: 91.04%\n",
      "[233] Test Loss: 0.3045, Accuracy: 91.06%\n",
      "[234] Test Loss: 0.3064, Accuracy: 91.13%\n",
      "[235] Test Loss: 0.3054, Accuracy: 91.08%\n",
      "[236] Test Loss: 0.3046, Accuracy: 91.12%\n",
      "[237] Test Loss: 0.3063, Accuracy: 91.12%\n",
      "[238] Test Loss: 0.3055, Accuracy: 91.10%\n",
      "[239] Test Loss: 0.3078, Accuracy: 90.92%\n",
      "[240] Test Loss: 0.3057, Accuracy: 91.12%\n",
      "[241] Test Loss: 0.3049, Accuracy: 91.03%\n",
      "[242] Test Loss: 0.3061, Accuracy: 91.05%\n",
      "[243] Test Loss: 0.3066, Accuracy: 91.00%\n",
      "[244] Test Loss: 0.3073, Accuracy: 91.05%\n",
      "[245] Test Loss: 0.3056, Accuracy: 91.07%\n",
      "[246] Test Loss: 0.3053, Accuracy: 90.93%\n",
      "[247] Test Loss: 0.3034, Accuracy: 91.08%\n",
      "[248] Test Loss: 0.3069, Accuracy: 91.02%\n",
      "[249] Test Loss: 0.3059, Accuracy: 90.99%\n",
      "[250] Test Loss: 0.3061, Accuracy: 91.09%\n",
      "[251] Test Loss: 0.3083, Accuracy: 90.95%\n",
      "[252] Test Loss: 0.3067, Accuracy: 91.09%\n",
      "[253] Test Loss: 0.3065, Accuracy: 91.12%\n",
      "[254] Test Loss: 0.3029, Accuracy: 91.10%\n",
      "[255] Test Loss: 0.3056, Accuracy: 91.08%\n",
      "[256] Test Loss: 0.3069, Accuracy: 91.10%\n",
      "[257] Test Loss: 0.3051, Accuracy: 91.19%\n",
      "[258] Test Loss: 0.3050, Accuracy: 91.05%\n",
      "[259] Test Loss: 0.3058, Accuracy: 91.15%\n",
      "[260] Test Loss: 0.3068, Accuracy: 90.99%\n",
      "[261] Test Loss: 0.3039, Accuracy: 91.09%\n",
      "[262] Test Loss: 0.3055, Accuracy: 90.99%\n",
      "[263] Test Loss: 0.3052, Accuracy: 91.06%\n",
      "[264] Test Loss: 0.3059, Accuracy: 91.08%\n",
      "[265] Test Loss: 0.3067, Accuracy: 91.17%\n",
      "[266] Test Loss: 0.3064, Accuracy: 91.19%\n",
      "[267] Test Loss: 0.3054, Accuracy: 91.13%\n",
      "[268] Test Loss: 0.3047, Accuracy: 90.97%\n",
      "[269] Test Loss: 0.3039, Accuracy: 91.13%\n",
      "[270] Test Loss: 0.3066, Accuracy: 91.07%\n",
      "[271] Test Loss: 0.3072, Accuracy: 91.04%\n",
      "[272] Test Loss: 0.3089, Accuracy: 90.98%\n",
      "[273] Test Loss: 0.3063, Accuracy: 91.12%\n",
      "[274] Test Loss: 0.3081, Accuracy: 91.01%\n",
      "[275] Test Loss: 0.3066, Accuracy: 91.10%\n",
      "[276] Test Loss: 0.3044, Accuracy: 91.00%\n",
      "[277] Test Loss: 0.3062, Accuracy: 91.02%\n",
      "[278] Test Loss: 0.3065, Accuracy: 91.11%\n",
      "[279] Test Loss: 0.3029, Accuracy: 91.04%\n",
      "[280] Test Loss: 0.3058, Accuracy: 91.07%\n",
      "[281] Test Loss: 0.3061, Accuracy: 91.11%\n",
      "[282] Test Loss: 0.3076, Accuracy: 91.11%\n",
      "[283] Test Loss: 0.3059, Accuracy: 91.11%\n",
      "[284] Test Loss: 0.3054, Accuracy: 91.10%\n",
      "[285] Test Loss: 0.3042, Accuracy: 91.19%\n",
      "[286] Test Loss: 0.3067, Accuracy: 90.89%\n",
      "[287] Test Loss: 0.3051, Accuracy: 91.12%\n",
      "[288] Test Loss: 0.3044, Accuracy: 91.16%\n",
      "[289] Test Loss: 0.3050, Accuracy: 91.00%\n",
      "[290] Test Loss: 0.3039, Accuracy: 91.12%\n",
      "[291] Test Loss: 0.3044, Accuracy: 91.11%\n",
      "[292] Test Loss: 0.3045, Accuracy: 91.08%\n",
      "[293] Test Loss: 0.3058, Accuracy: 90.95%\n",
      "[294] Test Loss: 0.3050, Accuracy: 91.15%\n",
      "[295] Test Loss: 0.3055, Accuracy: 91.07%\n",
      "[296] Test Loss: 0.3086, Accuracy: 91.03%\n",
      "[297] Test Loss: 0.3060, Accuracy: 91.10%\n",
      "[298] Test Loss: 0.3066, Accuracy: 91.07%\n",
      "[299] Test Loss: 0.3060, Accuracy: 91.03%\n",
      "[300] Test Loss: 0.3084, Accuracy: 91.12%\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, EPOCHS + 1):\n",
    "    scheduler.step()\n",
    "    train(model, train_loader, optimizer, epoch)\n",
    "    test_loss, test_accuracy = test(model, test_loader)\n",
    "    \n",
    "    print('[{}] Test Loss: {:.4f}, Accuracy: {:.2f}%'.format(\n",
    "          epoch, test_loss, test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}