07-순차적인_데이터를_처리하는_RNN/02-sequence-to-sequence.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Seq2Seq 기계 번역\n",
    "\n",
    "이번 프로젝트에선 임의로 Seq2Seq 모델을 아주 간단화 시켰습니다.\n",
    "한 언어로 된 문장을 다른 언어로 된 문장으로 번역하는 덩치가 큰 모델이 아닌\n",
    "영어 알파벳 문자열(\"hello\")을 스페인어 알파벳 문자열(\"hola\")로 번역하는 Mini Seq2Seq 모델을 같이 구현해 보겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import random\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello ->  [104, 101, 108, 108, 111]\n",
      "hola  ->  [104, 111, 108, 97]\n"
     ]
    }
   ],
   "source": [
    "vocab_size = 256  # 총 아스키 코드 개수\n",
    "x_ = list(map(ord, \"hello\"))  # 아스키 코드 리스트로 변환\n",
    "y_ = list(map(ord, \"hola\"))   # 아스키 코드 리스트로 변환\n",
    "print(\"hello -> \", x_)\n",
    "print(\"hola  -> \", y_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.LongTensor(x_)\n",
    "y = torch.LongTensor(y_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size):\n",
    "        super(Seq2Seq, self).__init__()\n",
    "        self.n_layers = 1\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.encoder = nn.GRU(hidden_size, hidden_size)\n",
    "        self.decoder = nn.GRU(hidden_size, hidden_size)\n",
    "        self.project = nn.Linear(hidden_size, vocab_size)\n",
    "\n",
    "    def forward(self, inputs, targets):\n",
    "        # 인코더에 들어갈 입력\n",
    "        initial_state = self._init_state()\n",
    "        embedding = self.embedding(inputs).unsqueeze(1)\n",
    "        # embedding = [seq_len, batch_size, embedding_size]\n",
    "        \n",
    "        # 인코더 (Encoder)\n",
    "        encoder_output, encoder_state = self.encoder(embedding, initial_state)\n",
    "        # encoder_output = [seq_len, batch_size, hidden_size]\n",
    "        # encoder_state  = [n_layers, seq_len, hidden_size]\n",
    "\n",
    "        # 디코더에 들어갈 입력\n",
    "        decoder_state = encoder_state\n",
    "        decoder_input = torch.LongTensor([[0]])\n",
    "        \n",
    "        # 디코더 (Decoder)\n",
    "        outputs = []\n",
    "        \n",
    "        for i in range(targets.size()[0]):\n",
    "            decoder_input = self.embedding(decoder_input)\n",
    "            decoder_output, decoder_state = self.decoder(decoder_input, decoder_state)\n",
    "            projection = self.project(decoder_output)\n",
    "            outputs.append(projection)\n",
    "            \n",
    "            #티처 포싱(Teacher Forcing) 사용\n",
    "            decoder_input = torch.LongTensor([[targets[i]]])\n",
    "\n",
    "        outputs = torch.stack(outputs).squeeze()\n",
    "        return outputs\n",
    "    \n",
    "    def _init_state(self, batch_size=1):\n",
    "        weight = next(self.parameters()).data\n",
    "        return weight.new(self.n_layers, batch_size, self.hidden_size).zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seq2Seq(\n",
      "  (embedding): Embedding(256, 16)\n",
      "  (encoder): GRU(16, 16)\n",
      "  (decoder): GRU(16, 16)\n",
      "  (project): Linear(in_features=16, out_features=256, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "seq2seq = Seq2Seq(vocab_size, 16)\n",
    "print(seq2seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(seq2seq.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 반복:0 오차: 5.538308620452881\n",
      "['q', 'C', '\\x92', '\\x8e']\n",
      "\n",
      " 반복:100 오차: 2.039297103881836\n",
      "['h', 'o', 'a', 'a']\n",
      "\n",
      " 반복:200 오차: 0.5287023782730103\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:300 오차: 0.2585223317146301\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:400 오차: 0.166521817445755\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:500 오차: 0.1202026903629303\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:600 오차: 0.09239096939563751\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:700 오차: 0.07387516647577286\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:800 오차: 0.06072056293487549\n",
      "['h', 'o', 'l', 'a']\n",
      "\n",
      " 반복:900 오차: 0.05094281584024429\n",
      "['h', 'o', 'l', 'a']\n"
     ]
    }
   ],
   "source": [
    "log = []\n",
    "for i in range(1000):\n",
    "    prediction = seq2seq(x, y)\n",
    "    loss = criterion(prediction, y)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    loss_val = loss.data\n",
    "    log.append(loss_val)\n",
    "    if i % 100 == 0:\n",
    "        print(\"\\n 반복:%d 오차: %s\" % (i, loss_val.item()))\n",
    "        _, top1 = prediction.data.topk(1, 1)\n",
    "        print([chr(c) for c in top1.squeeze().numpy().tolist()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYHPV95/H3t3t6bs0ljYRmNNJIiEs3YsBgMLZZwBiD\n/axPbGN7Y2KSrBPb6zybJyRO8mST3WTtxAnOxn4CxneCg23sYBkbGwzB2FgwAoQuhIQOJHTM6JrR\nSJqju7/7R9fIg9BRkqamurs+r+fpp7uO7vqWSs+nfvOry9wdEREpf6m4CxARkYmhwBcRSQgFvohI\nQijwRUQSQoEvIpIQCnwRkYRQ4IuIJIQCX0QkIRT4IiIJURF3AWNNmTLFOzs74y5DRKRkrFixYo+7\nt4aZt6gCv7Ozk+7u7rjLEBEpGWa2Ney86tIREUkIBb6ISEIo8EVEEkKBLyKSEAp8EZGEUOCLiCSE\nAl9EJCFKPvAHR3Lc/fgmfrVxT9yliIgUtZIP/IqUcdcvNvGVX26OuxQRkaJW+oGfTvGupTN4dH0v\nPQcH4y5HRKRolXzgA7ynawa5vPP9Z16JuxQRkaJVFoF/bms9XbOaua97G+4edzkiIkWpLAIfCq38\nl3oP8ey2A3GXIiJSlMom8N+2qI2aTJrvdG+LuxQRkaJUNoFfX1XB2xZN54crd3J4OBt3OSIiRads\nAh/gvV0dDAxl+fGqXXGXIiJSdMoq8C/tbKZzci33qVtHROQ1yirwzYx3LZ3B8s372Nl3JO5yRESK\nSlkFPsDNi9sAWLZyZ8yViIgUl7IL/M4pdSya0cgPn98RdykiIkWl7AIf4OZFbTy/vY/New7FXYqI\nSNEoy8C/afF0AJatVCtfRGRUWQb+9MYaLuts4YGVO3SrBRGRQFkGPsDbFk1nQ88AL/WqW0dEBCIO\nfDPbYmarzOw5M+uOclnHunbeNAAeXrd7IhcrIlK0JqKF/2Z3X+LuXROwrKPam2qY39bAw2sV+CIi\nUMZdOgDXXjSNFS/vZ+/AUNyliIjELurAd+CnZrbCzG6PeFmvcd28abjDz1/omehFi4gUnagD/0p3\nXwq8Ffi4mV197AxmdruZdZtZd29v77gufH5bA9Mbq/mZunVERKINfHffEbz3AN8HLjvOPHe5e5e7\nd7W2to7r8s2May6cyi837mEklx/X3xYRKTWRBb6Z1ZnZpNHPwPXA6qiWdyJvOK+VQ8M5nn1ZT8IS\nkWSLsoU/DXjCzFYCTwE/cvefRLi847ri3MmkDJ7YML7dRSIipaYiqh92903A4qh+P6zGmgyLO5r4\nxcY9fPr6C+IuR0QkNmV9WuaoN8ydwsptB+g7PBJ3KSIisUlE4F91Xit5hyc37Ym7FBGR2CQi8C+e\n2URdZZpfbFDgi0hyJSLwM+kUl81uYfnmfXGXIiISm0QEPsCls1vY2DOg2yyISGIlJvBfN7sFgKe3\n7I+5EhGReCQm8Be2N1FVkeLpLerWEZFkSkzgV1akWNLRpMAXkcRKTOBDoVtn9St9DAxl4y5FRGTC\nJSrwL53dQt7hma3qxxeR5ElU4C+d2Uw6ZerWEZFESlTg11VVcMG0STy3TXfOFJHkSVTgAyyZ2cRz\n2w6Qz3vcpYiITKjkBX5HEwcHs2zacyjuUkREJlTiAv/ijiYAdeuISOIkLvDPba1nUlUFz23TmToi\nkiyJC/xUyljU0ahHHopI4iQu8KHQj//CroMcGc7FXYqIyIRJaOA3k8s7q3f0xV2KiMiESWjgBwdu\n1a0jIgmSyMBvnVRFe1MNz21X4ItIciQy8AEWzWhk9Svq0hGR5Ehs4C9ob2Tr3sP0HRmJuxQRkQmR\n6MAHWKMDtyKSEMkN/LYGANa80h9zJSIiEyOxgT+5voq2xmpWqR9fRBIisYEPML+9Uefii0hiRB74\nZpY2s2fNbFnUyzpdC9sb2bznkB55KCKJMBEt/E8C6yZgOadtQXsD7rB2h/rxRaT8RRr4ZjYDeBvw\n5SiXc6ZGz9RRP76IJEHULfx/BP4IyEe8nDMydVI1UydVsUaBLyIJcMrAN7NPmlmDFdxjZs+Y2fUh\nvncT0OPuK04x3+1m1m1m3b29vadR+vhY2N6oFr6IJEKYFv5H3b0fuB5oBX4L+NsQ37sSeLuZbQG+\nDVxjZt86diZ3v8vdu9y9q7W1NXzl42R+eyMv9Q5weFgHbkWkvIUJfAvebwS+6u4rx4w7IXe/w91n\nuHsncAvwc3e/9YwrjcjC9kbyDut26sCtiJS3MIG/wsx+SiHwHzKzSRRpn/yZWNBeuOJ21XZ164hI\neasIMc9twBJgk7sfNrMWCt06obn7Y8Bjp13dBDinoZop9ZWs0i0WRKTMhWnhXwGsd/cDZnYr8Bmg\nbJrDZsaC9kbdRE1Eyl6YwP8ScNjMFlM4xXIr8I1Iq5pgC9oa2dAzwOCInnErIuUrTOBn3d2BdwB3\nuvudwKRoy5pYC9obyOWdF3YdjLsUEZHIhAn8g2Z2B/Ah4EdmlgYy0ZY1sXTFrYgkQZjAfx8wROF8\n/F1AO/C5SKuaYO1NNTTVZnTFrYiUtVMGfhDy/wo0BlfPDrp7WfXhm5muuBWRshfm1grvBZ4C3gO8\nF1huZu+OurCJNr+tkRd3H2QoqwO3IlKewpyH/6fApe7eA2BmrcDDwHejLGyiLWxvZCTnvLhrgIUz\nGuMuR0Rk3IXpw0+Nhn1gb8jvlZTRK271BCwRKVdhWvg/MbOHgHuD4fcBD0ZXUjxmttQyqbqC1erH\nF5EydcrAd/f/aWbvonD3SwPucvfvR17ZBDMzFrQ1KvBFpGyFaeHj7t8DvhdxLbFb0N7A15/cykgu\nTyZddr1WIpJwJwx8MzsI+PEmAe7uDZFVFZMF7Y0MZ/Ns2D3AvLayWz0RSbgTBr67l9XtE8IYveJ2\n9Y4+Bb6IlB31W4wxe3IddZVpXXErImVJgT9GKmXMb9MVtyJSnhT4x1jQ3sjanf3k8sc7fCEiUrrC\n3Frh982seSKKKQYL2hsYHMnzUu9A3KWIiIyrMC38c4Cnzew+M7vBzE75APNSdvTArbp1RKTMhLlb\n5meA84B7gP8GbDCz/2Nm50ZcWyzOba2nOpNSP76IlJ1QffjBE692Ba8s0Ax818w+G2FtsUinjHnT\nG1ijh5qLSJkJ04f/CTNbAXwW+CWw0N1/D7gEeFfE9cViYfBQ87wO3IpIGQnTwp8CvNPd3+Lu33H3\nEQB3zwM3RVpdTOa3N3JoOMfmvYfiLkVEZNyE6cP/c2By0NL/AzNbOmbaukiri8lCHbgVkTIUpkvn\nz4CvA5MptPa/amafibqwOM2dWk9lRYpV2xX4IlI+wtwt8wPAxe4+CGBmfws8A/x1lIXFKZNOMb+t\ngecV+CJSRsL04W8BqscMVwEvRVJNEVnS0cSqV/rI5vJxlyIiMi7CBP4QsMbMvmZmXwVWAwNm9gUz\n+8KJvmRm1Wb2lJmtNLM1ZvaX41X0RFjS0cSRkRzrdx+MuxQRkXERpkvn+8Fr1GMhf3sIuMbdB8ws\nAzxhZj9291+fZo2xWNLRBMDKbX3Mb9NDzUWk9IV5xOHXzawSOD8YtX701MxTfM+B0RvSZIJXyZzY\nPrOllubaDM9t288HXjcz7nJERM5amLN03gRsAP4Z+CLwopldHebHzSxtZs8BPcDP3H35WdQ6ocyM\nxR1NrNymA7ciUh7C9OH/PXC9u7/R3a8G3gL8Q5gfd/ecuy8BZgCXmdmCY+cxs9vNrNvMunt7e0+n\n9sgt6WjixZ6DDAxl4y5FROSshQn8jLuvHx1w9xcpdM+E5u4HKPT933CcaXe5e5e7d7W2tp7Oz0Zu\ncUcT7uh8fBEpC2ECv9vM7jGzNwWvu4EVp/qSmbWaWVPwuQa4Fnjh7MqdWEtmFA7cPrftQMyViIic\nvTBn6fwe8HHgE4ABj1Poyz+V6cDXzSxNYcdyn7svO9NC49BcV8msybU8t21/3KWIiJy1kwZ+ENb3\nuPutwOdP54fd/Xng4rOorSgs6Whi+aZ9cZchInLWTtql4+45oDU4LTORFs9oYlf/ILv6BuMuRUTk\nrITp0tkC/NLMHgCO3i/Y3U+rxV+qlsz8TT/+DY3nxFyNiMiZC3PQdgewLJh3UvCqj7KoYjJvegOZ\ntOnArYiUvDAt/LXu/p2xI8zsPRHVU3SqM2nmtTXyzFYduBWR0hamhX9HyHFlq2tWMyu3H2A4qztn\nikjpOmEL38zeCtwItB9zV8wGCg8yT4yuWc3c88RmVu/oY+nM5rjLERE5Iydr4e8AuoFBChdajb4e\noHB7hcS4pLMQ8iu2qFtHRErXCVv47r4SWGlm/xbm7pjlbOqkama21NK9dR8fY07c5YiInJEwffiX\nmdnPzOxFM9tkZpvNbFPklRWZrlnNrNi6n8Jdn0VESk+YwL+HwlW2VwGXAl3Be6J0dbawZ2CYLXsP\nx12KiMgZCXNaZp+7/zjySopcV9CP371lH7On1MVcjYjI6QvTwn/UzD5nZleY2dLRV+SVFZm5rfU0\nVFewQufji0iJCtPCf13w3jVmnAPXjH85xSuVMi6Z1Uy3Al9ESlSYZ9q+eSIKKQVdnS08un49Bw4P\n01Sb2PvJiUiJCvNM22nBA1B+HAzPM7Pboi+t+Fwya7QfX618ESk9YfrwvwY8BLQFwy8Cn4qqoGK2\npKOJyooUT23R/fFFpPSECfwp7n4fkAdw9yyQi7SqIlWdSXNxRxNPvrQ37lJERE5bmMA/ZGaTKRyo\nxcwuBxL7VO/L50xmzY4++o4k+uJjESlBYQL/0xTun3Oumf0S+AbwB5FWVcSuOHcyeYenN6tbR0RK\nS5izdJ4xszcCF1B4iPn6JN9bZ7Qf/9eb9nLtvGlxlyMiElqY8/BH++3XRFxLSajOpLlkZjNPblI/\nvoiUljBdOnKMy+dMZu3OfvoOJ/YPHREpQQr8M3D5nBbcYflmtfJFpHSEufDqSjOrCz7famafN7NZ\n0ZdWvJbMbKKqIsWvN+nArYiUjjAt/C8Bh81sMfBHwFYKZ+okVlVFmktmqR9fREpLmMDPeuGpH+8A\n7nT3O4FJ0ZZV/K6YM5l1O/vZf2g47lJEREIJE/gHzewO4FbgR2aWBjLRllX8rjxvCgBPbNwTcyUi\nIuGECfz3AUPAbe6+C2gHPneqL5lZh5k9ambrzGyNmX3yLGstKotnNNFYk+HxF3vjLkVEJJQw5+Ef\npNCVkzOz84ELgXtDfC8L/GFw4dYkYIWZ/czd155FvUUjnTKumjuFxzf04u6YWdwliYicVJgW/uNA\nlZm1A48Av0XhDpon5e473f2Z4PNBYB2Fvw7KxhvOm8Lu/iFe3D0QdykiIqcUJvDN3Q8D7wT+yd3/\nKzD/dBZiZp3AxcDy0y2wmF19fiuAunVEpCSECnwzuwL4IPCjYFw67ALMrB74HvApd+8/zvTbzazb\nzLp7e0srONuaapg7tZ7HN5RW3SKSTGEC/1PAHcD33X2Nmc0BHg3z42aWoRD2/+ru9x9vHne/y927\n3L2rtbU1bN1F4+rzWlm+eR+DI4l8RICIlJBTBr67/6e7vx34opnVu/smd//Eqb5nhaOY9wDr3P3z\n41BrUbr6/CkMZ/Ms1+2SRaTIhbm1wkIzexZYDaw1sxVmFqYP/0rgQ8A1ZvZc8LrxLOstOq+bPZnK\nihSPre+JuxQRkZMK06XzL8Cn3X2Wu88E/hC4+1Rfcvcn3N3cfZG7LwleD55twcWmpjLNVXOn8PC6\n3RQuSBYRKU5hAr/O3Y/22bv7Y0BdZBWVoGsvmsa2fUdYv/tg3KWIiJxQmMDfZGZ/ZmadweszwOao\nCysl1140FYCH1+6OuRIRkRMLE/gfBVqB+4PXFAoXX0lgakM1Szqa+JkCX0SK2ElvrRDcKO1PwpyV\nk3TXzZvG5x5az+7+QaY1VMddjojIa5y0he/uOeCSCaqlpF0XPND84XVq5YtIcQrTpfOsmT1gZh8y\ns3eOviKvrMScN7WeWZNr1a0jIkUrzN0yW4C9wDVjxjmF/nwJmBnXXTSNbzy5lf7BERqqE//IABEp\nMqcMfHfXAdqQ3rZoOl9+YjM/XbObd18yI+5yREReJcyVtl83s6Yxw81m9pVoyypNSzqamNFcww9X\n7oi7FBGR1wjTh7/I3Q+MDrj7fgq3OpZjmBk3LWrjiY172Kdn3YpIkQkT+Ckzax4dMLMWwvX9J9LN\ni6eTyzs/Xr0z7lJERF4lTOD/PfArM/srM/tfwK+Az0ZbVumaN72BOa116tYRkaIT5vbI3wDeBewG\neoF3uvs3oy6sVJkZNy9qY/nmfezsOxJ3OSIiR4Vp4ePua939/7n7P5XLQ8ij9M6l7bjD/c+8Encp\nIiJHhQp8OT2zJtdx+ZwW7uveRj6vWyaLSHFQ4EfkvV0dbN17mKe26ElYIlIcFPgReeuC6UyqquC+\np7fFXYqICKDAj0xNZZq3L2njwdU76R8cibscEREFfpRuuXQmgyN5vtu9Pe5SREQU+FFaOKORS2Y1\n8/Unt5DTwVsRiZkCP2IfvXI2W/ce5ucv9MRdiogknAI/Ym+ZP422xmq+8oQeAywi8VLgR6wineLD\nr+/kyU17WbujP+5yRCTBFPgT4P2XzqSuMs0XH9sYdykikmAK/AnQWJvhI6/v5EerdrJh98G4yxGR\nhFLgT5DffsMcajNpvvBztfJFJB4K/AnSUlfJh1/fybLnd6iVLyKxiCzwzewrZtZjZqujWkap+VjQ\nyv/cQ+vjLkVEEijKFv7XgBsi/P2S01JXyX9/81x+unY3v3ppT9zliEjCRBb47v44oFtFHuO2q2bT\n3lTDXy1bp6tvRWRCqQ9/glVn0vzxWy9k3c5+7uvWnTRFZOLEHvhmdruZdZtZd29vb9zlTIibFk3n\nss4W/ubBdfT0D8ZdjogkROyB7+53uXuXu3e1trbGXc6EMDP+5l0LGczm+bP/WI27unZEJHqxB35S\nndtaz6evO5+H1uzmR6t2xl2OiCRAlKdl3gs8CVxgZtvN7LaollWqfvuq2Sya0chnfrCaVw4cibsc\nESlzUZ6l8353n+7uGXef4e73RLWsUlWRTnHnLReTzTl/8G/PMJLLx12SiJQxdenEbPaUOv7mnQt5\n5uUDfPYnL8RdjoiUsYq4CxC4eXEbT23ex92/2Mz50ybxnq6OuEsSkTKkFn6R+POb53HV3Cnccf8q\nfrVRV+GKyPhT4BeJTDrFF29dyuwpdfzOt1awantf3CWJSJlR4BeRhuoMX/voZTTWZLj1nuWsfkWh\nLyLjR4FfZNqbarj3Y5dTX1XBrfcs55mX98ddkoiUCQV+EepoqeXej11OY02GD9z9a366ZlfcJYlI\nGVDgF6mZk2v53u+9ngvOaeB3vrWCux/fpFswiMhZUeAXsSn1VXz7Y5dzw/xz+N8PruN3v7WCviMj\ncZclIiVKgV/kairTfPGDS/nM2y7ikXU93PRPv9DDU0TkjCjwS4CZ8dtvmMO//84VpM34wN3LueP+\nVfQPqrUvIuEp8EvIJbOa+fEnr+b2q+fw70+/zDV/95/c+9TLenKWiISiwC8xNZVp/uTGi/jBx6+k\nc3Itd9y/ihvv/AU/XbOLvIJfRE5CgV+iFs1o4ju/ewVf+uBSBrM5bv/mCm6483F+8OwrZHXXTRE5\nDiumU/26urq8u7s77jJKTjaXZ9nzO/nnRzeyoWeAcxqqueWyDm65dCbnNFbHXZ6IRMjMVrh7V6h5\nFfjlI593Hnmhh2/+eiuPv9hLOmW8+YJWbl7cxrUXTaOuSjdHFSk3pxP4SoAykkoZ182bxnXzpvHy\n3sP821Mv84NnX+HhdT1UZ1Jcc+FUblw4nTfMbaWxNhN3uSIywdTCL3P5vNO9dT/Lnt/Bg6t2smdg\nmHTKuLijiTee38obL2hlflsj6ZTFXaqInAF16chx5fLOc9v289j6Xh5b38uq4G6c9VUVXDyziUs7\nW+jqbObijmZqKtMxVysiYSjwJZQ9A0P8cuMent6yj+4t+1m/+yDukE4Zc1vrmd/ewIK2Rua3NTCv\nrYFJ1eoGEik2Cnw5I32HR1jx8j6e2XqANTv6WLOjn56DQ0entzfVMHdqPee21jN3an3wuY7J9VUx\nVi2SbDpoK2eksTbDNRdO45oLpx0d19M/yJod/azZ0ceGngE29gywfPNeBkd+c65/Y02GjpYaOppr\nmdFcQ0dLLR3NtXS01DCjuZbqjLqHRIqBAl9OampDNVMbqnnzhVOPjsvnnR19R9gY7AC27D3Etn1H\nWL/7II+80MNw9tUXfjXXZpjWUB28qjgn+M3R4WkN1bTUVZJJ6zpAkSgp8OW0pVLGjOZaZjTX8qYL\npr5qWj7v7BkYYtv+w2zbd4Rt+w6zq3+Q3f1D9BwcZN3OfvYMDHG8u0A0VFcwub6KlrpKmmsrmVxX\nSUt94b25tvC5pbaShpoMjTUZJlVXaCchchoU+DKuUik7+lfBJbOOP082l2fvoWF2BzuCXf2D7BsY\nZt+hIfYdHmHfoSG27z/M89sPsP/wMCO5Ex9nqq1M01CdoaGmgsaaTPA5Q0N1xdEdQ31VBXVVFdRV\npamtrKC+qoLaynQwroLaTJqUTkuVBFDgy4SrSKeOdvGcirtzcCjLvoFh9h4aZv+hYQ4OjdB3eIT+\nwSz9R0boHxyh/0iW/sERdvUP8mLPwaPDYc9JqMmkj+4U6ip/s3Ooq0pTnSm8ajJpqjMpqivS1FSm\nqcqkqa5IjZkWTD/m8+g0XesgcVPgS1Ezs0KrvTpD55S60/puPu8cGs5ycDDL4eEsh4ZyHBrKcmh4\n9D3L4aEcA0OF6QNDuWC+wrwHDg+zfX+WwZE8gyM5BkdyHBnJHbc7KoxM2qiqSFNZkSKTNiorUlSm\nU1QG46rSqVdPq0gH01NUVYyZlk4H01PHfK8wPZNOUZE2KlKF4XTqteMq0ikyqcJ7Yfpvpplpx1Su\nIg18M7sBuBNIA19297+NcnkiY6VSxqTqzLheP+DujOScwWxhBzA0kudIsDMY3TGMDh877chIjuFs\nnuFc4X0k5wxn8wxl8wzn8gxnCzuc4VyekawH44Lp2dzR4ajvgp1OGRWv2kkUdgYVozuTYEeROWba\n6PfSKSNlRkU6eE8ZqZSRPsG4dDqYdpxx6dQxr7MYl7LglYK0FXZsKePoNDv2s736O6ngN82Cz8F8\npbSDjCzwzSwN/DNwHbAdeNrMHnD3tVEtUyRqZkZlRaEF3hDThWi5fGFHMZzNMxTsPIaDnUY254zk\n8mTzwXvOyeVfOy6bL+xwssH4bL7weSSYVvid4HMwLZtzRo4zXzZfWH427+TdyeYK77l88DrBuFwu\neA/GZUv0eQ5jdw6v2WmM3dHYcXYawecpdVXc97tXRF5rlC38y4CN7r4JwMy+DbwDUOCLnIV0yqip\nTAe3vyivq5/z+VfvBI63Y3jVtJOMywc7kdHfyLuTd4L34HP++J9z7njwW3nnVZ/z7sG8Y3/rOL+R\nD37DT/0bk6onpnc9yqW0A9vGDG8HXnfsTGZ2O3A7wMyZMyMsR0SKXSplpDB0rV40ojyJ+XgdW6/5\nm83d73L3Lnfvam1tjbAcEZFkizLwtwMdY4ZnADsiXJ6IiJxElIH/NHCemc02s0rgFuCBCJcnIiIn\nEVkfvrtnzez3gYconJb5FXdfE9XyRETk5CI9NOzuDwIPRrkMEREJR3eeEhFJCAW+iEhCKPBFRBKi\nqB5xaGa9wNYz/PoUYM84llMKtM7JoHUuf2ezvrPcPdRFTEUV+GfDzLrDPtexXGidk0HrXP4man3V\npSMikhAKfBGRhCinwL8r7gJioHVOBq1z+ZuQ9S2bPnwRETm5cmrhi4jISZR84JvZDWa23sw2mtkf\nx13PeDGzDjN71MzWmdkaM/tkML7FzH5mZhuC9+ZgvJnZF4J/h+fNbGm8a3DmzCxtZs+a2bJgeLaZ\nLQ/W+d+Dm/FhZlXB8MZgemecdZ8pM2sys++a2QvB9r6i3Lezmf2P4P/1ajO718yqy207m9lXzKzH\nzFaPGXfa29XMPhLMv8HMPnI2NZV04I95jOJbgXnA+81sXrxVjZss8IfufhFwOfDxYN3+GHjE3c8D\nHgmGofBvcF7wuh340sSXPG4+CawbM/x/gX8I1nk/cFsw/jZgv7vPBf4hmK8U3Qn8xN0vBBZTWPey\n3c5m1g58Auhy9wUUbq54C+W3nb8G3HDMuNParmbWAvwFhYdHXQb8xehO4ox48CivUnwBVwAPjRm+\nA7gj7roiWtf/oPB84PXA9GDcdGB98PlfgPePmf/ofKX0ovDchEeAa4BlFB6ksweoOHabU7gT6xXB\n54pgPot7HU5zfRuAzcfWXc7bmd88Da8l2G7LgLeU43YGOoHVZ7pdgfcD/zJm/KvmO91XSbfwOf5j\nFNtjqiUywZ+wFwPLgWnuvhMgeJ8azFYu/xb/CPwRkA+GJwMH3D0bDI9dr6PrHEzvC+YvJXOAXuCr\nQTfWl82sjjLezu7+CvB3wMvATgrbbQXlvZ1Hne52HdftXeqBH+oxiqXMzOqB7wGfcvf+k816nHEl\n9W9hZjcBPe6+Yuzo48zqIaaVigpgKfAld78YOMRv/sw/npJf56BL4h3AbKANqKPQpXGsctrOp3Ki\ndRzXdS/1wC/rxyiaWYZC2P+ru98fjN5tZtOD6dOBnmB8OfxbXAm83cy2AN+m0K3zj0CTmY0+u2Hs\neh1d52B6I7BvIgseB9uB7e6+PBj+LoUdQDlv52uBze7e6+4jwP3A6ynv7TzqdLfruG7vUg/8sn2M\nopkZcA+wzt0/P2bSA8DokfqPUOjbHx3/4eBo/+VA3+ifjqXC3e9w9xnu3klhW/7c3T8IPAq8O5jt\n2HUe/bd4dzB/SbX83H0XsM3MLghG/RdgLWW8nSl05VxuZrXB//PRdS7b7TzG6W7Xh4Drzaw5+Mvo\n+mDcmYn7oMY4HBS5EXgReAn407jrGcf1uorCn27PA88Frxsp9F0+AmwI3luC+Y3CGUsvAasonAER\n+3qcxfq/CVgWfJ4DPAVsBL4DVAXjq4PhjcH0OXHXfYbrugToDrb1D4Dmct/OwF8CLwCrgW8CVeW2\nnYF7KRw4JWQ0AAAATUlEQVSjGKHQUr/tTLYr8NFg3TcCv3U2NelKWxGRhCj1Lh0REQlJgS8ikhAK\nfBGRhFDgi4gkhAJfRCQhFPgiIgmhwBcRSQgFvohIQvx/DeW4LqDiTkMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121ff50b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(log)\n",
    "plt.ylabel('cross entropy loss')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}