shallow_net_in_keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Based on the BooK: Deep Learning Illustrated (2020)\n",
    "https://github.com/the-deep-learners/deep-learning-illustrated\n",
    "\n",
    "Run from within a Docker container"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shallow Neural Network in Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build a shallow neural network to classify MNIST digits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/the-deep-learners/deep-learning-illustrated/blob/master/notebooks/shallow_net_in_keras.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data and split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_valid, y_valid) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5], dtype=uint8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAE8CAYAAAD+LWvAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHpFJREFUeJzt3WeYlNUVwPE/YMFesRfAEiyxoGJPFFTsisaGqJTYeyUmBgsYiQZR7AXEFhuJyWN51FhiRbGAsSKKoqgoWFAUUEo+rGdfZgss7Mzcmdn/78s+uzuzc7k7nD3vfc89t9ns2bORJBVf89QDkKSmygAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSmShIr9epW67a5Z6ADi3heTcFlaTnV8zYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpRIscvQVEE++eQTAK688koABg4cCMDpp58OwKmnngrAmmuumWB0UukzA5akRJoV+Uiigr3YrFmzAJg+fXqd37/11lsB+OGHHwB4++23AbjiiisA+OMf/8jVV18NwGKLLQbAgAEDADj++OPn9fKlUNBetF/kp59+CsCmm24KwLffflvn45ZbbjkAJk6c2JiXa1JzO7/eeecddtllFwBGjRoFQKtWrRr69FKYWyih+b3pppsAOO6444CquDJ69GgA1l9//fn9cW7EkKRSVTZrwJMnTwZg5syZALz++usAPPbYY0CWhd14440N+nmtW7cG4MwzzwRg8ODBLLPMMgDsuOOOAHTs2DEPI68c48aNA2CnnXYC4JtvvgGgWbOqP/Qxf4suuigAX375JQBjx45l7bXXBqBFixZFG2+xjBkzBsjmo0OHDkV77ZdeeolOnToV7fUq1RNPPAHAGWecAUDz5lluGu/vQij5ADx+/HgANttsMyB7ky+omNjBgwcD2XJDr169WGmllQBYcsklgfm6lKtIP//8M5AF3t133x3Ibr7VFL+jiy++GIAddtgBgPXWW6/6D2OvXr0KN+BE4j/vu+++CxQnAMfS4ZgxY3jvvfcK/nqVLuZw2rRpRX1dlyAkKZGSz4BXWGEFAFZeeWWg4RnwbrvtlvP8f/7zn0B2eRyX0arf2WefDVB9c3Jenn76aSC70dmlSxegau5HjhxZgBGWhkGDBgHZe64YpkyZAsAll1xSXe7X1K/YFkTcjL/gggtyvt6+fXugaolziSWWKNjrmwFLUiIlnwHHGu3QoUMBGDZsGADbbrstAAceeGDO42Pd8d///jcAiyyyCAATJkwAsk0Dql+s8d5xxx1Att4YIrONue/WrRuQbbjYYIMNAOjduzdQ9TsrcrljUcWN4WKKMinI5lsN9/777wOw5557AvD111/nfL9///5AdmO5UMyAJSmRks+Aw1ZbbQXAJptsAmSZ7TnnnAPApZdeCkDfvn1zvh9WWWUVoGrNTHWLDRabb745kJX2RRnO4YcfDmTF6rF+Fp8feuihACy++OIArLbaakBV5cntt98OwB/+8AegMrYnf/bZZ0A2b8U0Z8a26667Fv31y93NN98M1K7oOeCAAwDYeeedizIOM2BJSqRsMuAQVQwhtruGuCMdmykKWURdKSZNmgTAX//6VyCrNInKkzZt2gDZluy4uoi63/g4Nz/++CMAl112GZD9nspZbAKKf1sxRIXJG2+8Uf21qPTRvNV8H8a+gJjDuIIuFjNgSUqk7DLgmk477TQARowYAcD9998PwFtvvQXAxhtvnGZgZWDGjBkAnHXWWUBW9RB3fh999FEA1l13XSDbGdcYH374YaN/Rql48803cz5vyJVAY/3pT38CsvXnTTbZpNb9DtUW9zP222+/Or8fdcDt2rUr1pAAM2BJSqbsM+D46x+9BmJffvyl23///QHYfvvtgayG1bVh+Pjjj4Es8w0vvvgiULv9XtRkq25bb7113n5WtFV99dVXgez9fc899+Q8btCgQbRs2TJvr1upnn32WQBeeOGFnK8fdNBBAHTv3r3YQwLMgCUpmbLPgMPyyy8PZOuW0bkrGq7HxyFDhgDZLq7ofNYUnXjiiUC20y2uDhag8fRczZo1q/pucyXviKuvMT1ka7ZxcED0zYg18Z9++gmAq666Csh210UfgugzEdlurMe7C27uXn75ZQCOOuqonK/vs88+QFbDnuoqwgxYkhKpmAw4RC/WqIKIAyLvu+8+AHr27AnABx98AGQdv5ZaaqmijjO1kSNH8swzzwDZenish+Vb8+bNq19jyy23LMhrpBA7/uLftu+++wLwq1/9qtZjhw8fDmRXAAstVPVfL67AYv04KlKijj0qKyITjh2EUQ9sB7S6xdXINttsU+f3o7KnkJ3OGsIMWJISqZhDOesTHe7jzn4cYBj/7t/97ndA7bvL86kUSirma26HDx9enWVFz4bo7dDYdfGoL47dbmeffXZ1dn3bbbcBtXt1zEXJz20c+Prf//53nj+oa9euQJaBxS7DeXn44YcB2HvvvYGsXjV+ZwuoFOYWChAX/vznPwNZV7OaYk2+wFcQHsopSaWq4taAa4q7m3ECRhwKGVnav/71LwBGjx5d59pdUxBzlK/M97rrrgOyTnWtW7eu3sFVibu24g57zTvt+fTggw/mfB73MpQrOtNF3/CaevToAZTO2rkZsCQlUrEZcKzxxFlwcRc6srQQfYbzXftaTo444ohGPT+yjuimdu211wJZthG1lsqf6FurXFFlEx3+QufOnYGGn29YLGbAkpRIxWTAEydOBOCaa64B4JZbbgFg/PjxdT4+1oJbt24NNL3eELNnz66uBInz9uLOcUPdddddAJx88slA1kf4lFNOAWDgwIH5GKrUYF9++SWQ9fkNcT5hqd2DMAOWpETKNgOeMmUKAA888AAAF110EQDvvffeXJ/XsWNHIKsP3GKLLQo1xJLWrFmz6qw/rhJiDnv16gVkuwNjV+ENN9wAZJ2lPvroIwDWWWcdIDsTLjJg5V9ctYwbNw6Atm3bphxOyYgdhNFro6Y4S7LUlE0Ajq2XcYheHIU+cuTIuT4vmphceOGFQHbTraktOcxNNH6JADx48GAga3A05/E3c9pjjz2ArPHRSSedVNBxKnvf1hdompqaZWex9BBHl51//vlA+i3H9XEJQpISKekMeOrUqdVHDj333HMAvPvuu3N9zp577glAnz59gKyZycILL1yoYZaljTbaqHpb9uOPP57zvViSqHnc+korrQRkh3PO70075c+TTz4JQKdOnRKPJK1Yiqz5Xo2b63HzrVSZAUtSIiWVAcdNnb/85S9AVWYWNxvqEy0B4zjpE044ASi9cpNSs/TSS1evm0WDnPpunvXr1w+Ao48+GvAY9JQquaF9U2QGLEmJlFQG/I9//API7sLPqX379gAcdthhQNbQ+phjjgHSHSlSzqL5Tlw1xEeVnjhC6/rrr088ktKy+uqrA7DXXnsBWVlquTADlqREKr4he5GUQlGxc1s4zm1hNdn5NQOWpEQMwJKUiAFYkhIxAEtSIgZgSUqk2FUQkqRfmAFLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKZGFivx6s4v8esXSLPUAcG4LybktrCY7v2bAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKZFil6FJmkPfvn0B6NOnDwAdOnQA4LHHHgNgmWWWSTMwFYUZsCQl0mz27KLWQDfZgusiKPrcTp8+HYCff/4ZgOeeew6ATz/9FICjjjoKgIUWatSFVkXO7bfffgvAeuutB8DXX38NQLNmVf/ckSNHAvDrX/863y89p1KYWyjA/E6aNAmAGTNmADBixAgA9ttvPwCaN29Y7tmjRw9uuOEGAFq0aDG/w3AjhiSVKteA1WCRtQ0YMACAJ598EoCXXnqpzsdHJhzrm8osvvjiAOy7774ADB06NOFoyt+ECRMAuO222wC48cYbAZg1axYAH3/8MZBlvnGlMS9Dhw5lueWWA6Bfv34ALLroonkadQUtQXz00UdA9kZ+5JFHAHj55ZdzHnfnnXcCsOaaawLwn//8B4Du3bvTunXrBX35UriUy/vcTpw4EYArr7wy5+PUqVOrXvCX906bNm0AWGGFFQB49dVXAVh55ZUBGDVqFK1atVrQYVTk3Ib4T33++ecDLkEsqO7duwNwxx13zP2FfnnPNjQAz2n06NEArLPOOg19iksQklSqyn4J4vnnnwfg4IMPBuCLL74Asr90BxxwAACffPIJAN26dct5fjxu4sSJXHPNNYUfcAmbNm0akGVl1113HQCTJ0+u8/GRnT399NNAdsMjMt/4XUyePLkxGXBFirmOTFeNs88++wC1M+DVVlsNgLPOOgvIliRq3oR79tlnAbj//vsLOs6azIAlKZGyy4DjL1is+e61114ATJkyBYD9998fyLK4KPOZOXMmAD179gTg7rvvzvm52223XQFHXR7iaqJ///5zfdyGG24IwDPPPAPA0ksvDcBXX31VwNFVlijde/vtt+v8/osvvgjAWmutBbghY166dOkCZOV8ITLdJZdccq7PP/bYYwHYYIMNgOymHWQxY+21187PYOccX95/oiSpQcouA37qqacA6Ny5c87XDznkEACGDBkC1C4ViU0CNTPfqHyIv6BNWX2lUOuvvz4AHTt2BODiiy8Gssw3jBs3rnCDqzBLLbUUAKeffjoAxx9/fM734/OoLIl7GapbZLo135MN9dprrwHZBo45xVVIIzcU1ckMWJISKZsMeNCgQUCWMUQdXxT59+7dG6i/SPq0006r8+v33HMPkBXGN2XXXnstANtuuy0Au+++O5BVNSyxxBJzff6XX35ZwNFVpmOOOQaonQGrOOLKOGrcf/zxx1qPOfvsswv2+mbAkpRIyWfA119/PZBlvpHhHnrooQCce+65ACy88MI5z4ua1Ndffx2AMWPGAFndb2TUW265ZcHGXm5iXfKEE05YoOfH1mTNv/rqU5VfUblz5plnAvDWW28B8NNPP9X5+B133LGgvxN/25KUSElnwNOmTatuWB1rvpH5RrVDTVEHGFURUTURot7v6KOPzv+AK9ywYcMA+O6774Da++qjB0SIGu22bdsWa4hla36bxChXNIq69957AXj44YfrfNwDDzwA1D/Pyy67LJA19dlhhx1qXV3nkxmwJCVS0hnwzJkzq/sJhIEDBwLwww8/AFlWFtUMw4cPB7IsLf7Sxcff//73ACyyyCKFHHpZi11an332GZBVmtTcZ1/fumV0mrvlllvq/L6UL59//jkAO+20EwAffPBBo35e9JTYc889G/VzGsr/GZKUSElnwC1atGCVVVYBsobLyy+/PFD/Gk7sWom1nOiCFrWs7du3L9yAy1T0yRg/fjyQZRMxd1EjHZntHnvsAcBdd90FZH04QlSgPPTQQwB07dp1QY5zkRos7kfMq7/5vKpNYu331FNPBWCzzTbL1xDrZAYsSYmUdAbcsmXL6p0q22yzDZCd0hAduY444ggAjjzySCDbrRVfjyzOnUa1ReY7atQoALbeeuuc78fOuE6dOgHZSQBxIsb//vc/oPaRRHG10qNHD6CqCiJ+diH201eC+jKzOLHFXhB1W3XVVYHs5Jv77rsPgN122w2Y972ewYMHA9mJJMVmBixJiVTMmXAhdrxFB6/IKKI+8MADDyzEy5ZC8eZ8ze3MmTOr97+fc845Od/r2rUrkB1s2LJlSyDbJ7/33nsD2UkYsTvxsssuA7KMOqogIDuxJCoqavZnXWONNeobatnN7YKINfL67m3EAadxLyNPSmFuoQjzW584maTm+/GVV14BGr0G7JlwklSqKm5BLv6i1dxZFHfum7pYa7ziiiuqO8hFD4joBxy9liPzjT6/sXsw9tPHmXDRY7ldu3YATJ8+HYCTTz4ZqNq1eOuttwLZlUiIXXLvvfdefv6BZeq8884Dsl7LNd100005j1N+RB/gVMyAJSmRisuAIytT3R588EGgqn9yrHvF/vgtttgCgNGjRwNZJ7rYARfVD1dffTWQrRXXPIUg1oQ32WQToCrbjrX3yORC7Gxs6mKuNG8zZ87kjTfeAGCjjTYCandDnJeoLjnooIPyO7j5ZAYsSYlUXBVE/GWMu5exBhy9IQp08kUp3E1u0NxGtcGECROq13gj8508eTIAb775Zp3Pve666wDo1asXULQeD2Uzt/kQV3A1T0uOtfs4eTp2hDZSKcwtNHB+o8LpggsuqO79Et0P53UWXFy9jRgxAsjqquM9HyI+xOPivsYCmuf8VtwSxNixY1MPoaTFIaQTJkyovmEZx9GHbt26AbDrrrsC2Q3M2N5tc53C6dChAwDvvPNOztedc+jevTuQu/EnlrDmFYBjmS1KJ2uW+0VAjkbtjQy8DeZvVZISqbgliGhPt9pqqwFZ5vD9998DLkFEidjw4cOrM9/YzhlN7GNpokQa6JTN3OZDHKEVy0LVA/jl/2lsxW+KSxDbb789UHvr+3y90C/zuPrqqwNZy4ILL7wQyPtWeTdiSFKpqrgMOMTNjFhLiwX8Nm3aFOLlSiGTSLads8Ca1NzGTaFoJhPHPJkBZ+1SBw0axOWXX96gHxxNu2KNOOY1NhXF1V+BmAFLUqmq2Az4iSeeALJttV26dAGyTQQV2NTEDLhwnNvCmq/5nTFjBo888giQHTE2adIkAHr27AnAvvvuC2SHC9RstlMkZsCSVKoqNgOOu/3RFDyawMTaT7RizNPhnKWQSZilFY5zW1hNdn7NgCUpkYrNgENkwv379wegb9++QN4bXJdCJtFks4gicG4Lq8nOrxmwJCVS8RlwkZRCJuHcFo5zW1hNdn7NgCUpkWJnwJKkX5gBS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpEQOwJCViAJakRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSmRhYr8erOL/HrF0iz1AHBuC8m5LawmO79mwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJVLsOmA1cQcddBAAs2dXlX4OGzYs5XCS+eKLLwB49NFHAejfvz8AHTt2BKBDhw45jz/88MMBaNGiRbGGqCIwA5akRComA545cyYAH3zwAQCnnXYaAA8//HCyMSlz8cUXA/DQQw8BcPrpp6ccTjIPPvggAF27dgXg+++/z/n+O++8A8A111yT8/XIiNu1a1foIaqIzIAlKZGKyYCnT58OZBnCGmusAcCUKVMAWHLJJdMMTAwYMKA6A15kkUUA2GuvvVIOKZlOnToB2fuxZgZcn+233x6Ap59+GoCNN964AKNTsVVMAK5p/PjxAEyePBkwAKf03HPP8dNPPwGwzz77ALDddtulHFIyiy22GAA33HADAIcddhgAP/zwAwBt27YFYOzYsTnP+/rrrwF44IEHAANwsUyePLn6vXvvvfcC0K9fv5zHxA3Sv/3tb/P9812CkKREKjYDjjInNd6YMWMA6NOnDwBDhgwBsmyuPs8++ywAL7zwAhtuuCEAAwcOLNQwy0pcCWy66aZA1RwBrLjiikDtDDgcd9xxRRhd0/X2228DcPfddwNVN0O/+eYbAJo1q7u75BNPPLHAr2cGLEmJNCtypliwF/vxxx+B2mu977//PpCtrRVIKTS2LtjcbrbZZgC88cYbAIwePRqAddddd67P22qrrQB45ZVXeOmll4DaGwwaoKLn9sUXXwTgrLPOAuD555+f6+NjA8dKK62Uj5cvhbmFhA3Ze/fuDcBrr70G1J3NLrPMMgCcfPLJAOy4444A7LzzzgAstFC9Cwk2ZJekUlWxa8Bh1KhRQMEz4Iq29NJLA9kaWNwVrs+nn34KZGvHzZs3ry4TVK5tttkGgEceeQSAXXbZBaD6iqGm8847D4Abb7yxCKOrPFOnTgXgoosuAuCyyy4DoFWrVgDstNNOAFxyySVAVdyI0snIhPPJDFiSEqmYDLh586q/JcsttxxA9Z3L2Nqp+XfVVVcBMHz4cAA233xzAFq3bl3n4yMzjuwhNsF07ty5ydb9zsszzzwDZBnviBEj5vr42MihBTNgwAAALr30UgAuvPBCIFsLjmy3WMyAJSmRismAW7ZsCWT1lbfddlvK4ZS17777DshaJC688MIA3HnnnQAsvvjidT4vsonrr78egLXWWguwIdKcJk6cCMBuu+0GwJtvvgnAjBkzGvT8eJ7m7ueffwaytfJBgwYB8Pe//x2A3XffHcgqfOZSyVBQZsCSlEjFZMBqvM8//xzI7sRHzWlktuuvv36dz4vMuOZe+Mg6lPnwww8BePfdd4GGZ74h5vT888/P78AqzNVXXw1k9dXHH388kO08TJXx1mQGLEmJlMafgQKaNGlS6iGUrFmzZgHw1FNPAdn6Ynw9KkuiBeIqq6wCwFFHHQXAtGnTABg6dCiQ9d+IZut77713QcdfjmIn4O233w7AkUceCWT1qfMSNdaauzPOOAPIatd79OgBlE7mG8yAJSmRiukFEbp37w5kVRDLLrsskPVTLZBS2FM/33MbmW3N2tJ4T2y00UZA1iEqxMGRsdPtk08+AbIMOXox50lZzm1Dvf7660BWeRLiiK0uXboA8O233wJw9NFHA3nbCVcKcwsFmN9dd90VgCeffBKAtddeG8j6Kcd7u8DsBSFJpariMuDo4xmHHpoB1+3555+v3vcedb7LL788AI8//jgASy21FJAdcHr//ffnvuAv751YZ4uPcRzUq6++mvNzF1DZzW1eXvCXub322msBOOmkkwDYYIMNgGx3YiP7E5TC3EIj5vejjz4CYM011wSgRYsWQLamfssttwBZJ7PoaxId/fLUVa4+ZsCSVKpK65ZgHrRp0ybn8+hPEGfDFaKjUTkaOHBgdT/fqC2NdbOaoqYysoro3FVTZG37778/0OjMt0mLNeDIfMOiiy4K1H86Q1MwZcqU6kNdI5O95557APjtb38LZKe1xD2hyIBjrT36lBQ4A54nM2BJSqTiMuBYAwqRlcXecFU55JBD6Ny5M5Cti9UnsoZYdwxx5ts666yT8/VYd9eCu/zyy+v8euzsmtfvrJK1a9euuiokqp0i863p5ptvzvn84IMPBmD11Vcv4AgbzgxYkhKpuCqI0L59eyA7ESNOEohO+HlWCgtyeZ/b2OkWXdH69u0LUH3CcZwRV2BlN7dTp06t7j3Qs2dPAH7zm9806LmxNhl39SPTC1HNE32vG6kU5hbmc36HDBnCKaecAmRnQda08cYbA1m3ubjfEWe+xfwWmFUQklSqKm4NOBxwwAFA1n2qT58+KYdTlqJ3ar9+/QBYddVVgXmf3NvU9e7dm1tvvRXIrsDuvfdeAFZccUUgqxCJXYRRz3ruuecCtTPfuAqJ2uymrGfPntXVIHGSyLBhw3IeE32Xu3XrBmQnYaywwgrFGmaDVOwSRASNuJnx1VdfAQUr3ymFS7m8zW2U7MWBke+//z4AV1xxBQAnnnhivl6qIcpubseOHVs9RzVL9tZbbz0Att56ayDbGhtzHuJ9Gg3D4/j6PB+ZUwpzCwmPpS8wlyAkqVRV7BJEiEu5OOwwMg/Vb4cddgCyZjunnnoqUPTMt2y1bdu2uiwqbsbtt99+QDan8bE+can82muvFWqYKgFmwJKUSMWuAceBkNGQfdy4cQC0atWqEC9XCmtpeZvbwYMHA3DssccC2U23RFcPZT23ceTQXXfdlfP1uCKLbd4hysuiTWWBy6VKYW7BNWBJUrFVbAYc65WxhhZ3owvUjKcUMokmm0UUgXNbWE12fs2AJSmRis2Ai6wUMgnntnCc28JqsvNrBixJiRiAJSkRA7AkJWIAlqREDMCSlEixqyAkSb8wA5akRAzAkpSIAViSEjEAS1IiBmBJSsQALEmJGIAlKREDsCQlYgCWpEQMwJKUiAFYkhIxAEtSIgZgSUrEACxJiRiAJSkRA7AkJWIAlqREDMCSlIgBWJISMQBLUiIGYElKxAAsSYkYgCUpkf8DkVo09UXlLuYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fada0d8a668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,5))\n",
    "for k in range(12):\n",
    "    plt.subplot(3, 4, k+1)\n",
    "    plt.imshow(X_train[k], cmap='Greys')\n",
    "    plt.axis('off')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fad9b565cc0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADVlJREFUeJzt3W+IXfWdx/HPZ2OjwRZ1zGhCGp1YpI6KTcoQg8riUgx2LcQ8iHSUkmJp+qDKFvtAzZNGQQzLtjUPlkK6iYna2hbamAiyNsiKKWhwlKGapm40zjbZxGRCirEiVDPffTAn3Wmce+7N/Xfu5Pt+Qbj3nu/58+WSz5x77+/e83NECEA+/1B1AwCqQfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyR1TjcPNnfu3BgYGOjmIYFUxsbGdOzYMTeybkvht32rpA2SZkn6j4hYX7b+wMCARkZGWjkkgBJDQ0MNr9v0y37bsyT9u6SvSrpa0rDtq5vdH4DuauU9/1JJb0fE/oj4q6RfSFrRnrYAdFor4V8g6cCUxweLZX/H9hrbI7ZHxsfHWzgcgHZqJfzTfajwqd8HR8TGiBiKiKH+/v4WDgegnVoJ/0FJC6c8/rykQ621A6BbWgn/q5KutL3I9mxJX5e0oz1tAei0pof6IuIT2/dIel6TQ32bI2JP2zoD0FEtjfNHxHOSnmtTLwC6iK/3AkkRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kFRLs/TaHpP0gaSTkj6JiKF2NAWg81oKf+GfIuJYG/YDoIt42Q8k1Wr4Q9Jvbb9me007GgLQHa2+7L8xIg7ZvkTSTtt/jIiXpq5Q/FFYI0mXXXZZi4cD0C4tnfkj4lBxe1TSNklLp1lnY0QMRcRQf39/K4cD0EZNh9/2+bY/d+q+pOWS3mxXYwA6q5WX/ZdK2mb71H5+HhH/2ZauAHRc0+GPiP2SvtTGXgB0EUN9QFKEH0iK8ANJEX4gKcIPJEX4gaTa8au+FF555ZWatQ0bNpRuu2DBgtL6nDlzSuurV68urff19TVVQ26c+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcb5G1Q21r5v376OHvuRRx4prV9wwQU1a8uWLWt3OzPGwMBAzdqDDz5Yum2GS85x5geSIvxAUoQfSIrwA0kRfiApwg8kRfiBpBjnb9AzzzxTszY6Olq67TXXXFNa37NnT2l99+7dpfXt27fXrD3//POl2y5atKi0/u6775bWW3HOOeX//ebPn19aP3DgQNPHLvsOgCTdf//9Te97puDMDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ1R3nt71Z0tckHY2Ia4tlfZJ+KWlA0pikOyLiz51rs3qDg4NN1Rpx3XXXldaHh4dL6+vXr69ZGxsbK9223jj//v37S+utmD17dmm93jh/vd7Hx8dr1q666qrSbTNo5My/RdKtpy17QNILEXGlpBeKxwBmkLrhj4iXJB0/bfEKSVuL+1sl3d7mvgB0WLPv+S+NiMOSVNxe0r6WAHRDxz/ws73G9ojtkbL3YAC6q9nwH7E9X5KK26O1VoyIjRExFBFD/f39TR4OQLs1G/4dkk5dzna1pNo/KwPQk+qG3/bTkl6W9EXbB21/S9J6SbfY3ifpluIxgBmk7jh/RNQaZP5Km3tBk84777yatVbHs1v9DkMr6l3H4NixY6X166+/vmZt+fLlTfV0NuEbfkBShB9IivADSRF+ICnCDyRF+IGkuHQ3KvPhhx+W1leuXFlan5iYKK0/9thjNWtz5swp3TYDzvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBTj/KjMli1bSuvvvfdeaf3iiy8urV9++eVn2lIqnPmBpAg/kBThB5Ii/EBShB9IivADSRF+ICnG+dFR77zzTs3afffd19K+X3755dL6vHnzWtr/2Y4zP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kVXec3/ZmSV+TdDQiri2WrZP0bUnjxWprI+K5TjWJmevZZ5+tWfv4449Lt121alVp/YorrmiqJ0xq5My/RdKt0yz/cUQsLv4RfGCGqRv+iHhJ0vEu9AKgi1p5z3+P7d/b3mz7orZ1BKArmg3/TyR9QdJiSYcl/bDWirbX2B6xPTI+Pl5rNQBd1lT4I+JIRJyMiAlJP5W0tGTdjRExFBFD/f39zfYJoM2aCr/t+VMerpT0ZnvaAdAtjQz1PS3pZklzbR+U9ANJN9teLCkkjUn6Tgd7BNABdcMfEcPTLN7UgV4wA9Ubq9+2bVvN2rnnnlu67aOPPlpanzVrVmkd5fiGH5AU4QeSIvxAUoQfSIrwA0kRfiApLt2NlmzaVD7qu2vXrpq1O++8s3RbfrLbWZz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApxvlRanR0tLR+7733ltYvvPDCmrWHH364qZ7QHpz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApxvmT++ijj0rrw8PTXbn9/508ebK0ftddd9Ws8Xv9anHmB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk6o7z214o6QlJ8yRNSNoYERts90n6paQBSWOS7oiIP3euVTRjYmKitH7bbbeV1t96663S+uDgYGn9oYceKq2jOo2c+T+R9P2IGJS0TNJ3bV8t6QFJL0TElZJeKB4DmCHqhj8iDkfE68X9DyTtlbRA0gpJW4vVtkq6vVNNAmi/M3rPb3tA0hJJuyVdGhGHpck/EJIuaXdzADqn4fDb/qykX0v6XkScOIPt1tgesT0yPj7eTI8AOqCh8Nv+jCaD/7OI+E2x+Ijt+UV9vqSj020bERsjYigihvr7+9vRM4A2qBt+25a0SdLeiPjRlNIOSauL+6slbW9/ewA6pZGf9N4o6RuS3rB96jrOayWtl/Qr29+S9CdJqzrTIlpx/Pjx0vqLL77Y0v6ffPLJ0npfX19L+0fn1A1/RPxOkmuUv9LedgB0C9/wA5Ii/EBShB9IivADSRF+ICnCDyTFpbvPAu+//37N2rJly1ra91NPPVVaX7JkSUv7R3U48wNJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUozznwUef/zxmrX9+/e3tO+bbrqptD55rRfMRJz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApxvlngH379pXW161b151GcFbhzA8kRfiBpAg/kBThB5Ii/EBShB9IivADSdUd57e9UNITkuZJmpC0MSI22F4n6duSxotV10bEc51qNLNdu3aV1k+cONH0vgcHB0vrc+bMaXrf6G2NfMnnE0nfj4jXbX9O0mu2dxa1H0fEv3WuPQCdUjf8EXFY0uHi/ge290pa0OnGAHTWGb3ntz0gaYmk3cWie2z/3vZm2xfV2GaN7RHbI+Pj49OtAqACDYff9mcl/VrS9yLihKSfSPqCpMWafGXww+m2i4iNETEUEUP9/f1taBlAOzQUftuf0WTwfxYRv5GkiDgSEScjYkLSTyUt7VybANqtbvg9eXnWTZL2RsSPpiyfP2W1lZLebH97ADqlkU/7b5T0DUlv2B4tlq2VNGx7saSQNCbpOx3pEC254YYbSus7d+4srTPUd/Zq5NP+30ma7uLsjOkDMxjf8AOSIvxAUoQfSIrwA0kRfiApwg8kxaW7Z4C77767pTowHc78QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5CUI6J7B7PHJf3PlEVzJR3rWgNnpld769W+JHprVjt7uzwiGrpeXlfD/6mD2yMRMVRZAyV6tbde7Uuit2ZV1Rsv+4GkCD+QVNXh31jx8cv0am+92pdEb82qpLdK3/MDqE7VZ34AFakk/LZvtf2W7bdtP1BFD7XYHrP9hu1R2yMV97LZ9lHbb05Z1md7p+19xe2006RV1Ns62/9bPHejtv+5ot4W2v4v23tt77H9L8XySp+7kr4qed66/rLf9ixJ/y3pFkkHJb0qaTgi/tDVRmqwPSZpKCIqHxO2/Y+S/iLpiYi4tlj2r5KOR8T64g/nRRFxf4/0tk7SX6qeubmYUGb+1JmlJd0u6Zuq8Lkr6esOVfC8VXHmXyrp7YjYHxF/lfQLSSsq6KPnRcRLko6ftniFpK3F/a2a/M/TdTV66wkRcTgiXi/ufyDp1MzSlT53JX1VoorwL5B0YMrjg+qtKb9D0m9tv2Z7TdXNTOPSYtr0U9OnX1JxP6erO3NzN502s3TPPHfNzHjdblWEf7rZf3ppyOHGiPiypK9K+m7x8haNaWjm5m6ZZmbpntDsjNftVkX4D0paOOXx5yUdqqCPaUXEoeL2qKRt6r3Zh4+cmiS1uD1acT9/00szN083s7R64LnrpRmvqwj/q5KutL3I9mxJX5e0o4I+PsX2+cUHMbJ9vqTl6r3Zh3dIWl3cXy1pe4W9/J1embm51szSqvi567UZryv5kk8xlPGYpFmSNkfEI11vYhq2r9Dk2V6avLLxz6vszfbTkm7W5K++jkj6gaRnJP1K0mWS/iRpVUR0/YO3Gr3drMmXrn+bufnUe+wu93aTpF2S3pA0USxeq8n315U9dyV9DauC541v+AFJ8Q0/ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJ/R8EiLFW9B5y7gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fada0d64f60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_valid[0], cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  84, 185, 159, 151,  60,  36,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0, 222, 254, 254, 254, 254, 241, 198,\n",
       "        198, 198, 198, 198, 198, 198, 198, 170,  52,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  67, 114,  72, 114, 163, 227, 254,\n",
       "        225, 254, 254, 254, 250, 229, 254, 254, 140,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  17,  66,\n",
       "         14,  67,  67,  67,  59,  21, 236, 254, 106,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,  83, 253, 209,  18,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,  22, 233, 255,  83,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0, 129, 254, 238,  44,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,  59, 249, 254,  62,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0, 133, 254, 187,   5,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   9, 205, 248,  58,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0, 126, 254, 182,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,  75, 251, 240,  57,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         19, 221, 254, 166,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   3,\n",
       "        203, 254, 219,  35,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  38,\n",
       "        254, 254,  77,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  31, 224,\n",
       "        254, 115,   1,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 133, 254,\n",
       "        254,  52,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  61, 242, 254,\n",
       "        254,  52,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 121, 254, 254,\n",
       "        219,  40,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 121, 254, 207,\n",
       "         18,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0]], dtype=uint8)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''you can see the seven when ploting pixel levels'''\n",
    "X_valid[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Flattening 2 dimensional images to 1D and convert to float'''\n",
    "X_train = X_train.reshape(60000, 784).astype('float32') \n",
    "X_valid = X_valid.reshape(10000, 784).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''change range to 0-1'''\n",
    "X_train /= 255\n",
    "X_valid /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.32941177,  0.72549021,  0.62352943,\n",
       "        0.59215689,  0.23529412,  0.14117648,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.87058824,  0.99607843,  0.99607843,  0.99607843,  0.99607843,\n",
       "        0.94509804,  0.7764706 ,  0.7764706 ,  0.7764706 ,  0.7764706 ,\n",
       "        0.7764706 ,  0.7764706 ,  0.7764706 ,  0.7764706 ,  0.66666669,\n",
       "        0.20392157,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.26274511,  0.44705883,\n",
       "        0.28235295,  0.44705883,  0.63921571,  0.89019608,  0.99607843,\n",
       "        0.88235295,  0.99607843,  0.99607843,  0.99607843,  0.98039216,\n",
       "        0.89803922,  0.99607843,  0.99607843,  0.54901963,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.06666667,  0.25882354,  0.05490196,  0.26274511,\n",
       "        0.26274511,  0.26274511,  0.23137255,  0.08235294,  0.9254902 ,\n",
       "        0.99607843,  0.41568628,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.32549021,  0.99215686,  0.81960785,  0.07058824,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.08627451,  0.9137255 ,\n",
       "        1.        ,  0.32549021,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.50588238,  0.99607843,  0.93333334,  0.17254902,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.23137255,  0.97647059,\n",
       "        0.99607843,  0.24313726,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.52156866,  0.99607843,  0.73333335,  0.01960784,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.03529412,  0.80392158,\n",
       "        0.97254902,  0.22745098,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.49411765,  0.99607843,  0.71372551,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.29411766,  0.98431373,\n",
       "        0.94117647,  0.22352941,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.07450981,  0.86666667,  0.99607843,  0.65098041,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.01176471,  0.79607844,  0.99607843,\n",
       "        0.85882354,  0.13725491,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.14901961,  0.99607843,  0.99607843,  0.3019608 ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.12156863,  0.87843138,  0.99607843,\n",
       "        0.4509804 ,  0.00392157,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.52156866,  0.99607843,  0.99607843,  0.20392157,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.23921569,  0.94901961,  0.99607843,\n",
       "        0.99607843,  0.20392157,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.47450981,  0.99607843,  0.99607843,  0.85882354,  0.15686275,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.47450981,  0.99607843,\n",
       "        0.81176472,  0.07058824,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 10 #10 possible digits\n",
    "y_train = keras.utils.to_categorical(y_train, n_classes) #convert to one hot format\n",
    "y_valid = keras.utils.to_categorical(y_valid, n_classes) #convert to one hot format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(64, activation='sigmoid', input_shape=(784,))) # 64 Sigmoid neurons. 28x28 pixels > 784-elements\n",
    "model.add(Dense(10, activation='softmax')) #10 classes digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 64)                50240     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 50,890\n",
      "Trainable params: 50,890\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50176"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(64*784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50240"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Parameters first layer\n",
    "(64*784)+64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "650"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Parameters second layer\n",
    "(10*64)+10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/200\n",
      "60000/60000 [==============================] - 2s 35us/step - loss: 0.0926 - acc: 0.0815 - val_loss: 0.0918 - val_acc: 0.0863\n",
      "Epoch 2/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0913 - acc: 0.0903 - val_loss: 0.0909 - val_acc: 0.1029\n",
      "Epoch 3/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0906 - acc: 0.1165 - val_loss: 0.0904 - val_acc: 0.1369\n",
      "Epoch 4/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0902 - acc: 0.1531 - val_loss: 0.0899 - val_acc: 0.1742\n",
      "Epoch 5/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0897 - acc: 0.1891 - val_loss: 0.0896 - val_acc: 0.2142\n",
      "Epoch 6/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0894 - acc: 0.2222 - val_loss: 0.0892 - val_acc: 0.2454\n",
      "Epoch 7/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0891 - acc: 0.2631 - val_loss: 0.0889 - val_acc: 0.2906\n",
      "Epoch 8/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0887 - acc: 0.3110 - val_loss: 0.0886 - val_acc: 0.3394\n",
      "Epoch 9/200\n",
      "60000/60000 [==============================] - 2s 39us/step - loss: 0.0884 - acc: 0.3579 - val_loss: 0.0883 - val_acc: 0.3853\n",
      "Epoch 10/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0881 - acc: 0.4023 - val_loss: 0.0880 - val_acc: 0.4217\n",
      "Epoch 11/200\n",
      "60000/60000 [==============================] - 3s 50us/step - loss: 0.0878 - acc: 0.4366 - val_loss: 0.0876 - val_acc: 0.4529\n",
      "Epoch 12/200\n",
      "60000/60000 [==============================] - 2s 39us/step - loss: 0.0875 - acc: 0.4627 - val_loss: 0.0873 - val_acc: 0.4740\n",
      "Epoch 13/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0872 - acc: 0.4813 - val_loss: 0.0870 - val_acc: 0.4889\n",
      "Epoch 14/200\n",
      "60000/60000 [==============================] - 2s 33us/step - loss: 0.0869 - acc: 0.4942 - val_loss: 0.0867 - val_acc: 0.4991\n",
      "Epoch 15/200\n",
      "60000/60000 [==============================] - 2s 39us/step - loss: 0.0866 - acc: 0.5050 - val_loss: 0.0864 - val_acc: 0.5072\n",
      "Epoch 16/200\n",
      "60000/60000 [==============================] - 2s 35us/step - loss: 0.0862 - acc: 0.5117 - val_loss: 0.0861 - val_acc: 0.5155\n",
      "Epoch 17/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0859 - acc: 0.5177 - val_loss: 0.0857 - val_acc: 0.5207\n",
      "Epoch 18/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0856 - acc: 0.5229 - val_loss: 0.0854 - val_acc: 0.5240\n",
      "Epoch 19/200\n",
      "60000/60000 [==============================] - 2s 35us/step - loss: 0.0853 - acc: 0.5243 - val_loss: 0.0851 - val_acc: 0.5252\n",
      "Epoch 20/200\n",
      "60000/60000 [==============================] - 2s 32us/step - loss: 0.0849 - acc: 0.5255 - val_loss: 0.0847 - val_acc: 0.5264\n",
      "Epoch 21/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0846 - acc: 0.5275 - val_loss: 0.0844 - val_acc: 0.5270\n",
      "Epoch 22/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0842 - acc: 0.5278 - val_loss: 0.0840 - val_acc: 0.5259\n",
      "Epoch 23/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0839 - acc: 0.5277 - val_loss: 0.0836 - val_acc: 0.5258\n",
      "Epoch 24/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0835 - acc: 0.5273 - val_loss: 0.0833 - val_acc: 0.5260\n",
      "Epoch 25/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0831 - acc: 0.5264 - val_loss: 0.0829 - val_acc: 0.5243\n",
      "Epoch 26/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0828 - acc: 0.5255 - val_loss: 0.0825 - val_acc: 0.5235\n",
      "Epoch 27/200\n",
      "60000/60000 [==============================] - 1s 21us/step - loss: 0.0824 - acc: 0.5256 - val_loss: 0.0821 - val_acc: 0.5228\n",
      "Epoch 28/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0820 - acc: 0.5248 - val_loss: 0.0817 - val_acc: 0.5232\n",
      "Epoch 29/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0815 - acc: 0.5223 - val_loss: 0.0813 - val_acc: 0.5221\n",
      "Epoch 30/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0811 - acc: 0.5214 - val_loss: 0.0808 - val_acc: 0.5215\n",
      "Epoch 31/200\n",
      "60000/60000 [==============================] - 1s 21us/step - loss: 0.0807 - acc: 0.5218 - val_loss: 0.0804 - val_acc: 0.5212\n",
      "Epoch 32/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0803 - acc: 0.5215 - val_loss: 0.0800 - val_acc: 0.5210\n",
      "Epoch 33/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0798 - acc: 0.5210 - val_loss: 0.0795 - val_acc: 0.5214\n",
      "Epoch 34/200\n",
      "60000/60000 [==============================] - 3s 46us/step - loss: 0.0794 - acc: 0.5212 - val_loss: 0.0790 - val_acc: 0.5212\n",
      "Epoch 35/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0789 - acc: 0.5215 - val_loss: 0.0786 - val_acc: 0.5219\n",
      "Epoch 36/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0784 - acc: 0.5220 - val_loss: 0.0781 - val_acc: 0.5223\n",
      "Epoch 37/200\n",
      "60000/60000 [==============================] - 1s 19us/step - loss: 0.0780 - acc: 0.5232 - val_loss: 0.0776 - val_acc: 0.5229\n",
      "Epoch 38/200\n",
      "60000/60000 [==============================] - 1s 20us/step - loss: 0.0775 - acc: 0.5247 - val_loss: 0.0771 - val_acc: 0.5240\n",
      "Epoch 39/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0770 - acc: 0.5260 - val_loss: 0.0766 - val_acc: 0.5253\n",
      "Epoch 40/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0765 - acc: 0.5267 - val_loss: 0.0761 - val_acc: 0.5271\n",
      "Epoch 41/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0760 - acc: 0.5281 - val_loss: 0.0756 - val_acc: 0.5292\n",
      "Epoch 42/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0755 - acc: 0.5302 - val_loss: 0.0750 - val_acc: 0.5314\n",
      "Epoch 43/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0750 - acc: 0.5323 - val_loss: 0.0745 - val_acc: 0.5342\n",
      "Epoch 44/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0744 - acc: 0.5347 - val_loss: 0.0740 - val_acc: 0.5378\n",
      "Epoch 45/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0739 - acc: 0.5371 - val_loss: 0.0735 - val_acc: 0.5406\n",
      "Epoch 46/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0734 - acc: 0.5400 - val_loss: 0.0729 - val_acc: 0.5436\n",
      "Epoch 47/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0729 - acc: 0.5427 - val_loss: 0.0724 - val_acc: 0.5476\n",
      "Epoch 48/200\n",
      "60000/60000 [==============================] - 2s 41us/step - loss: 0.0723 - acc: 0.5461 - val_loss: 0.0719 - val_acc: 0.5517\n",
      "Epoch 49/200\n",
      "60000/60000 [==============================] - 2s 37us/step - loss: 0.0718 - acc: 0.5480 - val_loss: 0.0713 - val_acc: 0.5566\n",
      "Epoch 50/200\n",
      "60000/60000 [==============================] - 2s 37us/step - loss: 0.0713 - acc: 0.5516 - val_loss: 0.0708 - val_acc: 0.5589\n",
      "Epoch 51/200\n",
      "60000/60000 [==============================] - 2s 33us/step - loss: 0.0708 - acc: 0.5542 - val_loss: 0.0703 - val_acc: 0.5629\n",
      "Epoch 52/200\n",
      "60000/60000 [==============================] - 2s 41us/step - loss: 0.0702 - acc: 0.5582 - val_loss: 0.0697 - val_acc: 0.5657\n",
      "Epoch 53/200\n",
      "60000/60000 [==============================] - 3s 44us/step - loss: 0.0697 - acc: 0.5618 - val_loss: 0.0692 - val_acc: 0.5689\n",
      "Epoch 54/200\n",
      "60000/60000 [==============================] - 2s 36us/step - loss: 0.0692 - acc: 0.5652 - val_loss: 0.0687 - val_acc: 0.5729\n",
      "Epoch 55/200\n",
      "60000/60000 [==============================] - 2s 32us/step - loss: 0.0687 - acc: 0.5694 - val_loss: 0.0681 - val_acc: 0.5768\n",
      "Epoch 56/200\n",
      "60000/60000 [==============================] - 2s 31us/step - loss: 0.0681 - acc: 0.5718 - val_loss: 0.0676 - val_acc: 0.5811\n",
      "Epoch 57/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0676 - acc: 0.5756 - val_loss: 0.0671 - val_acc: 0.5839\n",
      "Epoch 58/200\n",
      "60000/60000 [==============================] - 2s 36us/step - loss: 0.0671 - acc: 0.5796 - val_loss: 0.0666 - val_acc: 0.5875\n",
      "Epoch 59/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 2s 35us/step - loss: 0.0666 - acc: 0.5828 - val_loss: 0.0660 - val_acc: 0.5912\n",
      "Epoch 60/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0661 - acc: 0.5864 - val_loss: 0.0655 - val_acc: 0.5951\n",
      "Epoch 61/200\n",
      "60000/60000 [==============================] - 2s 32us/step - loss: 0.0656 - acc: 0.5904 - val_loss: 0.0650 - val_acc: 0.5980\n",
      "Epoch 62/200\n",
      "60000/60000 [==============================] - 2s 41us/step - loss: 0.0651 - acc: 0.5932 - val_loss: 0.0645 - val_acc: 0.6025\n",
      "Epoch 63/200\n",
      "60000/60000 [==============================] - 4s 63us/step - loss: 0.0646 - acc: 0.5968 - val_loss: 0.0640 - val_acc: 0.6061\n",
      "Epoch 64/200\n",
      "60000/60000 [==============================] - 2s 41us/step - loss: 0.0641 - acc: 0.6003 - val_loss: 0.0635 - val_acc: 0.6088\n",
      "Epoch 65/200\n",
      "60000/60000 [==============================] - 3s 51us/step - loss: 0.0636 - acc: 0.6036 - val_loss: 0.0630 - val_acc: 0.6123\n",
      "Epoch 66/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0631 - acc: 0.6073 - val_loss: 0.0625 - val_acc: 0.6161\n",
      "Epoch 67/200\n",
      "60000/60000 [==============================] - 2s 38us/step - loss: 0.0627 - acc: 0.6105 - val_loss: 0.0621 - val_acc: 0.6199\n",
      "Epoch 68/200\n",
      "60000/60000 [==============================] - 3s 42us/step - loss: 0.0622 - acc: 0.6139 - val_loss: 0.0616 - val_acc: 0.6236\n",
      "Epoch 69/200\n",
      "60000/60000 [==============================] - 2s 33us/step - loss: 0.0617 - acc: 0.6172 - val_loss: 0.0611 - val_acc: 0.6263\n",
      "Epoch 70/200\n",
      "60000/60000 [==============================] - 2s 38us/step - loss: 0.0613 - acc: 0.6197 - val_loss: 0.0606 - val_acc: 0.6299\n",
      "Epoch 71/200\n",
      "60000/60000 [==============================] - 3s 51us/step - loss: 0.0608 - acc: 0.6226 - val_loss: 0.0602 - val_acc: 0.6337\n",
      "Epoch 72/200\n",
      "60000/60000 [==============================] - 3s 45us/step - loss: 0.0603 - acc: 0.6257 - val_loss: 0.0597 - val_acc: 0.6367\n",
      "Epoch 73/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0599 - acc: 0.6287 - val_loss: 0.0593 - val_acc: 0.6401\n",
      "Epoch 74/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0595 - acc: 0.6315 - val_loss: 0.0588 - val_acc: 0.6426\n",
      "Epoch 75/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0590 - acc: 0.6345 - val_loss: 0.0584 - val_acc: 0.6449\n",
      "Epoch 76/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0586 - acc: 0.6372 - val_loss: 0.0579 - val_acc: 0.6479\n",
      "Epoch 77/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0582 - acc: 0.6393 - val_loss: 0.0575 - val_acc: 0.6497\n",
      "Epoch 78/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0577 - acc: 0.6424 - val_loss: 0.0571 - val_acc: 0.6519\n",
      "Epoch 79/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0573 - acc: 0.6441 - val_loss: 0.0567 - val_acc: 0.6540\n",
      "Epoch 80/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0569 - acc: 0.6472 - val_loss: 0.0562 - val_acc: 0.6570\n",
      "Epoch 81/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0565 - acc: 0.6494 - val_loss: 0.0558 - val_acc: 0.6593\n",
      "Epoch 82/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0561 - acc: 0.6522 - val_loss: 0.0554 - val_acc: 0.6627\n",
      "Epoch 83/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0557 - acc: 0.6553 - val_loss: 0.0550 - val_acc: 0.6654\n",
      "Epoch 84/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0553 - acc: 0.6579 - val_loss: 0.0546 - val_acc: 0.6680\n",
      "Epoch 85/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0549 - acc: 0.6605 - val_loss: 0.0542 - val_acc: 0.6707\n",
      "Epoch 86/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0545 - acc: 0.6640 - val_loss: 0.0538 - val_acc: 0.6742\n",
      "Epoch 87/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0541 - acc: 0.6677 - val_loss: 0.0534 - val_acc: 0.6777\n",
      "Epoch 88/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0537 - acc: 0.6713 - val_loss: 0.0530 - val_acc: 0.6803\n",
      "Epoch 89/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0533 - acc: 0.6744 - val_loss: 0.0527 - val_acc: 0.6844\n",
      "Epoch 90/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0530 - acc: 0.6781 - val_loss: 0.0523 - val_acc: 0.6890\n",
      "Epoch 91/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0526 - acc: 0.6818 - val_loss: 0.0519 - val_acc: 0.6927\n",
      "Epoch 92/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0522 - acc: 0.6861 - val_loss: 0.0515 - val_acc: 0.6971\n",
      "Epoch 93/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0519 - acc: 0.6904 - val_loss: 0.0512 - val_acc: 0.7015\n",
      "Epoch 94/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0515 - acc: 0.6953 - val_loss: 0.0508 - val_acc: 0.7061\n",
      "Epoch 95/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0512 - acc: 0.6995 - val_loss: 0.0505 - val_acc: 0.7101\n",
      "Epoch 96/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0508 - acc: 0.7042 - val_loss: 0.0501 - val_acc: 0.7140\n",
      "Epoch 97/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0505 - acc: 0.7083 - val_loss: 0.0497 - val_acc: 0.7181\n",
      "Epoch 98/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0501 - acc: 0.7128 - val_loss: 0.0494 - val_acc: 0.7209\n",
      "Epoch 99/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0498 - acc: 0.7163 - val_loss: 0.0491 - val_acc: 0.7246\n",
      "Epoch 100/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0494 - acc: 0.7204 - val_loss: 0.0487 - val_acc: 0.7280\n",
      "Epoch 101/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0491 - acc: 0.7241 - val_loss: 0.0484 - val_acc: 0.7313\n",
      "Epoch 102/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0488 - acc: 0.7269 - val_loss: 0.0480 - val_acc: 0.7355\n",
      "Epoch 103/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0484 - acc: 0.7305 - val_loss: 0.0477 - val_acc: 0.7391\n",
      "Epoch 104/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0481 - acc: 0.7334 - val_loss: 0.0474 - val_acc: 0.7431\n",
      "Epoch 105/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0478 - acc: 0.7365 - val_loss: 0.0471 - val_acc: 0.7469\n",
      "Epoch 106/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0475 - acc: 0.7396 - val_loss: 0.0467 - val_acc: 0.7501\n",
      "Epoch 107/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0472 - acc: 0.7420 - val_loss: 0.0464 - val_acc: 0.7532\n",
      "Epoch 108/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0469 - acc: 0.7444 - val_loss: 0.0461 - val_acc: 0.7555\n",
      "Epoch 109/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0466 - acc: 0.7467 - val_loss: 0.0458 - val_acc: 0.7575\n",
      "Epoch 110/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0463 - acc: 0.7490 - val_loss: 0.0455 - val_acc: 0.7595\n",
      "Epoch 111/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0460 - acc: 0.7511 - val_loss: 0.0452 - val_acc: 0.7614\n",
      "Epoch 112/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0457 - acc: 0.7535 - val_loss: 0.0449 - val_acc: 0.7642\n",
      "Epoch 113/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0454 - acc: 0.7552 - val_loss: 0.0446 - val_acc: 0.7648\n",
      "Epoch 114/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0451 - acc: 0.7571 - val_loss: 0.0443 - val_acc: 0.7665\n",
      "Epoch 115/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0448 - acc: 0.7586 - val_loss: 0.0440 - val_acc: 0.7678\n",
      "Epoch 116/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0445 - acc: 0.7599 - val_loss: 0.0437 - val_acc: 0.7689\n",
      "Epoch 117/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0442 - acc: 0.7618 - val_loss: 0.0434 - val_acc: 0.7712\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0439 - acc: 0.7631 - val_loss: 0.0432 - val_acc: 0.7737\n",
      "Epoch 119/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0437 - acc: 0.7643 - val_loss: 0.0429 - val_acc: 0.7747\n",
      "Epoch 120/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0434 - acc: 0.7660 - val_loss: 0.0426 - val_acc: 0.7769\n",
      "Epoch 121/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0431 - acc: 0.7674 - val_loss: 0.0423 - val_acc: 0.7783\n",
      "Epoch 122/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0428 - acc: 0.7690 - val_loss: 0.0421 - val_acc: 0.7792\n",
      "Epoch 123/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0426 - acc: 0.7700 - val_loss: 0.0418 - val_acc: 0.7808\n",
      "Epoch 124/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0423 - acc: 0.7712 - val_loss: 0.0415 - val_acc: 0.7822\n",
      "Epoch 125/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0421 - acc: 0.7728 - val_loss: 0.0413 - val_acc: 0.7842\n",
      "Epoch 126/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0418 - acc: 0.7743 - val_loss: 0.0410 - val_acc: 0.7859\n",
      "Epoch 127/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0416 - acc: 0.7759 - val_loss: 0.0408 - val_acc: 0.7874\n",
      "Epoch 128/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0413 - acc: 0.7774 - val_loss: 0.0405 - val_acc: 0.7899\n",
      "Epoch 129/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0410 - acc: 0.7794 - val_loss: 0.0403 - val_acc: 0.7915\n",
      "Epoch 130/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0408 - acc: 0.7810 - val_loss: 0.0400 - val_acc: 0.7927\n",
      "Epoch 131/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0406 - acc: 0.7828 - val_loss: 0.0398 - val_acc: 0.7946\n",
      "Epoch 132/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0403 - acc: 0.7844 - val_loss: 0.0395 - val_acc: 0.7960\n",
      "Epoch 133/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0401 - acc: 0.7862 - val_loss: 0.0393 - val_acc: 0.7973\n",
      "Epoch 134/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0398 - acc: 0.7881 - val_loss: 0.0390 - val_acc: 0.7991\n",
      "Epoch 135/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0396 - acc: 0.7898 - val_loss: 0.0388 - val_acc: 0.8007\n",
      "Epoch 136/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0394 - acc: 0.7916 - val_loss: 0.0386 - val_acc: 0.8022\n",
      "Epoch 137/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0391 - acc: 0.7938 - val_loss: 0.0383 - val_acc: 0.8034\n",
      "Epoch 138/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0389 - acc: 0.7958 - val_loss: 0.0381 - val_acc: 0.8046\n",
      "Epoch 139/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0387 - acc: 0.7976 - val_loss: 0.0379 - val_acc: 0.8066\n",
      "Epoch 140/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0385 - acc: 0.7993 - val_loss: 0.0376 - val_acc: 0.8084\n",
      "Epoch 141/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0382 - acc: 0.8012 - val_loss: 0.0374 - val_acc: 0.8098\n",
      "Epoch 142/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0380 - acc: 0.8029 - val_loss: 0.0372 - val_acc: 0.8111\n",
      "Epoch 143/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0378 - acc: 0.8042 - val_loss: 0.0370 - val_acc: 0.8127\n",
      "Epoch 144/200\n",
      "60000/60000 [==============================] - 2s 33us/step - loss: 0.0376 - acc: 0.8061 - val_loss: 0.0368 - val_acc: 0.8141\n",
      "Epoch 145/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0374 - acc: 0.8078 - val_loss: 0.0365 - val_acc: 0.8158\n",
      "Epoch 146/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0372 - acc: 0.8096 - val_loss: 0.0363 - val_acc: 0.8173\n",
      "Epoch 147/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0370 - acc: 0.8111 - val_loss: 0.0361 - val_acc: 0.8195\n",
      "Epoch 148/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0368 - acc: 0.8128 - val_loss: 0.0359 - val_acc: 0.8218\n",
      "Epoch 149/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0366 - acc: 0.8142 - val_loss: 0.0357 - val_acc: 0.8235\n",
      "Epoch 150/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0363 - acc: 0.8155 - val_loss: 0.0355 - val_acc: 0.8257\n",
      "Epoch 151/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0361 - acc: 0.8171 - val_loss: 0.0353 - val_acc: 0.8268\n",
      "Epoch 152/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0360 - acc: 0.8183 - val_loss: 0.0351 - val_acc: 0.8281\n",
      "Epoch 153/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0358 - acc: 0.8198 - val_loss: 0.0349 - val_acc: 0.8295\n",
      "Epoch 154/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0356 - acc: 0.8214 - val_loss: 0.0347 - val_acc: 0.8304\n",
      "Epoch 155/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0354 - acc: 0.8225 - val_loss: 0.0345 - val_acc: 0.8319\n",
      "Epoch 156/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0352 - acc: 0.8237 - val_loss: 0.0343 - val_acc: 0.8332\n",
      "Epoch 157/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0350 - acc: 0.8248 - val_loss: 0.0341 - val_acc: 0.8345\n",
      "Epoch 158/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0348 - acc: 0.8259 - val_loss: 0.0340 - val_acc: 0.8355\n",
      "Epoch 159/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0346 - acc: 0.8272 - val_loss: 0.0338 - val_acc: 0.8359\n",
      "Epoch 160/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0344 - acc: 0.8281 - val_loss: 0.0336 - val_acc: 0.8369\n",
      "Epoch 161/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0343 - acc: 0.8293 - val_loss: 0.0334 - val_acc: 0.8378\n",
      "Epoch 162/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0341 - acc: 0.8307 - val_loss: 0.0332 - val_acc: 0.8390\n",
      "Epoch 163/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0339 - acc: 0.8314 - val_loss: 0.0331 - val_acc: 0.8401\n",
      "Epoch 164/200\n",
      "60000/60000 [==============================] - 2s 29us/step - loss: 0.0337 - acc: 0.8322 - val_loss: 0.0329 - val_acc: 0.8399\n",
      "Epoch 165/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0336 - acc: 0.8333 - val_loss: 0.0327 - val_acc: 0.8414\n",
      "Epoch 166/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0334 - acc: 0.8341 - val_loss: 0.0325 - val_acc: 0.8427\n",
      "Epoch 167/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0332 - acc: 0.8352 - val_loss: 0.0324 - val_acc: 0.8437\n",
      "Epoch 168/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0331 - acc: 0.8360 - val_loss: 0.0322 - val_acc: 0.8439\n",
      "Epoch 169/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0329 - acc: 0.8370 - val_loss: 0.0320 - val_acc: 0.8451\n",
      "Epoch 170/200\n",
      "60000/60000 [==============================] - 2s 28us/step - loss: 0.0327 - acc: 0.8378 - val_loss: 0.0319 - val_acc: 0.8464\n",
      "Epoch 171/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0326 - acc: 0.8388 - val_loss: 0.0317 - val_acc: 0.8472\n",
      "Epoch 172/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0324 - acc: 0.8395 - val_loss: 0.0316 - val_acc: 0.8476\n",
      "Epoch 173/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0323 - acc: 0.8403 - val_loss: 0.0314 - val_acc: 0.8486\n",
      "Epoch 174/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0321 - acc: 0.8409 - val_loss: 0.0312 - val_acc: 0.8487\n",
      "Epoch 175/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0320 - acc: 0.8413 - val_loss: 0.0311 - val_acc: 0.8492\n",
      "Epoch 176/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0318 - acc: 0.8423 - val_loss: 0.0309 - val_acc: 0.8501\n",
      "Epoch 177/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0317 - acc: 0.8428 - val_loss: 0.0308 - val_acc: 0.8506\n",
      "Epoch 178/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0315 - acc: 0.8436 - val_loss: 0.0306 - val_acc: 0.8512\n",
      "Epoch 179/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0314 - acc: 0.8442 - val_loss: 0.0305 - val_acc: 0.8520\n",
      "Epoch 180/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0312 - acc: 0.8449 - val_loss: 0.0303 - val_acc: 0.8522\n",
      "Epoch 181/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0311 - acc: 0.8457 - val_loss: 0.0302 - val_acc: 0.8528\n",
      "Epoch 182/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0310 - acc: 0.8462 - val_loss: 0.0301 - val_acc: 0.8536\n",
      "Epoch 183/200\n",
      "60000/60000 [==============================] - 1s 22us/step - loss: 0.0308 - acc: 0.8471 - val_loss: 0.0299 - val_acc: 0.8539\n",
      "Epoch 184/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0307 - acc: 0.8475 - val_loss: 0.0298 - val_acc: 0.8544\n",
      "Epoch 185/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0305 - acc: 0.8480 - val_loss: 0.0296 - val_acc: 0.8556\n",
      "Epoch 186/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0304 - acc: 0.8483 - val_loss: 0.0295 - val_acc: 0.8559\n",
      "Epoch 187/200\n",
      "60000/60000 [==============================] - 2s 27us/step - loss: 0.0303 - acc: 0.8491 - val_loss: 0.0294 - val_acc: 0.8562\n",
      "Epoch 188/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0301 - acc: 0.8497 - val_loss: 0.0292 - val_acc: 0.8568\n",
      "Epoch 189/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0300 - acc: 0.8504 - val_loss: 0.0291 - val_acc: 0.8570\n",
      "Epoch 190/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0299 - acc: 0.8511 - val_loss: 0.0290 - val_acc: 0.8571\n",
      "Epoch 191/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0298 - acc: 0.8516 - val_loss: 0.0289 - val_acc: 0.8576\n",
      "Epoch 192/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0296 - acc: 0.8519 - val_loss: 0.0287 - val_acc: 0.8584\n",
      "Epoch 193/200\n",
      "60000/60000 [==============================] - 1s 25us/step - loss: 0.0295 - acc: 0.8521 - val_loss: 0.0286 - val_acc: 0.8589\n",
      "Epoch 194/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0294 - acc: 0.8526 - val_loss: 0.0285 - val_acc: 0.8596\n",
      "Epoch 195/200\n",
      "60000/60000 [==============================] - 2s 25us/step - loss: 0.0293 - acc: 0.8531 - val_loss: 0.0284 - val_acc: 0.8600\n",
      "Epoch 196/200\n",
      "60000/60000 [==============================] - 2s 26us/step - loss: 0.0291 - acc: 0.8537 - val_loss: 0.0282 - val_acc: 0.8602\n",
      "Epoch 197/200\n",
      "60000/60000 [==============================] - 2s 30us/step - loss: 0.0290 - acc: 0.8543 - val_loss: 0.0281 - val_acc: 0.8607\n",
      "Epoch 198/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0289 - acc: 0.8546 - val_loss: 0.0280 - val_acc: 0.8619\n",
      "Epoch 199/200\n",
      "60000/60000 [==============================] - 1s 24us/step - loss: 0.0288 - acc: 0.8553 - val_loss: 0.0279 - val_acc: 0.8624\n",
      "Epoch 200/200\n",
      "60000/60000 [==============================] - 1s 23us/step - loss: 0.0287 - acc: 0.8557 - val_loss: 0.0278 - val_acc: 0.8631\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fad9ec9c5f8>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, batch_size=128, epochs=200, verbose=1, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 38us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.027774476686120032, 0.86309999999999998]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''Validate'''\n",
    "model.evaluate(X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}