hyperparameter_tuning_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.704456 using {'batch_size': 15, 'epochs': 30, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.645921 (0.063867) with: {'batch_size': 15, 'epochs': 10, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.628877 (0.047837) with: {'batch_size': 15, 'epochs': 10, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.669374 (0.046195) with: {'batch_size': 15, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.670707 (0.055790) with: {'batch_size': 15, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.671946 (0.039743) with: {'batch_size': 15, 'epochs': 10, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.649826 (0.064914) with: {'batch_size': 15, 'epochs': 10, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.657584 (0.041975) with: {'batch_size': 15, 'epochs': 20, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.671938 (0.038311) with: {'batch_size': 15, 'epochs': 20, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.665402 (0.049061) with: {'batch_size': 15, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.675910 (0.061302) with: {'batch_size': 15, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.679696 (0.031480) with: {'batch_size': 15, 'epochs': 20, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.674544 (0.032038) with: {'batch_size': 15, 'epochs': 20, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.618700 (0.100603) with: {'batch_size': 15, 'epochs': 30, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.682353 (0.031154) with: {'batch_size': 15, 'epochs': 30, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.695365 (0.028962) with: {'batch_size': 15, 'epochs': 30, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.704456 (0.028992) with: {'batch_size': 15, 'epochs': 30, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.678406 (0.018816) with: {'batch_size': 15, 'epochs': 30, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.669357 (0.053710) with: {'batch_size': 15, 'epochs': 30, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.634267 (0.065792) with: {'batch_size': 20, 'epochs': 10, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.628936 (0.014608) with: {'batch_size': 20, 'epochs': 10, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.673237 (0.037176) with: {'batch_size': 20, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.658942 (0.046132) with: {'batch_size': 20, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.666769 (0.044885) with: {'batch_size': 20, 'epochs': 10, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.664120 (0.038535) with: {'batch_size': 20, 'epochs': 10, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.657550 (0.059606) with: {'batch_size': 20, 'epochs': 20, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.665436 (0.038368) with: {'batch_size': 20, 'epochs': 20, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.695357 (0.029130) with: {'batch_size': 20, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.695391 (0.031551) with: {'batch_size': 20, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.690094 (0.014243) with: {'batch_size': 20, 'epochs': 20, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.669264 (0.043832) with: {'batch_size': 20, 'epochs': 20, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.662889 (0.053700) with: {'batch_size': 20, 'epochs': 30, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.669417 (0.058237) with: {'batch_size': 20, 'epochs': 30, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.681037 (0.028856) with: {'batch_size': 20, 'epochs': 30, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.674501 (0.019381) with: {'batch_size': 20, 'epochs': 30, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.694084 (0.028067) with: {'batch_size': 20, 'epochs': 30, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.675868 (0.033997) with: {'batch_size': 20, 'epochs': 30, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.649639 (0.039432) with: {'batch_size': 30, 'epochs': 10, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.628979 (0.030248) with: {'batch_size': 30, 'epochs': 10, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.662838 (0.045837) with: {'batch_size': 30, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.666752 (0.044417) with: {'batch_size': 30, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.653739 (0.045669) with: {'batch_size': 30, 'epochs': 10, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.660224 (0.048500) with: {'batch_size': 30, 'epochs': 10, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.598990 (0.072984) with: {'batch_size': 30, 'epochs': 20, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.634148 (0.033427) with: {'batch_size': 30, 'epochs': 20, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.699279 (0.030120) with: {'batch_size': 30, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.687573 (0.033336) with: {'batch_size': 30, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.683694 (0.042642) with: {'batch_size': 30, 'epochs': 20, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.682344 (0.019489) with: {'batch_size': 30, 'epochs': 20, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.681105 (0.057940) with: {'batch_size': 30, 'epochs': 30, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.638078 (0.029104) with: {'batch_size': 30, 'epochs': 30, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.682336 (0.021642) with: {'batch_size': 30, 'epochs': 30, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.691503 (0.034244) with: {'batch_size': 30, 'epochs': 30, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.701884 (0.029810) with: {'batch_size': 30, 'epochs': 30, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.684976 (0.039660) with: {'batch_size': 30, 'epochs': 30, 'init': 'uniform', 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "# MLP for Pima Indians Dataset with grid search via sklearn\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import numpy\n",
    " \n",
    "# Function to create model, required for KerasClassifier\n",
    "def create_model(optimizer='rmsprop', init='glorot_uniform'):\n",
    "    # create mode\n",
    "    model = Sequential()\n",
    "    model.add(Dense(12, input_dim=8, kernel_initializer=init, activation='relu'))\n",
    "    model.add(Dense(8, kernel_initializer=init, activation='relu'))\n",
    "    model.add(Dense(1, kernel_initializer=init, activation='sigmoid'))\n",
    "    # Compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "    return model\n",
    " \n",
    "# fix random seed for reproducibility\n",
    "seed = 7\n",
    "numpy.random.seed(seed)\n",
    "# load pima indians dataset\n",
    "dataset = numpy.loadtxt(\"./pima-indians-diabetes.csv\", delimiter=\",\")\n",
    "# split into input (X) and output (Y) variables\n",
    "X = dataset[:,0:8]\n",
    "Y = dataset[:,8]\n",
    "# create model\n",
    "model = KerasClassifier(build_fn=create_model, verbose=0)\n",
    "# grid search epochs, batch size and optimizer\n",
    "optimizers = ['rmsprop', 'adam']\n",
    "init = ['glorot_uniform', 'normal', 'uniform']\n",
    "epochs = [10, 20, 30]\n",
    "batches = [15, 20, 30]\n",
    "param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid)\n",
    "grid_result = grid.fit(X, Y)\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}