10-DQN-Learns-From-Environment/01-cartpole-dqn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "from gym import wrappers\n",
    "import random\n",
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import deque\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper parameters\n",
    "EPISODES = 50  # number of episodes\n",
    "EPS_START = 0.9  # e-greedy threshold start value\n",
    "EPS_END = 0.05  # e-greedy threshold end value\n",
    "EPS_DECAY = 200  # e-greedy threshold decay\n",
    "GAMMA = 0.8  # Q-learning discount factor\n",
    "LR = 0.001  # NN optimizer learning rate\n",
    "HIDDEN_LAYER = 256  # NN hidden layer size\n",
    "BATCH_SIZE = 64  # Q-learning batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "    def __init__(self):\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(4, HIDDEN_LAYER),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(HIDDEN_LAYER, 2)\n",
    "        )\n",
    "        self.memory = deque(maxlen=10000)\n",
    "        self.optimizer = optim.Adam(self.model.parameters(), LR)\n",
    "        self.steps_done = 0\n",
    "    \n",
    "    def act(self, state):\n",
    "        eps_threshold = EPS_END + (EPS_START - EPS_END) * math.exp(-1. * self.steps_done / EPS_DECAY)\n",
    "        self.steps_done += 1\n",
    "        if random.random() > eps_threshold:\n",
    "            return self.model(state).data.max(1)[1].view(1, 1)\n",
    "        else:\n",
    "            return torch.LongTensor([[random.randrange(2)]])\n",
    "\n",
    "    def memorize(self, state, action, reward, next_state):\n",
    "        self.memory.append((state,\n",
    "                            action,\n",
    "                            torch.FloatTensor([reward]),\n",
    "                            torch.FloatTensor([next_state])))\n",
    "    \n",
    "    def learn(self):\n",
    "        \"\"\"Experience Replay\"\"\"\n",
    "        if len(self.memory) < BATCH_SIZE:\n",
    "            return\n",
    "        batch = random.sample(self.memory, BATCH_SIZE)\n",
    "        states, actions, rewards, next_states = zip(*batch)\n",
    "\n",
    "        states = torch.cat(states)\n",
    "        actions = torch.cat(actions)\n",
    "        rewards = torch.cat(rewards)\n",
    "        next_states = torch.cat(next_states)\n",
    "\n",
    "        current_q = self.model(states).gather(1, actions)\n",
    "        max_next_q = self.model(next_states).detach().max(1)[0]\n",
    "        expected_q = rewards + (GAMMA * max_next_q)\n",
    "        \n",
    "        loss = F.mse_loss(current_q.squeeze(), expected_q)\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = DQNAgent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[99m Episode 1 finished after 36 steps\n",
      "\u001b[99m Episode 2 finished after 59 steps\n",
      "\u001b[99m Episode 3 finished after 27 steps\n",
      "\u001b[99m Episode 4 finished after 14 steps\n",
      "\u001b[99m Episode 5 finished after 20 steps\n",
      "\u001b[99m Episode 6 finished after 10 steps\n",
      "\u001b[99m Episode 7 finished after 26 steps\n",
      "\u001b[99m Episode 8 finished after 16 steps\n",
      "\u001b[99m Episode 9 finished after 15 steps\n",
      "\u001b[99m Episode 10 finished after 10 steps\n",
      "\u001b[99m Episode 11 finished after 12 steps\n",
      "\u001b[99m Episode 12 finished after 14 steps\n",
      "\u001b[99m Episode 13 finished after 12 steps\n",
      "\u001b[99m Episode 14 finished after 10 steps\n",
      "\u001b[99m Episode 15 finished after 9 steps\n",
      "\u001b[99m Episode 16 finished after 15 steps\n",
      "\u001b[99m Episode 17 finished after 17 steps\n",
      "\u001b[99m Episode 18 finished after 39 steps\n",
      "\u001b[99m Episode 19 finished after 13 steps\n",
      "\u001b[99m Episode 20 finished after 15 steps\n",
      "\u001b[92m Episode 21 finished after 200 steps\n",
      "\u001b[99m Episode 22 finished after 90 steps\n",
      "\u001b[99m Episode 23 finished after 82 steps\n",
      "\u001b[99m Episode 24 finished after 99 steps\n",
      "\u001b[99m Episode 25 finished after 103 steps\n",
      "\u001b[99m Episode 26 finished after 83 steps\n",
      "\u001b[99m Episode 27 finished after 93 steps\n",
      "\u001b[99m Episode 28 finished after 146 steps\n",
      "\u001b[99m Episode 29 finished after 114 steps\n",
      "\u001b[99m Episode 30 finished after 173 steps\n",
      "\u001b[99m Episode 31 finished after 117 steps\n",
      "\u001b[99m Episode 32 finished after 139 steps\n",
      "\u001b[99m Episode 33 finished after 153 steps\n",
      "\u001b[99m Episode 34 finished after 128 steps\n",
      "\u001b[99m Episode 35 finished after 124 steps\n",
      "\u001b[99m Episode 36 finished after 141 steps\n",
      "\u001b[99m Episode 37 finished after 175 steps\n",
      "\u001b[99m Episode 38 finished after 179 steps\n",
      "\u001b[99m Episode 39 finished after 130 steps\n",
      "\u001b[99m Episode 40 finished after 122 steps\n",
      "\u001b[99m Episode 41 finished after 182 steps\n",
      "\u001b[99m Episode 42 finished after 129 steps\n",
      "\u001b[99m Episode 43 finished after 144 steps\n",
      "\u001b[99m Episode 44 finished after 125 steps\n",
      "\u001b[99m Episode 45 finished after 128 steps\n",
      "\u001b[99m Episode 46 finished after 142 steps\n",
      "\u001b[99m Episode 47 finished after 153 steps\n",
      "\u001b[99m Episode 48 finished after 160 steps\n",
      "\u001b[92m Episode 49 finished after 200 steps\n",
      "\u001b[99m Episode 50 finished after 132 steps\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "episode_durations = []\n",
    "\n",
    "for e in range(1, EPISODES+1):\n",
    "    state = env.reset()\n",
    "    steps = 0\n",
    "    while True:\n",
    "        env.render()\n",
    "        state = torch.FloatTensor([state])\n",
    "        action = agent.act(state)\n",
    "        next_state, reward, done, _ = env.step(action.item())\n",
    "\n",
    "        # negative reward when attempt ends\n",
    "        if done:\n",
    "            reward = -1\n",
    "\n",
    "        agent.memorize(state, action, reward, next_state)\n",
    "        agent.learn()\n",
    "\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "\n",
    "        if done:\n",
    "            print(\"{2} Episode {0} finished after {1} steps\"\n",
    "                  .format(e, steps, '\\033[92m' if steps >= 195 else '\\033[99m'))\n",
    "            episode_durations.append(steps)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}