rl_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "94e6157f-d831-41f7-b27c-f949e2253f3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "SB3 version: 2.2.1\n",
      "Using device: cuda\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['OPENBLAS_NUM_THREADS'] = '1'       #Disactivate multiprocessing for numpy\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import gymnasium as gym\n",
    "import yaml\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm\n",
    "\n",
    "import stable_baselines3\n",
    "\n",
    "from stable_baselines3.common.env_checker import check_env\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "from stable_baselines3.common.vec_env import VecNormalize, SubprocVecEnv,VecCheckNan\n",
    "from stable_baselines3 import PPO, A2C, SAC, TD3\n",
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise\n",
    "from stable_baselines3.common.callbacks import EvalCallback, CallbackList, CheckpointCallback, StopTrainingOnNoModelImprovement\n",
    "\n",
    "from sogym.mmc_optim import run_mmc\n",
    "from sogym.env import sogym\n",
    "from sogym.expert_generation import generate_expert_dataset, generate_mmc_solutions, generate_dataset\n",
    "from sogym.utils import profile_and_analyze,ImageDictExtractor, CustomBoxDense\n",
    "from sogym.callbacks import FigureRecorderCallback, MaxRewardCallback, GradientNormCallback, GradientClippingCallback\n",
    "from sogym.pretraining import pretrain_agent, ExpertDataSet\n",
    "\n",
    "import torch\n",
    "import torch as th\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "from torch.utils.data import random_split, Dataset\n",
    "from IPython.display import display\n",
    "import multiprocessing\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "print('SB3 version:', stable_baselines3.__version__)\n",
    "# Let's make the code device agnostic:\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print('Using device:', device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d02b8cad-4629-49cf-a15e-ebb7c9a5b6c1",
   "metadata": {},
   "source": [
    "---\n",
    "### Environment test and visualization:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "85a4374a-b735-4474-8427-30b1d55dfe22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 950x650 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 950x650 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Let's test the environment using the check_env util from SB3:\n",
    "observation_type = 'topopt_game'\n",
    "train_env = sogym(mode='train',observation_type=observation_type,vol_constraint_type='hard',resolution=50,check_connectivity = True)\n",
    "eval_env = sogym(mode='test',observation_type=observation_type,vol_constraint_type='hard',resolution=50,check_connectivity=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "27032137",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwEAAAOXCAYAAAB8IE1LAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmgUlEQVR4nO3de7xVdZ34//cB5CAZeEFQHBLtol0USJNBx6+HwgwdJr/zqywtzW7TxaZippKZSSJn8pLZlcZvjUZNeclGrUa/mpEnJ2XGvPAtFS3zboK3BAEF5azfH8TRA+ey9z577/VZaz2fj8d5FMd1zl5sDvp57fdan92RZVkWAABAZYzI+wQAAID2EgEAAFAxIgAAACpGBAAAQMWIAAAAqBgRAAAAFSMCAACgYkQAAABUjAgAAICKEQEAAFAxdUfAtddeG/PmzYvJkydHR0dHXHbZZUN+zfe///2YNm1ajB07Nnbfffd4z3veE48//ngj5wsAAAxT3RGwbt26mDZtWixevLim46+77ro4/vjj473vfW/cdtttcfHFF8cNN9wQ73//++s+WQAAYPhG1fsFc+fOjblz59Z8/LJly2Lq1Knxt3/7txERsddee8Xf/M3fxBlnnFHvQwMAAE1QdwTUa9asWfEP//APccUVV8TcuXPjkUceiR/+8Idx5JFHDvg1GzZsiA0bNvT+uqenJ5544onYZZddoqOjo9WnXDgPPPCAy6sAgFLbZZddYsqUKXmfRvKyLIunnnoqJk+eHCNGDHLRTzYMEZFdeumlQx73gx/8INthhx2yUaNGZRGRzZs3L9u4ceOAxy9cuDCLCB8+fPjw4cOHDx8+fDTw8cADDwy6Pu/402K+IR0dHXHppZfG0UcfPeAxt99+e8yZMyc+8YlPxBFHHBEPP/xwfPKTn4zXve51ce655/b7NVtPAlavXh0veclL4oEHHohx48Y1erqltHz58jjssMPim9/8Zuyzzz55nw4AQNPdeeed8YEPfCB+8YtfxPTp0/M+naStWbMmpkyZEk8++WSMHz9+wONafjnQaaedFoccckh88pOfjIiI/fffP170ohfFoYceGv/8z/8cu++++zZf09nZGZ2dndt8fty4cSJgKzvssENERBxwwAHx2te+NuezAQBovi3rnR122MFasEZDXULf8vcJWL9+/TbXI40cOTIiIoYxhAAAABpUdwSsXbs2li9fHsuXL4+IiHvuuSeWL18e999/f0RELFiwII4//vje4+fNmxeXXHJJ/Ou//mvcfffdcd1118Xf/u3fxkEHHRSTJ09uzu8CAACoWd2XA914440xe/bs3l/Pnz8/IiJOOOGEWLJkSTz88MO9QRAR8e53vzueeuqp+PrXvx5/93d/FzvuuGO8/vWvt0UoAADkpO4I6OrqGvQyniVLlmzzuY9+9KPx0Y9+tN6HAgAAWqDl9wQAAABpEQEAAFAxIgAAACpGBAAAQMW0/M3CAGBrHYv6volNttD7xgC0kwgAoGW2XuzXcpwgAGg9EQBAU9S64K/1+4gBgNYRAQA0rFkL/8G+txgAaD4RAEBdWrnwB6A9RAAAQ8pz4d+xqMM0AKDJbBEKwKC88g9QPiIAAAAqRgQAAEDFiAAABpTKpUCpnAdAWYgAAACoGLsDAQCV1N3R3efXXVlXLucBeRABABSCrUJppq0DYKDPRYgDysnlQAD0y3X4lNVAi/1mHQ9FIAIAgMpodEEvBCgbEQAAVMJwF/JCgDIRAQAUhkuUaFSzFvBCgLIQAQBAqTV74S4EKAMRAACUVqsW7N0d3WKAQhMBAGwj5ctuUj430mKRDgMTAQBA6bQrAIQGRSUCAACGQQhQRCIAgMJxSRCDyWNRLgQoGhEAAJSGxTjURgQAADSBAKFIRAAAheSSILaWwiI8hXOAWogAAIAmEgIUgQgAAArPwhvqIwIAKCyXBBEhAKARIgAAKCwBAI0RAQBAIaUaAF1ZV96nAEMSAQAAUDEiAAAonFSnAFAUIgAAACpGBABQaHYIqh5TABg+EQAAABUjAgCAwjAFgOYQAQBAIQgAaB4RAADQJN4jgKIQAQBA8kwBoLlEAACQNAEAzScCACg824SWV5ECwKVAFIkIAACAihEBAECSijQFgKIRAQAAw+RSIIpGBAAAyTEFgNYSAQAAUDEiAIBtZAuzvE+BCivaFMClQBSRCACgFGwTWg5FCwAoKhEAAAAVIwIAgCQUcQrgUiCKSgQAALkrYgBAkYkAAACoGBEAAOSqqFMAlwJRZCIAAAAqRgQAUBq2CS0eUwDIhwgAAICKEQEAQC5MASA/IgAAaLuiBgCUhQgAAKiRKQBlIQIA6Fe2MMv7FCgpUwDInwgAAICKEQFQeh1/+oBqsE1o2oo8BXApEGUyKu8TAFqpY4D//0Iu+QDao8gBAGVjEgClVeuroSYFAEMxBaBsRACUUiOLeiEAtI4pAKRFBEDpDGcxLwQAtmYKQBmJAGArQgBoLlMASI8IgFJp1gJeCABAmYkAKI1mL9yFAMVlm9B0FH0K4FIgykoEQCm0asFjIQU0rugBAGUmAoAhCIEqyxZ6HwmqyxSAMhMBQA2EAFCfok8BBABlJwKg8Nq1QBcCAFAWIgCogxAAhmYKAOkTAUCdhAAAFJ0IABogBEifbULzYQoAxSACgAZZYAFAUYkAKLw8t3AUAsDzij4FgCoRAcAwCQGgHAHgUiCqRAQAMChvGAZQPiIASiHvRZppAFSZKQAUT90RcO2118a8efNi8uTJ0dHREZdddtmQX7Nhw4b4x3/8x9hzzz2js7Mzpk6dGuedd14j5wskSwgAQFGMqvcL1q1bF9OmTYv3vOc98dd//dc1fc3b3va2WLVqVZx77rnxspe9LB5++OHo6emp+2SB1HVE/lMJeF7Hog6XM7WYKQAUU90RMHfu3Jg7d27Nx1955ZXxi1/8Iu6+++7YeeedIyJi6tSpg37Nhg0bYsOGDb2/XrNmTb2nCRWURRqvxgsBAEhdy+8J+PGPfxwHHnhgnHnmmbHHHnvEK17xivj7v//7ePrppwf8mtNOOy3Gjx/f+zFlypRWnybQVCnECNBqpgBQXHVPAup19913xy9/+csYM2ZMXHrppfHYY4/Fhz/84Xj88cfj29/+dr9fs2DBgpg/f37vr9esWSMEoHBMBAAgVS2PgJ6enujo6Ijvf//7MX78+IiIOPvss+Mtb3lLfOMb34jtt99+m6/p7OyMzs7OVp8alFAqlwRtIQSgrEwBoNhafjnQ7rvvHnvssUdvAEREvPKVr4wsy+LBBx9s9cMD0ARuruWFyhAAUHUtj4BDDjkk/vCHP8TatWt7P/fb3/42RowYEX/2Z3/W6oeHCkptsZbSZAJgM1MAqq7uCFi7dm0sX748li9fHhER99xzTyxfvjzuv//+iNh8Pf/xxx/fe/yxxx4bu+yyS5x44olx++23x7XXXhuf/OQn4z3veU+/lwIBZSQEoCxMAaAc6o6AG2+8MWbMmBEzZsyIiIj58+fHjBkz4pRTTomIiIcffrg3CCIidthhh7j66qvjySefjAMPPDCOO+64mDdvXnz1q19t0m8B2FZq04AIIQCkwhQAGrgxuKurK7Js4AXGkiVLtvncvvvuG1dffXW9DwWUjhuFaT9vGNY8pgBQHi2/JwDIS6qLHhMBKKKyBIApAGwmAgAAoGJEAJSaaQDN45Ka6jIFgPIRAUBOhAAA5EUEQOml/OqtEIDUmQJAOYkAqAQhAAA8TwQACRACkCJTACgvEQCVkfI0AABoJxEAlZJyCJgG0Dodi/x81csUAMpNBAAJsVADmkcAwMBEAFROytOACCGQNu8VUA1lmQIAAxMBUEmpL+SEADA8pgAwOBEAAPQyBYBqEAFQWaYBQF9lCQBTABiaCIBKEwIAUEWj8j4BgMF1RPqxMjwd3d01H5t1dbXsPKg2UwCoFhEAlZdF+q+4ly8E6ln4A0CzuRwIoI06uruHFQDioXHeMGxgpgBQPSIAiGK8yl78BVxZFvDeKwCg+EQA8CdFWNgVPwQgNaYAUE0iACiYYoZAWaYAAJSDCABeoAjTgIiihkCzCAqaxRQAqksEAFspSggAAI0SAUBBmQbAcJgCQLWJAKAfRZkGVDsEoFFlCQCgcSIAGIAQYGC2CSUFpgDQOBEAlEA1Q8AlQfXzhmGmAMBmIgAYhFd7m8FiHZrPFACGRwQAJVHNV3gFBvUwBQC2EAHAEIo0DahmCEDVmALA8IkAoAZCAIquLFMAAQDNIQKAEqpWCLgkCIB6iQCgRkWaBlSPEGAwpgDA1kQAUIcihUC1pgF58F4BAMUlAoASq1YImAbQH1MAaJ477rgjzj///Ni0aVPepzJsIgCoU9Fe/a1WCMALlSUAIAWbNm2KI/9yXhx33HHxhjmHx8MPP5z3KQ2LCAAaULQQgOd51+DiMQUgBT/+8Y/jnt/fFTvNfk8su/nX8Zr99o//+3//b96n1TARAFSARR/VYwoAzXXGmV+IsS95TYw76K9j4glfjWd23CuOPPLImD9/fmzcuDHv06ubCAAaVLRpgBCAIjIFIAXXX399/M9/L4sXve5/R0TEyLHjY8L/95nY6fXvi6989Wvx57MOjrvuuivns6yPCACGQQhAikwBoLnOOPPMGLPrlNj+pa/r/VxHx4gY97qjY+I7z4oV962MadOnx/nnn5/jWdZHBAAVU+4QaPcOQbYJpZVMAUjBb3/72/jJj38cYw84Ojo6tl06d+72stj1XV+Kjqmvi+OOOy7e/e4TY/369TmcaX1EADBMFoGQElMAaK4vfvGLsd0OO8YOr5494DEjOsfGLkf9XWz/soPixz/5SXR0pP+CkwgAmqBoIZD+v5yh6kwBSMEjjzwSS77z3Rg7/S+jY9ToQY/teeap2Hj/r+Njf/vR2H777dt0ho0TAUBFCQHKxxQAmmvx4sWxKYvYYcaRQx771C1XxMiOiA9/+MNtOLPhEwFAkxRtGhAhBCBNpgCkYP369fHVr309xu53eIzc/sWDHps9tzGevuXyeM+J745dd921TWc4PCIAgMop4xuGlWUKIABIxZIlS2L1k0/Giw9885DHrr315/Hsuidj/vz5bTiz5hABQBOZBqSg3TsEAZTNpk2b4syzvhhj9z0ktttxt0GPzbKeWH/Tj+LNRx8dL3/5y9t0hsMnAoAmEwJVY5vQ/JkCQHNddtllcd89d8eLX/fXQx779F2/imceeyA+/alPteHMmkcEAESEEKCoyhIAkIosy+L0M86MsXvuH527D/3K/rpfXRJ/Puvg+PM///M2nF3ziACgBYr6yrAQgLyYApCK6667Lm781Q3xotf97yGP3fDQHbH+gdvi5E8XawoQIQKAlilqCEBxmAJA851x5pkxZuKesf3eBwx57FO/ujT2ftnLY968eW04s+YSAQB9mAZAu5kCkIo777wz/vMnP4kXHfDm6OgYfJn87B//EOt+e318+pN/HyNGFG9JXbwzBgqkqNOA4oeAHYLKzxQAmu+ss86K0S/eOV70qtlDHrvmVz+KXXaZEMcff3wbzqz5RADQYkIAGJgpAKlYtWpVfOe7/x5jZ/xldIzabtBjN61fHU/f9rP42N9+NMaMGdOmM2wuEQC0QVFDgFoVcZvQIr9hmCkANN/Xv/71yDpGxA4zjhzy2KduuSJGjRgRH/7wh9twZq0hAgAGVNxFIhSBKQCpWLduXXzt64tj+/3eGCPH7DDosT3Pboinl18e733PibHLLru06QybTwQAbVK8V4o3EwKkxRQAmu/b3/52rFmzJsYd+OYhj11369J4bv2a+Lu/+7s2nFnriACgjYRAO7k5mJSZApCK5557Ls4864sxdt+/iFHjJw56bNazKdbf9KP433/917H33nu36QxbQwQAQEGUZQogAEjJpZdeGg/cd2+8uIY3B3v6rv+JZx5/KD71yU+24cxaSwQAbVa9aUDW1dW80wCgabIsi9PPODPGTp0Wnbu9bMjj1/3qsjjkLw6Ngw46qA1n11oiAMhB9UIAhssUAJrv2muvjZtvujF2eN1fD3nsMw+uiPUP3h6f/lTxpwARIgCgTkJgIEXcJrQoyhIAkJozz/xCjJk4Ncbs9dohj137q0viZa/YJ4466qg2nFnriQAgJ0VeMBYnBNwcTEpMAUjJ7bffHldccXm86MD/HR0dg/97/dknHop1v/vvOPlTn4wRI8qxfC7H7wIoqCKHALSHKQC0xhe/+MUYPW5CvOhV/2vIY9f86tKYMGHXOO6449pwZu0hAgAaUpxpAAMr8rsGF40pACl5+OGH49///Xsx9rV/GR0jtxv02E3rnoynb7smPvHxj8WYMWPadIatJwKAnBV5GlCMBaRLgorLFABq95Of/CRe/4Y5ccsttwx57Ne+9rXIRoyMF09705DHPnXz5TF6u5HxwQ9+sBmnmQwRACRACECZmQLQDv/nm9+Ma36+NGbO/PP4yle+ElnW/39b1q5dG19f/I3Yfv8jYsSYHQb9nj3PPhPr/98V8b73vjd23nnnVpx2bkQAkIgihwA0nykA1Oc3t94eL37tX8aYaXPj4x//eMyb91fx2GOPbXPcueeeG+vWrY1xB755yO+57jdLY9PTT8UnPvGJVpxyrkQAwLCZBmxhm1C2ZgpAOzzzzDPx4P33xnYT946d3/D+2PX/OyV+2v1f8Zr99o/uF1wS+dxzz8UXvnh2bL/voTFq3K6Dfs+sZ1Osv+lH8Za3vCX22muvFv8O2k8EAAkp8gIy7RBwX0CxlGUKIABol9/+9rfR09MT2+0yJSIixr7soJh4wldizegJ8frXvz5OOeWUeO655+KHP/xhPPTA/TGuhjcHW/+7/45nnvhDfPKT5XhzsK2NyvsEAPrKIvUF9cA6otghA1BMt99+e0REbDdhSu/nRr14Qux6zD/H6mU/iH/+53+JpT+/Jp588skYu9f0GD1p70G/X5Zlse5Xl8ah/+uwOPDAA1t67nkRAQCQEFMAqN+KFSuic9zOMXKrG307RoyMHQ95R4zZc/+46fIvxoYnH4mJb/vckN9vw4O3xdMP3REnf/OLrTrl3IkAIEGmAQDU7vbbV8TInf5swH8+5s9eHbse/5XYtPbx2G7CnkN+v7W/uiz22feV8aY3Db2FaFG5JwBIVJEX0mkGjPsC+pfSG4aZAkBjfn3rrTFylymDHjNy+xfH6F2nRkfH4H/nn338gVj3u/+OT3/qkzFiRHmXyuX9nQHkKp2FJUCZPffcc3H37+/qvSl4uNb86rKYOGm3OPbYY5vy/VIlAoCEFXkaEPHCEMi6uvI7jTazTWhjTAGgMXfffXc89+yzMaoJEbBp3R/j6duuiU98/GPR2dnZhLNLlwgAEmdBSfmVJQAgD1t2Bhq9y0uG/b3W3PSf0dm5XXzwgx8c9vdKnQgACqDIIeCyIKrDFIA8rFixIrbb/sUx4kU7Duv79Gx8JtYvvyIOOfjgGDduXHNOLmEiAKDlhAADMwWA4VmxYkVst8ufDXnD71A6Ro6Ksfu/Ma6++up409y5sWrVqiadYZpEAFAQRZ4GRKQSAnYIolVMAcjLr2+9LUbsPPz7ATpGjoqduk6MiW9dFL+4/lfxmv32j5/97GdNOMM0iQCgQIoeAtBXWaYAAoC89PT0xG/vvCO222Xg9wio1/Z7HxATT/hqrN9hj3jjG98YJ598cjz77LNN+/6pEAEAABTSgw8+GE+vX9+07UG3GLnDTjHhrYti/GEnxBfOOisO+YtD4957723qY+RNBAAFYxpQBLYJHZopAAzflp2Btpsw/J2BttbRMSLGz3xLTDz2zPj17+6L/fafFhdffHHTHycvIgAoIAtMmiuldw0GardixYoYOXpMjBy3a8seo3PyPjHx+C9Htsf+8ba3vS1+9KMfteyx2kkEAECbmQJAc6xYsSI6d/mz6Oho7ZK2o/NF0dG5Q4wYOTJ23nnnlj5Wu4zK+wQAGpNFKjvuAJCP39x6W3Ts1LybggeyetlFsfb/XRnf/va349BDD23547VD3dl07bXXxrx582Ly5MnR0dERl112Wc1fe91118WoUaNi+vTp9T4sQD+KdVlQ1jU771OICNuE5s0UAJojy7I/vUdAc28K3traX18dq//re3HqqafGu9/97pY+VjvVHQHr1q2LadOmxeLFi+v6uieffDKOP/74eMMb3lDvQwIMolghAEBzPProo7H6yT+2NAKe/v2N8cRVX48PfOAD8Y//+I8te5w81H050Ny5c2Pu3Ll1P9AHP/jBOPbYY2PkyJFDTg82bNgQGzZs6P31mjVr6n48AEiNKQA0z/M7A7UmAjY8/Lt44idnxFFHHRmLFy8e9jsSp6YtNwZ/+9vfjrvvvjsWLlxY0/GnnXZajB8/vvdjypTWjnmAojMNSJFtQvsqSwBAKlasWBEdI0fGqB13b/r3fvaPD8cTl3wuZkzbPy668MIYNap8t9G2PAJ+97vfxcknnxzf+973an4CFyxYEKtXr+79eOCBB1p8lkDxWXBCO5gCkIoVK1bEmF32iI6RzV2gb1q/Op74j8/Gn03aJa64/D9j7NixTf3+qWhp1mzatCmOPfbYWLRoUbziFa+o+es6Ozujs7OzhWcGAO1TlimAACAlt956W3Ts2NydgXo2PhOPX/K52GHEs3H1T6+NCRMmNPX7p6SlEfDUU0/FjTfeGLfcckucdNJJERHR09MTWZbFqFGj4qc//Wm8/vWvb+UpAJVi29BadXR3R9bVlfdpADTstttXxHZ7/6+mfb+sZ1M88ZMzo+PJB+Oqa6+Nvffeu2nfO0UtjYBx48bFb37zmz6f+8Y3vhE///nP44c//GHstdderXx4oJLSDoGsa3Z0dF+T92nQj45FHS25j8EUAJpv9erV8ciqh2PCQc25bzTLsnjip9+IZ+69OS7/z/+M1772tU35vimrOwLWrl0bd911V++v77nnnli+fHnsvPPO8ZKXvCQWLFgQDz30UHz3u9+NESNGxGte85o+Xz9x4sQYM2bMNp8HAIBarFixIiKiaduDrr7+wlj7/66KJUuWxBFHHNGU75m6uiPgxhtvjNmzn3/Dm/nz50dExAknnBBLliyJhx9+OO6///7mnSFA3dKeBlAdpgDQGitWrIjo6IhRO+8x7O+19tc/jdW//H78y7/8S5xwwglNOLtiqDsCurq6IssGHpcuWbJk0K//7Gc/G5/97GfrfViAOgmBFGQLs+hY5M8BaK4VK1bEmJ12ixHbjRnW93n697+KJ65aHB/84AdjwYIFTTq7YmjL+wQA5MO2oeTHFABa57bbbosROw1vCrDh4d/G4z8+I/7yL4+Kr3/966V7M7ChiAAAAArlN7fdHqN2eUnDX//sH/8QT1zyuThgxvS48IILYuTIkU08u2IQAUDJpTcNyLpmD31QG3R0d+d9CqVlCgCt8/TTT8eD998X2+3S2HsEbFr3ZDz+w8/GlN0nlvrNwIYiAoAKSC8EAGjMnXfeGVmWNbQz0JY3A3vxyOfi6quujF122aUFZ1gMLX2fAACoGlMAaK1GtwfNejbF4z8+IzpWPxRXXXtt5d+vSgQAFWG3IKiVACBlK1asiM5xu8SIMTvU/DWb3wxscWy475a44vLLK/FmYENxORBQIS4LykMr3oW3VYa7nWlZpgCQsttuuz1G7lzf/QCrr7sg1v6/n8a3zzsv3vjGN7bozIpFBABUmJuD2ZopAKn7zW23x8ida78U6Kn/d1Wsvu78+PznPx/vete7WnhmxSICgIopzqvSFIspALTes88+G/fcfVfN9wOs//2v4o8//UZ86EMfipNPPrnFZ1csIgCooPxDIJVtQmmOsgSAKQCpe/DBB+O5Z5+tKQI2/OHOeOLHZ8S8eX8ZX/va1yr3ZmBDEQFAReUfAgDU55577omIiO0mDB4Bz/7xD/H4JZ+LA187o7JvBjYUEQAAw2AKAO1z7733xnZjXxwjxu444DGb1v0xHv/hwnjJ5Elx+X/+JLbffvv2nWCBiACgwkwDItwcDBTH3XffHdvtMmXAS3t6Nj4dj1/yuRg3qid+9tOrKv1mYEMRAUDFCYF2KNI2ofUwBYD2uuv398SInfrfHjTb9NzmNwNb83D89KorY+rUqe09uYIRAQAAFMJ9993b703Bz78Z2PK47JJLYvr06e0/uYIRAQCmAbxArW8YZgoA7bdxwzP9RsDqX54fa399dXz7vPPi8MMPz+HMikcEAESEEKCKBABFtN2El/T59VPLr4zV118Qp59+ujcDq4MIAOhV3RBwc3B9yjIFgKIZObozRo6b0Pvr9XfdEH/86TfiIx/5SHzqU5/K8cyKRwQA5MQbhpEnUwCKaPQuU6KjY/PydcMf7ownfnJG/NWb/yq+8pWveDOwOokAgD6qOw2gNqYAkJOOETFi5833Azz7xEPx+CWfi9cdcEBccP753gysASIAYBtCoBXKuk1oEZkCUDRZtvnfH9vtMuVPbwb22Zi6x27eDGwYRAAA1MgUAPLx+OOPR2Q9MfLFE+Lx//hcjB+dxdU/vSp23nnnvE+tsEblfQIAacoiwvWllI8pAEV0zz33RETEmmU/iNEbn4yf/vKXseeee+Z8VsVmEgAwoGpdvmKHoMGZAkB+tkRAz5qV8aPLLotp06blfEbFJwIAYAhlCQBTAIpq1apVERHx2YUL4w1veEPOZ1MOIgBgUNWaBrBZre8aXCQCgCL78Ic/HBERRx55ZM5nUh4iAGBIQqDKyjIFgCKzBWjzuTG4RlNPvryhr9vwhztj1UWfidG77hkT37ooRnSObfKZAe3RmhuFs67Z0dF9TdO/b6qyhVkpX2VPnSkAsDWTgBYSAADFZgoAlJUIaBEBAGVU/suC7BBUPqYAQH9EQAsIACiz8ocAm5kCAGUmAppMAACQClMAYCAioIkEAFSFaUDZXfPZ6tysDVSTCGgSAQBVIwRImykAMBgR0AQCAKpKCDQiW5j281aGKYAAAIYiAoZJAADDlXXNzvsUAKgYETAMKQRAz4b1bX9M4IXSflW7EVXeJtQUAKgKEdCgVALgkYsXtv1xga2VLwQAKDcR0ICUAmDjo/e1/bEBysgUAKgSEVCn1AJg0jGntv3xgf6YBhRZGQIAoB4ioA4pBkDn5H3afg7AQMoUAh15nwB1MgUA6iECaiQAAJonpW1CTQGAKhIBNRIAQG3SWdwOR0f3NWEaUBymAEC9RECNBABQu/pDIN33Cih3CJgCAFUlAmokAID6lGMisFm5Q6DoTAGARoiAGgkAgHIpwxRAAACNEgGJEgBQBqYBAKRJBCRIAECZCIEUmQIAVScCEiMAgFRs3iFom8827funtE0oQNWIgIQIACgri92UmAIAiIBkCAAouzKFQHkuCwKoKhGQAAEAVTF4CKT7XgH9KWYImAIAbCYCciYAgOIqZggAIAJyJQCgisp0WVCxmAIAPE8E5EQAQJWVKQSGNw2wQ1DtBADQTCIgBwIAKIr+twnd5qiWn8dwlWEKANBMIqDNBACwWdleAU83BMoQAKYAQLOJgDYSAEBf24ZAsXYIAqCoRECbCACgf2WaCKQ3DTAFAOifCGgDAQBUR3ohAMC2RECLCQBgaGWaBkSkEgKmAAADEwEtJACA2qUbArXtELTNV9V8pG1CAdpPBLSIAADIjykAwOBEQAsIAKAxZXtFPI3LgopIAACtJgKaTAAAw5OVbJvQ9odAGaYAAK0mAppIAAD0x0SgHqYAQDuIgCYRAEDzpHdZUGM3B7efKQBAbURAEwgAgKGYBtTCFABoFxEwTAIAoFYDh0Aztgk1BQConQgYBgEAUC8TgYGYAgDtJAIalEoAPHn9hbk8LkBKTAEA6iMCGpBSAKz+r+/l8tgAjWvuNKAMAWAKALSbCKhTagEw/tB35vL4QGtlXV15n0KLuSwIIE8ioA4pBsCOB789l3MAqqf524QOPwRMAQAaIwJqJAAAaDYBAORFBNRIAAC0wvPTgHq3CS3DFAAgLyKgRgIAoFWqeX+AKQCQJxFQIwEA0Er1hYApAMDwiIAaCQAAmsUUAMibCEiYAIBqS3Gb0ObvENTnu9d0lCkAwPCJgEQJAIByMgUAUjAq7xNgWwKAVHR0vGDnlqy+nVugFUwBAJrDJCAxAoBUvDAAtvx6ywe0SrYw7zNoLVMAIBUmAQkRAKSglkX+lmNMB2inok8BBACQEpOARAgAUlDvq/wmAwBQTHVHwLXXXhvz5s2LyZMnR0dHR1x22WWDHn/JJZfE4YcfHrvuumuMGzcuZs2aFVdddVWj51tKAoAUDGcxLwRoNVMAgOaqOwLWrVsX06ZNi8WLF9d0/LXXXhuHH354XHHFFXHTTTfF7NmzY968eXHLLbfUfbJlJABIQTMW8aYCrVG9bUIBaIe67wmYO3duzJ07t+bjv/zlL/f59ec///n40Y9+FD/5yU9ixowZ9T58qQgAUtDshXtHR4d7BWgqUwCA5mv7jcE9PT3x1FNPxc477zzgMRs2bIgNGzb0/nrNmjXtOLW2EgCkoFWv3LtxmGYpegAApKrtNwafddZZsXbt2njb29424DGnnXZajB8/vvdjypQpbTzD1hMApKAdl+64PIhGlGmbUFMAIFVtjYDzzz8/Fi1aFD/4wQ9i4sSJAx63YMGCWL16de/HAw880MazbC0BQNUIARplCgDQOm27HOjCCy+M973vfXHxxRfHnDlzBj22s7MzOjs723Rm7SMASEW7F+buE6CKTAGAlLVlEnDBBRfEiSeeGBdccEEcddRR7XjI5AgAUpHXK/MmAuXS6h2Cij4FEABA6uqeBKxduzbuuuuu3l/fc889sXz58th5553jJS95SSxYsCAeeuih+O53vxsRmy8BOuGEE+IrX/lKzJw5M1auXBkREdtvv32MHz++Sb+NtAkA2MxEAADSUPck4MYbb4wZM2b0bu85f/78mDFjRpxyyikREfHwww/H/fff33v8N7/5zXjuuefiIx/5SOy+++69Hx/72Mea9FtImwAgJSm8Gp/CORRNiu8V0ErdHaYAAK1W9ySgq6tr0FfylixZ0ufX3d3d9T5EaQgA6J+JAADkq+1bhFaFACA1qb0Cn9r5kAZTAID2EAEtIABITaoL7lTPCwDKTgQ0mQAAaIwpAED7iIAmEgCkyKvttEqrtwkFoHVEQJMIAFIkACgKUwCA9hIBTSAAgFar2jahRSIAgCISAcMkAEiVKQBFUfQpAEARiYBhSCEANvzhzlweFwBTAKC4RECDUgmAVRd9JpfHJm2mABSFKQBAPkRAA1IKgNG77pnL45MuAUA7DWeHoKIHgCkAUGQioE6pBcDEty7K5RwAACguEVCHFANgROfYXM6DNJkCUBRFnwLM/uzsvE8BYFhEQI0EAJA324QC0CwioEYCgNQVcQqQZVnep0AOTAEA8icCaiQASFkRAwAAyI8IqJEAAOhfPTsEmQIApEEEJEwAUAtTAGgPAQCUiQhIlACgFkUOAPcDVE/RpwBb61hU3L9/ACIgQQIAIC2mAEDZiIDECABqVeQpAI0r6jahZZsCABSdCEiIAABIjykAUEYiIBECgHoUfQrgfoBqMQUASI8ISIAAoB5FDwDKqZ5tQovEFAAoKxGQMwEAlJkpAECaRECOBAD1MgWgSIoeAKYAQJmJgJwIAKrK/QAUgQAAyk4E5EAA0AhTALYowjahRZ8CAJSdCGgzAQCQtnqmAN41GCgqEdBGAoBGlWUK4FKgctuyQ5ApAED6RECbCAAaVZYAgCJwLwBQFSKgDQQAUBWmAADFIAJaTAAwHKYA0D6mAECViIAWEgDwPPcDlN811tAAhSECWkQAAK1UhG1Ci8QUAKgaEdACAoBmcCkQRVLkKYAAAKpIBDSZAIBtuRQIANIiAppIAABVVPUpgDcMA4pIBDSJAAAAoChEQBMIAJrN/QAURdWnAABFJQKGSQDA4NwPAADpEQHDkEIA9GxY3/bHBNKQ9zahpgAAxSUCGpRKADxy8cK2Py5AkQMAABHQkJQCYOOj97X9saFWLgUiRaYAACKgbqkFwKRjTm3749N6Fs+krMhTAAEAsJkIqEOKAdA5eZ+2nwMAAMUmAmokAGi3ok8Din7+9M8UYCC29QWKRQTUSACQBwtpKIaORRFCACgSEVAjAQCkqJ3bhJoC1EIIAMUgAmokAMiLaQAA0GwioEYCgDwVLQSKdr4MzRSgHqYBQPpEQKIEAFuzsIYiEQJA2kRAggQAAxEC5MEUoFFCAEiXCEiMAGAoqYdA6udHdXhjMICBiYCECABqZaFNuxR5CpAG0wAgTSIgEQKAemVZJgaIiPZuE1oUaU0BhACQHhGQAAHAcKQUAimdC8NnCtBMQgBIiwjImQCgGUwFaLYiB0BeU4DN7xoMUAwiIEcCgGYTApAy0wAgHSIgJwKAVskrBARIeZgCtJIQANIgAnIgAGg1lwdByoQAkD8R0GYCgHYSAtTLFACgGkRAGwkA8tCOqYDYyJ9tQovGNADIlwhoEwFA3izUGYopQLsJASA/IqANBACpcK8AZVTMANhCCAD5EAEtJgBIUTNDQFSUQ5GnAADUTwS0kAAgZc2YCggA8lbsKcAWpgFA+4mAFhEAFEUjC3mXFZWLKUDzNP6uwUIAaC8R0AICgKKpZ1Fv8U8qurKy1YsQANpHBDSZAKDIhooBAZC2RrYJNQUAqCYR0EQCgLLYOgZc/kNqtkwBsoU5n0jTmQYA7TEq7xMoCwFAGVn4l5spQKo6IsLfPaC1TAKaQAAAtE9X1pX3KbSBiQDQWiJgmAQAUERFnQI8HwBVeKVcCACtIwKGQQAARVTUAACgeURAg1IJgCevvzCXxwVot20vAzINAGiUCGhASgGw+r++l8tjA2kaapvQ8k0B0guBxt8wbMDv2OxvCCAC6pVaAIw/9J25PD5AOw12M3D5tgntjxAAmksE1CHFANjx4Lfncg5A8ZRvCrBFetMAgNSJgBoJAIB8VGNL0FqYBgDNIwJqJACAIivvFKBqhADQHCKgRgIAoP1MAfojBIDhEwE1EgBAURV1CiAAAFpHBNRIAABFMdQ2oZSBaQAwPCIgYQIAGK6qTAGyhVXcIUgIAI0TAYkSAAAMTQgAjREBCRIAQDNUZQqQoua/azBAc9UdAddee23MmzcvJk+eHB0dHXHZZZcN+TXd3d3x2te+Njo7O+NlL3tZLFmypIFTrQYBAEB9TAOA+tUdAevWrYtp06bF4sWLazr+nnvuiaOOOipmz54dy5cvj49//OPxvve9L6666qq6T7bsBADQLKYAVSMEgPqMqvcL5s6dG3Pnzq35+HPOOSf22muv+OIXvxgREa985Svjl7/8ZXzpS1+KI444ot6HLy0BAMDwdEREFW+QBhrR8nsCli1bFnPmzOnzuSOOOCKWLVs24Nds2LAh1qxZ0+ejzAQA0EzdHd15n0JDTAEA2qflEbBy5cqYNGlSn89NmjQp1qxZE08//XS/X3PaaafF+PHjez+mTJnS6tPMjQAAmqnKAVDNbUK35rIgoDZJ7g60YMGCWL16de/HAw88kPcptYQAAKD5hAAwtLrvCajXbrvtFqtWrerzuVWrVsW4ceNi++237/drOjs7o7Ozs9WnlisBADRblacAbM39AcDgWj4JmDVrVixdurTP566++uqYNWtWqx86WQIAAIA81R0Ba9eujeXLl8fy5csjYvMWoMuXL4/7778/IjZfynP88cf3Hv/BD34w7r777vjUpz4Vd9xxR3zjG9+IH/zgB/GJT3yiOb+DghEAQCuYArAtlwUBA6s7Am688caYMWNGzJgxIyIi5s+fHzNmzIhTTjklIiIefvjh3iCIiNhrr73i8ssvj6uvvjqmTZsWX/ziF+Pf/u3fKrk9qAAAqI403jVYCAD9q/uegK6ursiyga8z7O/dgLu6uuKWW26p96FKRQAArWIKwODcHwBsK8ndgcpGAAC0h21CAWojAlpMAACtZApAbVwWBPQlAlpIAACQDiEAPE8EtIgAAFqtqFOA2ddEdHR3530aFSUEgM1EQAsIAID+zb4m7zMAIEIENJ0AANqhqFMAUmAaAIiAphIAAANr1xTADkG1EAJQdSKgSQQA0C6mADSHEIAqEwFNIACAdilqAFT1XoA03jUYYFt1v2MwfQkAgMZ0dHdH1tWV92lUnHcTrqzzh5gEHevnouxEwDCkEAAb/nBnLo8LtJ8pAK0hBEpnqAV+rd9DCJSaCGhQKgGw6qLP5PLYAJSJECi8Ziz8+/ueQqC03BPQgJQCYPSue+by+EB7mQLQem4ULqxWBECEACg5EVCn1AJg4lvddQakKc8AsE0oldGKADg2EwAVIALqkGIAjOgcm8t5AO1T1CkARWQaUCitCgAqQQTUSAAA1M5lQEUmBAqhVZcAURluDK6RAADyUPYpgG1CU+VG4SS1euFvClApIqBGAgCgNqYA0ETtesVfAFSOCKiRAADarexTgKroWBSRLcz7LBphGpCbdl/qIwAqSQQkTAAARWMKUDZCoK1c508biYBECQCoNlOA4csWZtGxyKJq+IRAS+W98DcFqCwRkCABABSRKQDUIe/Ff4QAqDhbhCZGAABFnAIMJwA6urubdh60SgIL1rI4v0MAkASTgIQIAADS5bKgYctz8W/Rz1ZEQCIEABBRvSkARSMEGpLX4t/Cn0GIgAQIACiRjj/9xz6r/z++RQwAYAh5BIDFPzUQATkTAFAiHR3b/v8GYqBITAGqyDSgJl79J3EiIEcCAEqkY4D/4L/w84MEgSlAa9gmtFWEQL/yvuFXAFAHuwPlRABAiQwUAP0dV+uxBWAKULuORXmfQSuU52d52FLY8UcAUCeTgBwIACiRRhb1HR19pgKmAJu3Cc26uvI+Dahd3ov+FxIANMAkoM0EAJTIcF7VL/hUwBSAzYr7MzwsKQUANMgkoI0EAJREExfvRZwCCAD6qtD9ASku/k0BaJAIaBMBACVR4FfvoXUqEAKpBYDFP8MkAtpAAAD96Y7ivaRuCkDlpLT4t/CniURAiwkAKBFTgEKyTWi7lHAakEIAWPjTIiKghQQAMBBTgG3ZIagMShACKSz8Iyz+aTkR0CICAErGFABqVNAQSGXxHyEAaAsR0AICAEqmyQFgCgAJsfinorxPQJMJAEhfd3dC/9GnMsr5rsEDKcDfsRTe5feFBABtZhLQRAIA0vbCxf8L/39X1yD/8TUFMAWgQYleFpTSwj/C4p/ciIAmEQCQtsFe/d/yzwaNgWacgwAgsijEq+RNk1AIpLb4jxAA5MrlQE0gACBttV7+093d0fsREW4GLpFsYUqLrZTOpSIEAGzDJGCYBACkazjX/nd3d0RX807FFKBGtgktoxynARb/MCARMAwpBEDPhvVtf0wogmbc/Nt9TUTX7CacDGzDZUEtZfEPQxIBDUolAB65eGHbHxdS18zdf5oRAqYA9E8INJ3FP9RMBDQgpQDY+Oh9bX9sqBoTAUicxT/UzY3BdUotACYdc2rbHx9S1qr3AOhu8JVxUwAGV7WFYpP/fqa213/E5sW/AKAATALqkGIAdE7ep+3nAKlq9ZuAbQkBU4FiyhZm0bEosQVjRLgsqAGpLfwjLPwpHJOAGgkASFs73wW41qmAKUBjOrq78z6FlkkzQvLQ4PPglX9oGhFQIwEA6WpnAPQ+ZgKLZcrEInJQKS7+Iyz+KTQRUCMBAGnKIwB6H3uQEDAFoH5VW1DW8Hc31cV/hACg8ERAjQQA0J+yTAQEQCqqtrAcZIGf8uJfAFACbgyukQCA9OQ5BXihrbcQLeIUAPKz1Y3CKS/+oURMAhIlAGBwqQTAFkWeCJgCpKaii02X/kBbmQQkSADA4FILgC26rwkr6sSlu03o1iq0bej5EUn+Xi38KTmTgMQIACi4a4r1JgKpNkuZtwnlT87/00dqXPNPRZgEJEQAQElcMzvd1TUFUtJpQIoL/wgLfyrHJCARAgBqk+qlQNsowESgK+vK+xQYUokWpl75h6SIgAQIACipAoQA7Vf/PQklWKCmuPiPsPin0lwOlDMBALUrzBTghRK9NMgUgLaw+IdkmQTkSABARSQ2ERAARVSwRWuql/5ECAD4ExGQEwEAFZNYCGyRdXXlfQoDatUOQdnCoi4CC3DeqS/+BQD0cjlQDgQAVFQClwaZAtASqS78Iyz8YQAioM0EAFRcAiFAkSW2bajFPxSWy4HaSAAAEZHbpUGmAGWRyOJWAEChiYA2EQAwfF1dJfoPe6L3CFAUOf5dcN0/lILLgdpAAAD9auOlQaYADFuqC/8IC39ogElAiwkAYFAmApVU/xuG9adNC1+v/EMpiYAWEgBATVocAkNNAWwTWmQt/H1Y/EOpiYAWEQBAXVoUAi4Dom4pL/4jIo6NSGqHJCgo9wS0gAAAGmL7UBrSpG1DU174R/xp8Q80i0lAkwkAaK1S7RDUnyZOBEwBqmSYfy9SDoBjY4AAMA2A4TAJaCIBADSFiQANaWAikPrif0gdkcz7JkDBmAQ0iQCA9in9NCBi2BMBUwAGlfJ1/wO+8j8QEwFohAhoAgEAtITtQyPCDkH1GeL3lPLiP8J1/9BGImCYBADkoxLTgIiGQqCRKUDK24RSr37+bhRh8T+sADANgHq5J2AYBADQFu4RoFEpL/wjmvzKv/sDoB4mAQ1KJQCevP7CXB4XaLMaJwLuBWCzLO0AGPYr/wMxEYBaiYAGpBQAq//re7k8NqSgMpcEbTFECAiAYulY1KIF6/kdmz9S1LLFP1AvlwPVKbUAGH/oO4UAAOku/CPavPB3WRDUwiSgDikGwI4Hvz2Xc4BUmAZsZgrQmFLsEJTyK/8ROb3yn/DzAYkQATUSAEAyKrh1aKu2CS20Iiz+XfoDyRIBNRIAkK6u6q2J+2jWFMA2oQVh8V+jhJ8jSIAIqJEAAJJSwWkAYfFft4SfL8iZG4NrJAAgUR2b/yPfNTuiu4Jb6bsXoCJSX/wnzY3C0B8RUCMBACTnmtlhcVNyKS/+IwoQAMBAGrocaPHixTF16tQYM2ZMzJw5M2644YZBj//yl78c++yzT2y//fYxZcqU+MQnPhHPPPNMQydcJQIAhtDRd4FU9XsDaEySOwS57r/JEn4uISd1R8BFF10U8+fPj4ULF8bNN98c06ZNiyOOOCIeeeSRfo8///zz4+STT46FCxfGihUr4txzz42LLroo/uEf/mHYJ19mAgAaU7UQ6O6uzuKmEjsEWfy3UMLPK+Sg7gg4++yz4/3vf3+ceOKJ8apXvSrOOeecGDt2bJx33nn9Hn/99dfHIYccEscee2xMnTo13vjGN8Y73vGOQacHGzZsiDVr1vT5qBIBADXo8B/0LaoUAqVl8d8mCT/H0GZ1RcDGjRvjpptuijlz5jz/DUaMiDlz5sSyZcv6/ZqDDz44brrppt5F/9133x1XXHFFHHnkkQM+zmmnnRbjx4/v/ZgyZUo9p1loAgCGr2rTgGayTWibWfwDOanrxuDHHnssNm3aFJMmTerz+UmTJsUdd9zR79cce+yx8dhjj8Vf/MVfRJZl8dxzz8UHP/jBQS8HWrBgQcyfP7/312vWrKlECAgAqJEpACWQvTzvMxhEqRf+dguCiDa8T0B3d3d8/vOfj2984xtx8803xyWXXBKXX355nHrqqQN+TWdnZ4wbN67PR9kJAGiuKk0DurosaIoke7kAyJ8XEqCuScCECRNi5MiRsWrVqj6fX7VqVey22279fs1nPvOZeNe73hXve9/7IiJiv/32i3Xr1sUHPvCB+Md//McYMcL7lQkAqIMpAAWV9MI/oiKL/xcyEaDa6lqBjx49Og444IBYunRp7+d6enpi6dKlMWvWrH6/Zv369dss9EeOHBkREVnmL58AgNap0jSA4WnlNqHpv/Kfbf4AKqXuNwubP39+nHDCCXHggQfGQQcdFF/+8pdj3bp1ceKJJ0ZExPHHHx977LFHnHbaaRERMW/evDj77LNjxowZMXPmzLjrrrviM5/5TMybN683BqpKAECdTAG2UbVLgTq6uwtz83LSC/8IC/+IMA2gyuqOgGOOOSYeffTROOWUU2LlypUxffr0uPLKK3tvFr7//vv7vPL/T//0T9HR0RH/9E//FA899FDsuuuuMW/evPiXf/mX5v0uCkgAQHt0zY7ovibvs6BKirv4z6Ka18oLAaqp7giIiDjppJPipJNO6vefdW/1Zi6jRo2KhQsXxsKFCxt5qFISANAAU4C2ybq6qvHGXC1Q3ADYQghAVbgrt80EALSfewNoNdf9A0XT0CSAxggAaJApQL+qdj9AipJe+Ec0uPA3DYAqMAloEwEA+TINoJnK/8p/VRfDVYwfqkoEtIEAgGEwBehXlacArbpfoZZtQsu/+H+hqv6M+XcO1SACWkwAQEJml2OboCoHQJ6qs/gHqsA9AS0kAGCYTAFIQNKL/4gWL/7dHwBlZRLQIgIA0tIdf5oCFHwa0K4pQFHekKuVqnXpz2CquhiuYvxQJSKgBQQANEErpwAFDQGXAbWHxT/PEwKUl8uBmkwAQHp6pwAFJgDa4PyO9Bf/uanqZUFQXiKgiQQANEkTpwADBsDsayKuKca+oQKgxc5PfHGbzKv+VQ0B9wdQTiKgSQQA0AoCoH8d3d3Dv2/B4r8BQgDKwj0BTSAAoInaMQXYoqD3BjBM53cIABqQ+M8M1MkkYJgEANAqpgAtYPHfBFWdBkC5mAQMQwoBsOEPd+byuNAS7ZwCbJHoNCCFACjVNqGpv/pfuB1/inSuzZTwzxDUySSgQakEwKqLPpPLYwOtk0IAlErCi/+O323+X3/iReL+AMrBJKABKQXA6F33zOXxoenymAJskdA0QAA0UcKv/nf87vkAKC4/q1BkIqBOqQXAxLcuyuUcgOYTAE2S8OI/ogyL/xeq6s9suj9fUCsRUIcUA2BE59hczgOaKs8pwBYJTQOoTUd3d99PFGDxX64A2EIIQBG5J6BGAgBoJVOAYUh44R9R1oU/m7k/gOIyCaiRAIAWSWEKsEVO0wAB0JjsD7OTDoDyvvLfHz/DUDQioEYCACqizSGQegCkuE1o9ofZmwMgUdVa/L9Q2j/LrZNuiMJgRECNBAC0QEpTgBykHgCpKfPiv2ORhWSx+fOjeERAwgQA5KQN0wABUJ+UF/8RVX3lvz9V/rkWAhSLCEiUAKD0Up8CtDAEBEB9Ug6A6l76Mxg/31AEdgdKkAAA2CzVALDwH0oW1Xxl3G5BFIcISIwAgPq09F6A2ddEXNPcRagpwNBSXfhHWPxTCyFAMbgcKCECgMpo0qVAbbkZuImXBQmAwaV846/LfhpR5Z/3Kk5BKBoRkAgBAOVW5ABo9TahFv9lVtyfeyg7EZAAAUClFGkKsMUwpwFFDoBWSnnxH8dmmz+gYaYBpE0E5EwAQEE0GgI5vQtxypJe/EdY/DddlZ9PIUC63BicIwFA5RRxCjAcs6+Jrqwr77NIRtIL/wiL/5aq6m5BkC6TgJwIACigel7VNwHoI+kAGOTSn2yhMGieqj6X4oc0mQTkQABA43KfAtSxuDcF2CzZAPDKP21j21DSIwLaTAAAVZJkAFj856jKlwUJAdIiAtpIAMDw5D4FqEPZpgBZV1d0dHfXfrzFPwOqcghAOkRAmwgAKq9JNwWTtiQX/xECgESYBpAONwa3gQCA4TMFSFuy2342Yb//dt0c3LGoSqFc5YVwlf6cSZlJQIsJAKiWqgVAkgv/CK/8F4LLgiBPIqCFBAA0R5GmAFVh8U9zVDUEXBZE/kRAiwgAqJ6yTwGSXfhHRMfkayLr6sr7NKAOQoB8iYAWEACwlWHcFGwKkIDz032ltmOyn4/iq+o0IEIIkCc3BjeZAIB+ZOX/j1xppwACICK8c3DreX6h3UwCmkgAQHOZAuQs0QDw6n9ZVXUiYBpAPkwCmkQAwBDqnAYUKQBKOQVIMAA6Jl8zaADU82ZmkJb0/r5RfiYBTSAAgFJJLAC88l8lVZ0GQPuJgGESAFCHLKvpJmFTgBwlEgAW/lVW1RBwWRDtJQKGIYUA6Nmwvu2PCZRUAgFg8U+1CQHaRwQ0KJUAeOTihW1/XGglU4Cc5BgAFv5sq6rTAGgfEdCAlAJg46P3tf2xYVhqvCQodYUPgARe9Y8oVgBkC7PoWNTa561jUYftSHtVNQRMA2gPuwPVKbUAmHTMqW1/fGiVIk0BCun8juc/EtCKALBDUNlUdTGcxt9Rys0koA4pBkDn5H3afg4wbAWfBhRuCpDIov+FijQBACgjEVAjAQBNtlUImAJUg8U/9XNZELSCCKiRAIAWKOBEwBSgMRb/DI8QgGYTATUSANAiWRbdHd15n0U55RwAFv40V1VDAFpDBNRIAACFmgLkEABVWPS3Y4cg6Ms0gNYQATUSANAapgDFV4XFP6mo6jRACNB8tghNlACAtJgC9C/FACj6NqEmDUOxGIZmEAEJEgBURVGmAAKgfykGAJSXOKS5REBiBADQsHYFwLGZACBnVZ0GCAGaRwQkRABQJaYATdbGAIA0+FmE4RABiRAAVElRAoCtCACSU8WfSdMAmsPuQAkQAJCmwkwBWsWiHxJltyCGzyQgZwKAqjEFSNSx2bYfg8i6utpzXg1q5Q5B2UKLr7T484BGmATkSABAugo3BRho0b71vQJe3aeUqvj+AaYBDI8IyIkAoIpMAXJg0U9lCAGoh8uBciAAIG2FmwIAQJ1EQJsJAKqqKFMAAQBFVsVXxas2/aBZREAbCQAAaLUqhgDUTwS0iQCgykwBKBs7BKWuan8+pgHUTwS0gQAAyqjK24RCeoQA9REBLSYAqDpTAMrKNCB1/nxgMCKghQQAAOSpaiFgGkDtRECLCAAwBaD8TAOKwJ8R9EcEtIAAAADyYRpAbURAkwkA2MwUgKowDSiCqv0ZCQGGJgKaSABAsQiA8rNDEM+rWgjA4ERAkwgAeF5RpgAMX+rbhEJfVQoB0wAGJwKaQABA8ZgC0EwuCQKKRgQMkwCAvkwBgLRVKdhMAxiYCBgGAQB9FSUATAFoBdOAIqnSn5UQoH8ioEGpBMCT11+Yy+MCQLFVKQRgWyKgASkFwOr/+l4ujw1bMwWA5jBRoPlMA9iWCKhTagEw/tB35vL4AEVRpG1CBUC7eb6pLhFQhxQDYMeD357LOcALmQJUm21Cn2cRX0RV+TMzDaAvEVAjAQDFJgBInYDIU1WeeyHA80RAjQQA9K8oUwBImQBIgT8DqqWhCFi8eHFMnTo1xowZEzNnzowbbrhh0OOffPLJ+MhHPhK77757dHZ2xite8Yq44oorGjrhvAgAKC5TAFImAGgv0wA2G1XvF1x00UUxf/78OOecc2LmzJnx5S9/OY444oi48847Y+LEidscv3Hjxjj88MNj4sSJ8cMf/jD22GOPuO+++2LHHXdsxvm3jQCAbZkCwPAIgNRkYZFMVdQ9CTj77LPj/e9/f5x44onxqle9Ks4555wYO3ZsnHfeef0ef95558UTTzwRl112WRxyyCExderUOOyww2LatGkDPsaGDRtizZo1fT7yJgCgmEwBiGjvDkG1LuwFQKqq8OcidKgzAjZu3Bg33XRTzJkz5/lvMGJEzJkzJ5YtW9bv1/z4xz+OWbNmxUc+8pGYNGlSvOY1r4nPf/7zsWnTpgEf57TTTovx48f3fkyZMqWe0ywNAUDKTAGgMdnCTAAkz58P5VfX5UCPPfZYbNq0KSZNmtTn85MmTYo77rij36+5++674+c//3kcd9xxccUVV8Rdd90VH/7wh+PZZ5+NhQsX9vs1CxYsiPnz5/f+es2aNZULAQEAw2cK0D5ZV1eh9uNvNYv8MnBpEOVW9z0B9erp6YmJEyfGN7/5zRg5cmQccMAB8dBDD8UXvvCFASOgs7MzOjs7W31qyRIApM4UAKDoOsLEo9rqioAJEybEyJEjY9WqVX0+v2rVqthtt936/Zrdd989tttuuxg5cmTv5175ylfGypUrY+PGjTF69OgGTru8BAA0hykAMHymAZRXXfcEjB49Og444IBYunRp7+d6enpi6dKlMWvWrH6/5pBDDom77rorenp6ej/329/+NnbffXcBsBUBQBEUYQogAIDm8Wo55VT37kDz58+Pb33rW/Gd73wnVqxYER/60Idi3bp1ceKJJ0ZExPHHHx8LFizoPf5DH/pQPPHEE/Gxj30sfvvb38bll18en//85+MjH/lI834XJSAAACBVQoDyqfuegGOOOSYeffTROOWUU2LlypUxffr0uPLKK3tvFr7//vtjxIjn22LKlClx1VVXxSc+8YnYf//9Y4899oiPfexj8elPf7p5v4uCEwAUhSkARdbR3R1ZV1fep0FhuTSIcmnoxuCTTjopTjrppH7/WXc/u0PMmjUr/vu//7uRhyo9AQAA5MPNwVVW9+VANI8AoEhMARiKV9kpPwtmykME5EQAAEARCQHKQQTkQABQNKYAAC8kBCg+EdBmAoCiEQAA/RECFJsIaCMBAJCvjn42r4DqEjJVJgLaRABQRKYAAIOxiKa4REAbCAAAKCshQDGJgBYTABSVKQCNsE0o1SQEKB4R0EICAACqomghULTzpdlEQIsIAIrMFACgERbWFIcIaAEBAABAykRAkwkAis4UgLKzTSitZRpAMYiAJhIA0HoCAEhf6iGQ+vnRDiKgSQQAZVCEKQBAMaS60E71vGg3EdAEAgDawxSgGGwTCluktuBO7XzIkwgYJgFAWZgCALRCKgvvVM6DVIiAYUghADb84c5cHhfazRQAKC4LcNIjAhqUSgCsuugzuTw25WIKQNXYIYj2yzMERAjbEgENSCkARu+6Zy6PD+1kCgCUQx6LcQFA/0RAnVILgIlvXZTLOVAepgAAZSUAGJgIqEOKATCic2wu5wHtYgoAlEu7FuYCgMGJgBoJAMrIFIBWsU0oDKbVC3QBwNBEQI0EALSfKQBQXlm0ZrEuAKiNCKiRAKBsUp8CCACgGpq5aBcA1E4E1EgAAJSLbUJJRzOmAgKA+oiAhAkAWsUUACBFjS7kBQD1EwGJEgAAUEX1TgUEAI0ZlfcJsC0BQCuZAgAUwdaL+44ajoHamQQkRgAAZWGbUGimrJ8PaJwISIgAoNVMAQCACBGQDAFA1QkA8mCHIKCqREACBADtkPoUAABoHxGQMwFAO6QeAKYAANBeIiBHAgAAgDyIgJwIANrFFIA82SEIIE0iIAcCAACAPImANhMAtJMpAADQHxHQRgIAnicASIVtQoEqEgFtIgBot9SnAABAfkRAGwgA6MsUAADyJQJaTACQB1MAAGAwIqCFBABsyxSgemwTCpAeEdAiAoC8mAIAAEMRAS0gAKB/pgAAkAYR0GQCgDylPAUQAKTMNqFA1YiAJhIAAAAUgQhoEgFA3kwBAIBaiYAmEAAAABSJCBgmAUAKTAFInW1CAdIiAoYhhQDo2bC+7Y8JAECxiYAGpRIAj1y8sO2PS1pMAaA57BAEVIkIaEBKAbDx0fva/thQCwEAAOkSAXVKLQAmHXNq2x+fdKQ8BQAA0iUC6pBiAHRO3qft5wBDMQUAgLSJgBoJAFJjCgAANEoE1EgAQG1MARiIbUIB0iECaiQASIkpAAAwHCKgRgIAhmYKQNHZJhSoChFQIwFAKkwBAIDhEgGJEgAUjSkAABSHCEiQAGAgqU4BBAAAFIsISIwAAACg1URAQgQAgzEFoAxsEwqQBhGQCAEAkAY7BAFVIAISIAAYiikAANBMIiBnAgAAgHYTATkSANTCFAAAaDYRkBMBQJEJAAAoNhGQAwFArVKdAgAAxSYC2kwAUHSmAAyXbUIB8icC2kgAUA9TAMiPbUKBshMBbSIAKANTAAAoBxHQBgKAeqU4BRAAAFAeIqDFBAAAAKkRAS0kAGiEKQAA0GoioEUEAI1IMQAAgPIRAS0gACgTUwBaoQjbhNohCCgzEdBkAoBGmQIAAO0iAppIAFA2pgAAUE4ioEkEAMOR4hRAAABAeYmAJhAAAAAUiQgYJgFAGZkCAEC5iYBhEAA0Q4qXAgEA5SYCGpRKADx5/YW5PC7lZQpAu9gmFCA/IqABKQXA6v/6Xi6PTXOYAgAAeWgoAhYvXhxTp06NMWPGxMyZM+OGG26o6esuvPDC6OjoiKOPPrqRh01CagEw/tB35vL4lJMpAABUQ90RcNFFF8X8+fNj4cKFcfPNN8e0adPiiCOOiEceeWTQr7v33nvj7//+7+PQQw9t+GTzlmIA7Hjw23M5BwAAiqvuCDj77LPj/e9/f5x44onxqle9Ks4555wYO3ZsnHfeeQN+zaZNm+K4446LRYsWxd577z2sE86LAKDZUrsUyBQAAKqjrgjYuHFj3HTTTTFnzpznv8GIETFnzpxYtmzZgF/3uc99LiZOnBjvfe97a3qcDRs2xJo1a/p85E0AAABQFnVFwGOPPRabNm2KSZMm9fn8pEmTYuXKlf1+zS9/+cs499xz41vf+lbNj3PaaafF+PHjez+mTJlSz2m2hACgmUwBAIA8tXR3oKeeeire9a53xbe+9a2YMGFCzV+3YMGCWL16de/HAw880MKzrI0AoKwEAHmyTShAPkbVc/CECRNi5MiRsWrVqj6fX7VqVey2227bHP/73/8+7r333pg3b17v53p6ejY/8KhRceedd8ZLX/rSbb6us7MzOjs76zm1lhMANEtqUwAAoHrqmgSMHj06DjjggFi6dGnv53p6emLp0qUxa9asbY7fd9994ze/+U0sX7689+Ov/uqvYvbs2bF8+fIkLvNJmQCg1UwBAKCa6poERETMnz8/TjjhhDjwwAPjoIMOii9/+cuxbt26OPHEEyMi4vjjj4899tgjTjvttBgzZky85jWv6fP1O+64Y0TENp+nLwFQTilNAQQAAFRX3RFwzDHHxKOPPhqnnHJKrFy5MqZPnx5XXnll783C999/f4wY4Y2Ih0MAlFNKAQAAVFvdERARcdJJJ8VJJ53U7z/rHuIGqiVLljTykJUhAGgHUwAAqDYv2SdEAJSXKQAUmx2CgLIRAYkQALSLKQCpKcI2oQBlIwISIABoFwEAAESIgNwJgPJzKRAAkBoRkCMBQDuZAgAAW4iAnAiAajAFAABSJAJyIABoN1MAAOCFRECbCQDaTQBQBEXYIcg2oUCZiIA2EgDV4lIgACBVIqBNBAB5MAUAAPojAtpAAFRPClMAAQAADEQEtJgAAAAgNSKghQRANZkCAACpEwEtIgDIiwAAAIYiAlpAAFRXClMAKCrbhAK0jwhoMgFAnkwBAIBaiIAmEgDVZgoAABSFCGgSAUDeTAEAgFqJgCYQAAAAFIkIGCYBQET+lwKZAgAA9RABw5BCAGz4w525PC7pEADQXnYIAspABDQolQBYddFncnls0iAAKJsibBMKUAYioAEpBcDoXffM5fF5Xl6XAgkAAKBRIqBOqQXAxLcuyuUcAAAoLhFQhxQDYETn2FzOg3yZAgAAwyECaiQA6E/euwIBADRCBNRIAJAKUwAAYLhEQI0EAABb2CYUKDoRUCMBwNbyuBTIFIAqsE0oQOuJgIQJAAAAWkEEJEoAsDVTAACgWURAggRA+uwKBAAUmQhIjACgP6YAAEAziYCECAD6IwAgTXYIAopMBCRCAAAA0C4iIAECgIGYAlBVtgkFaC0RkDMBUDztuilYAAAArSICciQAAADIgwjIiQBgMKYAAEAriYAcCAAGIwAAgFYTAW0mAADKwzahQFGJgDYSAAzFFAAAaAcR0CYCoDxatVAXANCXbUIBWkcEtIEAYCgCAABoJxHQYgKgnCzaAYAiEwEtJADKrVkhICgAgHYTAS0iAKphuAt4AQAA5EEEtIAAqJZGF/ICAADIiwhoMgFQTfUu6AUA1KYIOwR5rwCgiEblfQJlIgCqbeuFfXdHd03HAQC0mwhoEgHA1l642N8SBAIAAEiBCGgCAcBQLP4BgJS4J2CYBAAAAEUjAoYhhQDo2bC+7Y8JAECxiYAGpRIAj1y8sO2PC9BORdghCKBoREADUgqAjY/e1/bHBqAv24QCRSMC6pRaAEw65tS2Pz4AAMUmAuqQYgB0Tt6n7ecAAECxiYAaCQAAAMpCBNRIAAAAUBYioEYCACA/dggCaC4RUCMBAABAWYiAGgkAAAZjm1CgSERAogQAAACtIgISJAAAAGglEZAYAQAAQKuJgIQIAAAA2kEEJEIAAAzONqEAzSMCEiAAAABoJxGQMwEAAEC7iYAcCQCAcvFeAUBRiICcCAAAAPIiAnIgAAAAyJMIaDMBAABA3kRAGwkAgOGxTShAc4iANhEAAACkQgS0gQAAqA47BAFFIAJaTAAAAJAaEdBCAgAAgBSJgBYRAAAApEoEtIAAAGgdOwQBDJ8IaDIBAABA6kRAEwkAACLsEASkTwQ0iQAAAKAoREATCAAAAIpEBAyTAAAAoGgaioDFixfH1KlTY8yYMTFz5sy44YYbBjz2W9/6Vhx66KGx0047xU477RRz5swZ9PgiEQAAABRR3RFw0UUXxfz582PhwoVx8803x7Rp0+KII46IRx55pN/ju7u74x3veEdcc801sWzZspgyZUq88Y1vjIceemjYJ5+nVALgyesvzOVxAfJkm1CA4ak7As4+++x4//vfHyeeeGK86lWvinPOOSfGjh0b5513Xr/Hf//7348Pf/jDMX369Nh3333j3/7t36KnpyeWLl067JPPS0oBsPq/vpfLYwMAUFx1RcDGjRvjpptuijlz5jz/DUaMiDlz5sSyZctq+h7r16+PZ599NnbeeecBj9mwYUOsWbOmz0cqUguA8Ye+M5fHB2BwtgkFUlZXBDz22GOxadOmmDRpUp/PT5o0KVauXFnT9/j0pz8dkydP7hMSWzvttNNi/PjxvR9Tpkyp5zRbJsUA2PHgt+dyDgAAFFdbdwc6/fTT48ILL4xLL700xowZM+BxCxYsiNWrV/d+PPDAA208y/4JAAAAymJUPQdPmDAhRo4cGatWrerz+VWrVsVuu+026NeeddZZcfrpp8fPfvaz2H///Qc9trOzMzo7O+s5tZYTAAAAlEVdk4DRo0fHAQcc0Oem3i03+c6aNWvArzvzzDPj1FNPjSuvvDIOPPDAxs82RwIAAICyqPtyoPnz58e3vvWt+M53vhMrVqyID33oQ7Fu3bo48cQTIyLi+OOPjwULFvQef8YZZ8RnPvOZOO+882Lq1KmxcuXKWLlyZaxdu7Z5v4s2EAAAabFNKEDj6rocKCLimGOOiUcffTROOeWUWLlyZUyfPj2uvPLK3puF77///hgx4vm2+Nd//dfYuHFjvOUtb+nzfRYuXBif/exnh3f2bSQAAAAoi7ojICLipJNOipNOOqnff9a91ZZo9957byMPQQgAAABao627A1E7AQAAQKuIgAQJAIBy8IZhQKpEQGIEAEDt3BwM0BgRkBABAABAO4iARAgAAADaRQQkQAAAANBOIiBnAgAAgHYTATkSAADD5+ZggPqJgJwIAAAA8iICciAAAADIkwhoMwEAUC3eMAxIkQhoIwEAAEAKRECbCACA1nFzMEB9REAbCAAAAFIiAlpMAAAAkBoR0EICAACAFImAFhEAAACkSgS0gAAAACBlIqDJBABAPuwQBFA7EdBEAgAAgCIQAU0iAAAYiHcNBlIjAppAAAAAUCQiYJgEAAAARSMChiGFANjwhztzeVwAAIpLBDQolQBYddFncnlsgBTZIQigNiKgASkFwOhd98zl8QEAKC4RUKfUAmDiWxflcg4AABSXCKhDigEwonNsLucBAEBxiYAaCQAAAMpCBNRIAAAwHN4wDEiJCKiRAAAAoCxEQI0EAEAx2CYUYGgiIGECAACAVhABiRIAAAC0ighIkAAAAKCVREBiBAAAAK0mAhIiAAAAaIdReZ9AUdx7+lFteJSjIr47v66vuPnmm+OAM1p0OgAAlJJJAAClk+o2od4wDEiFCAAAgIoRAQAAUDEiAAAAKkYEAABAxYgAAACoGBEAQCmlukMQQApEAAC0kW1CgRSIAAAAqBgRAEBpuSQIoH8iAAAAKkYEAABAxYgAAErNJUEA2xIBAABQMSIAANrMNqFA3kQAAABUjAgAoPTcFwDQlwgAAICKEQEAAFAxIgAAACpGBABQCe4LAHieCAAAgIoRAQAAUDEiAAAAKkYEAABAxYgAACrDzcEAm4kAAACoGBEAAAAVIwIAAKBiRAAAAFSMCACgUtwcDCACAACgckQAAABUjAgAAICKEQEAAFAxIgAAACpGBABQOXYIAqpOBAAAQMWIAAAAqBgRAAAAFSMCAKgk9wUAVSYCAKisvEJAgAB5EwEAVJoFOVBFIgAA2kh0ACkQAQBUXrsW5gIASIUIAIA2EABASkQAAERrF+kCAEiNCACAFhIAQIoaioDFixfH1KlTY8yYMTFz5sy44YYbBj3+4osvjn333TfGjBkT++23X1xxxRUNnSwAtFKzF+wCAEhV3RFw0UUXxfz582PhwoVx8803x7Rp0+KII46IRx55pN/jr7/++njHO94R733ve+OWW26Jo48+Oo4++ui49dZbh33yAJCirKtLAABJ68iyLKvnC2bOnBmve93r4utf/3pERPT09MSUKVPiox/9aJx88snbHH/MMcfEunXr4j//8z97P/fnf/7nMX369DjnnHP6fYwNGzbEhg0ben+9evXqeMlLXhIPPPBAjBs3rp7TLb3ly5fHYYcdFt/85jdjn332yft0AErhsFtuqev4X8yY0aIzASIi7rzzzvjABz4Qv/jFL2L69Ol5n07S1qxZE1OmTIknn3wyxo8fP/CBWR02bNiQjRw5Mrv00kv7fP7444/P/uqv/qrfr5kyZUr2pS99qc/nTjnllGz//fcf8HEWLlyYRYQPHz58+PDhw4cPHz4a+HjggQcGXdePijo89thjsWnTppg0aVKfz0+aNCnuuOOOfr9m5cqV/R6/cuXKAR9nwYIFMX/+/N5f9/T0xBNPPBG77LJLdHR01HPKpbel9q644op40YtelPfpVMq6deviyCOP9NznwHOfL89/fjz3+fL852fLc++qkKFlWRZPPfVUTJ48edDj6oqAduns7IzOzs4+n9txxx3zOZmCOOSQQ/ylaLM1a9ZEhOc+D577fHn+8+O5z5fnPz9bnvtx48Z57msw6GVAf1LXjcETJkyIkSNHxqpVq/p8ftWqVbHbbrv1+zW77bZbXccDAACtVVcEjB49Og444IBYunRp7+d6enpi6dKlMWvWrH6/ZtasWX2Oj4i4+uqrBzweAABorbovB5o/f36ccMIJceCBB8ZBBx0UX/7yl2PdunVx4oknRkTE8ccfH3vssUecdtppERHxsY99LA477LD44he/GEcddVRceOGFceONN8Y3v/nN5v5OKqqzszMWLly4zeVTtJ7nPj+e+3x5/vPjuc+X5z8/nvvmq3uL0IiIr3/96/GFL3whVq5cGdOnT4+vfvWrMXPmzIiI6OrqiqlTp8aSJUt6j7/44ovjn/7pn+Lee++Nl7/85XHmmWfGkUce2bTfBAAAULuGIgAAACiuut8xGAAAKDYRAAAAFSMCAACgYkQAAABUjAhIzOLFi2Pq1KkxZsyYmDlzZtxwww2DHn/xxRfHvvvuG2PGjIn99tsvrrjiij7/PMuyOOWUU2L33XeP7bffPubMmRO/+93vWvlbKLR6nv9vfetbceihh8ZOO+0UO+20U8yZM2eb49/97ndHR0dHn483velNrf5tFFI9z/2SJUu2eV7HjBnT5xg/+7Wr57nv6ura5rnv6OiIo446qvcYP/e1ufbaa2PevHkxefLk6OjoiMsuu2zIr+nu7o7Xvva10dnZGS972cv67MS3Rb3/Hamqep//Sy65JA4//PDYddddY9y4cTFr1qy46qqr+hzz2c9+dpuf/X333beFv4tiqve57+7u7vffOytXruxznJ/9+oiAhFx00UUxf/78WLhwYdx8880xbdq0OOKII+KRRx7p9/jrr78+3vGOd8R73/veuOWWW+Loo4+Oo48+Om699dbeY84888z46le/Guecc078z//8T7zoRS+KI444Ip555pl2/bYKo97nv7u7O97xjnfENddcE8uWLYspU6bEG9/4xnjooYf6HPemN70pHn744d6PCy64oB2/nUKp97mP2PzW8S98Xu+7774+/9zPfm3qfe4vueSSPs/7rbfeGiNHjoy3vvWtfY7zcz+0devWxbRp02Lx4sU1HX/PPffEUUcdFbNnz47ly5fHxz/+8Xjf+97XZyHayN+lqqr3+b/22mvj8MMPjyuuuCJuuummmD17dsybNy9uueWWPse9+tWv7vOz/8tf/rIVp19o9T73W9x55519ntuJEyf2/jM/+w3ISMZBBx2UfeQjH+n99aZNm7LJkydnp512Wr/Hv+1tb8uOOuqoPp+bOXNm9jd/8zdZlmVZT09Ptttuu2Vf+MIXev/5k08+mXV2dmYXXHBBC34HxVbv87+15557Lnvxi1+cfec73+n93AknnJC9+c1vbvaplk69z/23v/3tbPz48QN+Pz/7tRvuz/2XvvSl7MUvfnG2du3a3s/5ua9fRGSXXnrpoMd86lOfyl796lf3+dwxxxyTHXHEEb2/Hu6fZ1XV8vz351WvelW2aNGi3l8vXLgwmzZtWvNOrAJqee6vueaaLCKyP/7xjwMe42e/fiYBidi4cWPcdNNNMWfOnN7PjRgxIubMmRPLli3r92uWLVvW5/iIiCOOOKL3+HvuuSdWrlzZ55jx48fHzJkzB/yeVdXI87+19evXx7PPPhs777xzn893d3fHxIkTY5999okPfehD8fjjjzf13Iuu0ed+7dq1seeee8aUKVPizW9+c9x22229/8zPfm2a8XN/7rnnxtvf/vZ40Yte1Ofzfu6bb6h/5zfjz5Pa9fT0xFNPPbXNv/N/97vfxeTJk2PvvfeO4447Lu6///6czrB8pk+fHrvvvnscfvjhcd111/V+3s9+Y0RAIh577LHYtGlTTJo0qc/nJ02atM01b1usXLly0OO3/G8937OqGnn+t/bpT386Jk+e3OdfQm9605viu9/9bixdujTOOOOM+MUvfhFz586NTZs2NfX8i6yR536fffaJ8847L370ox/F9773vejp6YmDDz44HnzwwYjws1+r4f7c33DDDXHrrbfG+973vj6f93PfGgP9O3/NmjXx9NNPN+XfY9TurLPOirVr18bb3va23s/NnDkzlixZEldeeWX867/+a9xzzz1x6KGHxlNPPZXjmRbf7rvvHuecc078x3/8R/zHf/xHTJkyJbq6uuLmm2+OiOb8N7yKRuV9AlAGp59+elx44YXR3d3d5wbVt7/97b3/f7/99ov9998/XvrSl0Z3d3e84Q1vyONUS2HWrFkxa9as3l8ffPDB8cpXvjL+z//5P3HqqafmeGbVcu6558Z+++0XBx10UJ/P+7mn7M4///xYtGhR/OhHP+pzXfrcuXN7///+++8fM2fOjD333DN+8IMfxHvf+948TrUU9tlnn9hnn316f33wwQfH73//+/jSl74U//7v/57jmRWbSUAiJkyYECNHjoxVq1b1+fyqVatit9126/drdtttt0GP3/K/9XzPqmrk+d/irLPOitNPPz1++tOfxv777z/osXvvvXdMmDAh7rrrrmGfc1kM57nfYrvttosZM2b0Pq9+9msznOd+3bp1ceGFF9a0sPFz3xwD/Tt/3Lhxsf322zfl7xJDu/DCC+N973tf/OAHP9jm8qyt7bjjjvGKV7zCz34LHHTQQb3Pq5/9xoiARIwePToOOOCAWLp0ae/nenp6YunSpX1e8XyhWbNm9Tk+IuLqq6/uPX6vvfaK3Xbbrc8xa9asif/5n/8Z8HtWVSPPf8TmHWhOPfXUuPLKK+PAAw8c8nEefPDBePzxx2P33XdvynmXQaPP/Qtt2rQpfvOb3/Q+r372azOc5/7iiy+ODRs2xDvf+c4hH8fPfXMM9e/8ZvxdYnAXXHBBnHjiiXHBBRf02RZ3IGvXro3f//73fvZbYPny5b3Pq5/9BuV9ZzLPu/DCC7POzs5syZIl2e2335594AMfyHbcccds5cqVWZZl2bve9a7s5JNP7j3+uuuuy0aNGpWdddZZ2YoVK7KFCxdm2223Xfab3/ym95jTTz8923HHHbMf/ehH2a9//evszW9+c7bXXntlTz/9dNt/f6mr9/k//fTTs9GjR2c//OEPs4cffrj346mnnsqyLMueeuqp7O///u+zZcuWZffcc0/2s5/9LHvta1+bvfzlL8+eeeaZXH6Pqar3uV+0aFF21VVXZb///e+zm266KXv729+ejRkzJrvtttt6j/GzX5t6n/st/uIv/iI75phjtvm8n/vaPfXUU9ktt9yS3XLLLVlEZGeffXZ2yy23ZPfdd1+WZVl28sknZ+9617t6j7/77ruzsWPHZp/85CezFStWZIsXL85GjhyZXXnllb3HDPXnyfPqff6///3vZ6NGjcoWL17c59/5Tz75ZO8xf/d3f5d1d3dn99xzT3bddddlc+bMySZMmJA98sgjbf/9paze5/5LX/pSdtlll2W/+93vst/85jfZxz72sWzEiBHZz372s95j/OzXTwQk5mtf+1r2kpe8JBs9enR20EEHZf/93//d+88OO+yw7IQTTuhz/A9+8IPsFa94RTZ69Ojs1a9+dXb55Zf3+ec9PT3ZZz7zmWzSpElZZ2dn9oY3vCG788472/FbKaR6nv8999wzi4htPhYuXJhlWZatX78+e+Mb35jtuuuu2XbbbZftueee2fvf/37/QhpAPc/9xz/+8d5jJ02alB155JHZzTff3Of7+dmvXb3/3rnjjjuyiMh++tOfbvO9/NzXbsu2h1t/bHm+TzjhhOywww7b5mumT5+ejR49Ott7772zb3/729t838H+PHlevc//YYcdNujxWbZ5y9bdd989Gz16dLbHHntkxxxzTHbXXXe19zdWAPU+92eccUb20pe+NBszZky28847Z11dXdnPf/7zbb6vn/36dGRZlrVl5AAAACTBPQEAAFAxIgAAACpGBAAAQMWIAAAAqBgRAAAAFSMCAACgYkQAAABUjAgAAICKEQEAAFAxIgAAACpGBAAAQMX8/3oj+0oFGuqOAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 750x900 with 1 Axes>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "reward = 0.0\n",
    "# while reward == 0.0:\n",
    "obs, info = train_env.reset()\n",
    "dones = False\n",
    "while not dones:\n",
    "    action = train_env.action_space.sample()\n",
    "    obs, reward, dones, truncated, info = train_env.step(action)\n",
    "\n",
    "train_env.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6edcde2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 750x900 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# It will check your custom environment and output additional warnings if needed\n",
    "check_env(sogym(mode='train',observation_type='topopt_game'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10de06f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Profiling the environment\n",
    "# # Specify the number of episodes to run\n",
    "num_episodes = 20\n",
    "result_df = profile_and_analyze(num_episodes, train_env)\n",
    "result_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7820005a-be57-4236-b66b-0b34ed558aff",
   "metadata": {},
   "source": [
    "---\n",
    "### Multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3e210c1-817a-42a8-ab28-2ea437ed2c4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 2 CPUs!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 950x650 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 950x650 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set number of CPUs to use automatically\n",
    "num_cpu = multiprocessing.cpu_count()\n",
    "num_cpu = 2\n",
    "print(f\"Using {num_cpu} CPUs!\")\n",
    "\n",
    "algorithm_name = \"PPO\"  # or \"TD3\"\n",
    "\n",
    "# Load the YAML file and extract parameters\n",
    "with open(\"algorithms.yaml\", \"r\") as file:\n",
    "    config = yaml.safe_load(file)\n",
    "\n",
    "common_params = config.get(\"common\", {})\n",
    "algorithm_params = config.get(algorithm_name, {})\n",
    "\n",
    "# Define other parameters\n",
    "params = {\n",
    "    'observation_type': \"topopt_game\",\n",
    "    'vol_constraint_type': \"hard\",\n",
    "    'use_std_strain': False,\n",
    "    'check_connectivity': True,\n",
    "    'resolution': 50\n",
    "}\n",
    "\n",
    "current_datetime = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "train_critic_only = False  # if True, we freeze everything except the critic\n",
    "pretrained_run = None\n",
    "restart_run = None  # or \"PPO_20240516_093938\"\n",
    "\n",
    "log_name = restart_run if restart_run else f\"{algorithm_name}_{current_datetime}\"\n",
    "\n",
    "# Create directory and config file if needed\n",
    "log_dir = f'./runs/{log_name}'\n",
    "if not os.path.exists(log_dir):\n",
    "    os.makedirs(log_dir)\n",
    "    # Combine all parameters for saving to the config file\n",
    "    save_params = {**algorithm_params, **params, 'algorithm_name': algorithm_name, **common_params}\n",
    "    with open(f'{log_dir}/config.yaml', 'w') as file:\n",
    "        yaml.dump(save_params, file)\n",
    "\n",
    "# Create training and evaluation environments\n",
    "train_env = sogym(mode='train', **params)\n",
    "env = make_vec_env(lambda: train_env, n_envs=num_cpu, vec_env_cls=SubprocVecEnv)\n",
    "env = VecCheckNan(env, raise_exception=True)\n",
    "\n",
    "eval_env = sogym(mode='test', **params)\n",
    "eval_env = make_vec_env(lambda: eval_env, n_envs=1, vec_env_cls=SubprocVecEnv)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "826b6f2e-5ceb-447f-ac10-a52f0379cb31",
   "metadata": {},
   "source": [
    "--- \n",
    "### Defining the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "8a1132db-34f4-4a72-bb12-1339a9e00a53",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thomas/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/ppo/ppo.py:155: UserWarning: You have specified a mini-batch size of 4096, but because the `RolloutBuffer` is of size `n_steps * n_envs = 246`, after every 0 untruncated mini-batches, there will be a truncated mini-batch of size 246\n",
      "We recommend using a `batch_size` that is a factor of `n_steps * n_envs`.\n",
      "Info: (n_steps=123 and n_envs=2)\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'runs/PPO_20240516_110357/checkpoints/best_model.zip.zip'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[76], line 54\u001b[0m\n\u001b[1;32m     51\u001b[0m     model\u001b[38;5;241m.\u001b[39mset_parameters(pretrained_checkpoint)\n\u001b[1;32m     53\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m restart_run:\n\u001b[0;32m---> 54\u001b[0m     model \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./runs/\u001b[39;49m\u001b[38;5;132;43;01m{}\u001b[39;49;00m\u001b[38;5;124;43m/checkpoints/best_model.zip\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlog_name\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menv\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/base_class.py:677\u001b[0m, in \u001b[0;36mBaseAlgorithm.load\u001b[0;34m(cls, path, env, device, custom_objects, print_system_info, force_reset, **kwargs)\u001b[0m\n\u001b[1;32m    674\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m== CURRENT SYSTEM INFO ==\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    675\u001b[0m     get_system_info()\n\u001b[0;32m--> 677\u001b[0m data, params, pytorch_variables \u001b[38;5;241m=\u001b[39m \u001b[43mload_from_zip_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    678\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    679\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    680\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcustom_objects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcustom_objects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    681\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprint_system_info\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprint_system_info\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    682\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    684\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m data \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo data found in the saved file\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    685\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m params \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo params found in the saved file\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/save_util.py:402\u001b[0m, in \u001b[0;36mload_from_zip_file\u001b[0;34m(load_path, load_data, custom_objects, device, verbose, print_system_info)\u001b[0m\n\u001b[1;32m    375\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload_from_zip_file\u001b[39m(\n\u001b[1;32m    376\u001b[0m     load_path: Union[\u001b[38;5;28mstr\u001b[39m, pathlib\u001b[38;5;241m.\u001b[39mPath, io\u001b[38;5;241m.\u001b[39mBufferedIOBase],\n\u001b[1;32m    377\u001b[0m     load_data: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    381\u001b[0m     print_system_info: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    382\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Any]], TensorDict, Optional[TensorDict]]:\n\u001b[1;32m    383\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    384\u001b[0m \u001b[38;5;124;03m    Load model data from a .zip archive\u001b[39;00m\n\u001b[1;32m    385\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    400\u001b[0m \u001b[38;5;124;03m        and dict of pytorch variables\u001b[39;00m\n\u001b[1;32m    401\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 402\u001b[0m     file \u001b[38;5;241m=\u001b[39m \u001b[43mopen_path\u001b[49m\u001b[43m(\u001b[49m\u001b[43mload_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msuffix\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzip\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    404\u001b[0m     \u001b[38;5;66;03m# set device to cpu if cuda is not available\u001b[39;00m\n\u001b[1;32m    405\u001b[0m     device \u001b[38;5;241m=\u001b[39m get_device(device\u001b[38;5;241m=\u001b[39mdevice)\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/functools.py:889\u001b[0m, in \u001b[0;36msingledispatch.<locals>.wrapper\u001b[0;34m(*args, **kw)\u001b[0m\n\u001b[1;32m    885\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m args:\n\u001b[1;32m    886\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfuncname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m requires at least \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    887\u001b[0m                     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m1 positional argument\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m--> 889\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdispatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__class__\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/save_util.py:239\u001b[0m, in \u001b[0;36mopen_path_str\u001b[0;34m(path, mode, verbose, suffix)\u001b[0m\n\u001b[1;32m    224\u001b[0m \u001b[38;5;129m@open_path\u001b[39m\u001b[38;5;241m.\u001b[39mregister(\u001b[38;5;28mstr\u001b[39m)\n\u001b[1;32m    225\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mopen_path_str\u001b[39m(path: \u001b[38;5;28mstr\u001b[39m, mode: \u001b[38;5;28mstr\u001b[39m, verbose: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m, suffix: Optional[\u001b[38;5;28mstr\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m io\u001b[38;5;241m.\u001b[39mBufferedIOBase:\n\u001b[1;32m    226\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    227\u001b[0m \u001b[38;5;124;03m    Open a path given by a string. If writing to the path, the function ensures\u001b[39;00m\n\u001b[1;32m    228\u001b[0m \u001b[38;5;124;03m    that the path exists.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    237\u001b[0m \u001b[38;5;124;03m    :return:\u001b[39;00m\n\u001b[1;32m    238\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 239\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopen_path_pathlib\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpathlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mPath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msuffix\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/save_util.py:290\u001b[0m, in \u001b[0;36mopen_path_pathlib\u001b[0;34m(path, mode, verbose, suffix)\u001b[0m\n\u001b[1;32m    284\u001b[0m         path\u001b[38;5;241m.\u001b[39mparent\u001b[38;5;241m.\u001b[39mmkdir(exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, parents\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m    286\u001b[0m \u001b[38;5;66;03m# if opening was successful uses the open_path() function\u001b[39;00m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;66;03m# if opening failed with IsADirectory|FileNotFound, calls open_path_pathlib\u001b[39;00m\n\u001b[1;32m    288\u001b[0m \u001b[38;5;66;03m#   with corrections\u001b[39;00m\n\u001b[1;32m    289\u001b[0m \u001b[38;5;66;03m# if reading failed with FileNotFoundError, calls open_path_pathlib with suffix\u001b[39;00m\n\u001b[0;32m--> 290\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopen_path_pathlib\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msuffix\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/save_util.py:271\u001b[0m, in \u001b[0;36mopen_path_pathlib\u001b[0;34m(path, mode, verbose, suffix)\u001b[0m\n\u001b[1;32m    269\u001b[0m             path, suffix \u001b[38;5;241m=\u001b[39m newpath, \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    270\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 271\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m error\n\u001b[1;32m    272\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    273\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/save_util.py:263\u001b[0m, in \u001b[0;36mopen_path_pathlib\u001b[0;34m(path, mode, verbose, suffix)\u001b[0m\n\u001b[1;32m    261\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    262\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 263\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m open_path(\u001b[43mpath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m, mode, verbose, suffix)\n\u001b[1;32m    264\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m error:\n\u001b[1;32m    265\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m suffix \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m suffix \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/pathlib.py:1119\u001b[0m, in \u001b[0;36mPath.open\u001b[0;34m(self, mode, buffering, encoding, errors, newline)\u001b[0m\n\u001b[1;32m   1117\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode:\n\u001b[1;32m   1118\u001b[0m     encoding \u001b[38;5;241m=\u001b[39m io\u001b[38;5;241m.\u001b[39mtext_encoding(encoding)\n\u001b[0;32m-> 1119\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_accessor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffering\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1120\u001b[0m \u001b[43m                           \u001b[49m\u001b[43mnewline\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'runs/PPO_20240516_110357/checkpoints/best_model.zip.zip'"
     ]
    }
   ],
   "source": [
    "# The noise objects for TD3\n",
    "n_actions = env.action_space.shape[-1]\n",
    "action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.5 * np.ones(n_actions))\n",
    "\n",
    "chosen_policy = \"MlpPolicy\" if observation_type == 'box_dense' else \"MultiInputPolicy\"\n",
    "feature_extractor = ImageDictExtractor if observation_type == 'image' or observation_type==\"topopt_game\" else CustomBoxDense\n",
    "\n",
    "\n",
    "policy_kwargs = dict(\n",
    "    features_extractor_class=feature_extractor,\n",
    "    net_arch = config['common']['net_arch'],\n",
    "    share_features_extractor = False,\n",
    ")\n",
    "\n",
    "# Create the model based on the algorithm name and parameters\n",
    "if algorithm_name == \"SAC\":\n",
    "    model = SAC(env=env,\n",
    "                policy = chosen_policy, \n",
    "                policy_kwargs=policy_kwargs,\n",
    "                #action_noise = action_noise,\n",
    "                ent_coef = 0.0,\n",
    "                device=device, \n",
    "                **algorithm_params)\n",
    "\n",
    "elif algorithm_name == \"PPO\":\n",
    "    model = PPO(env=env, \n",
    "                policy = chosen_policy, \n",
    "                policy_kwargs=policy_kwargs,\n",
    "                n_steps= 64*386 // num_cpu//100,\n",
    "                batch_size= 16384//4,\n",
    "                tensorboard_log  ='./runs/{}'.format(log_name),\n",
    "                device = device, \n",
    "                **algorithm_params)\n",
    "\n",
    "elif algorithm_name == \"TD3\":\n",
    "    # Create the action noise object\n",
    "    n_actions = env.action_space.shape[-1]\n",
    "    action_noise_params = algorithm_params.pop(\"action_noise\")\n",
    "    action_noise = NormalActionNoise(mean=action_noise_params[\"mean\"] * np.ones(n_actions),\n",
    "                                     sigma=action_noise_params[\"sigma\"] * np.ones(n_actions))\n",
    "    model = TD3(env=env,\n",
    "                policy =chosen_policy, \n",
    "                policy_kwargs=policy_kwargs,\n",
    "                action_noise=action_noise,\n",
    "                device=device, \n",
    "                **algorithm_params)\n",
    "\n",
    "if load_from_pretrained and pretrained_checkpoint is not None:\n",
    "    model.set_parameters(pretrained_checkpoint)\n",
    "\n",
    "if restart_run:\n",
    "    model = model.load(\"./runs/{}/checkpoints/best_model.zip\".format(log_name),env=env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "3ec8bc79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a checkpoint every 1000 steps\n",
    "checkpoint_callback = CheckpointCallback(\n",
    "  save_freq=250_000//num_cpu,\n",
    "  save_path=\"./runs/{}/checkpoints/\".format(log_name),\n",
    "  name_prefix=log_name,\n",
    "  save_replay_buffer=True,\n",
    "  save_vecnormalize=True,\n",
    ")\n",
    "\n",
    "eval_callback = EvalCallback(eval_env,\n",
    "                             log_path='./runs/{}/'.format(log_name), \n",
    "                             eval_freq=10_000//num_cpu,\n",
    "                             deterministic=True,\n",
    "                             n_eval_episodes=10,\n",
    "                             render=False,\n",
    "                             best_model_save_path='./runs/{}/checkpoints/'.format(log_name),\n",
    "                             verbose=0)\n",
    "\n",
    "callback_list = CallbackList([eval_callback,\n",
    "                         checkpoint_callback,\n",
    "                         MaxRewardCallback(verbose=1),\n",
    "                         GradientClippingCallback(clip_value=1.0, verbose=1),\n",
    "                         GradientNormCallback(verbose=1),\n",
    "                         FigureRecorderCallback(eval_env=eval_env, check_freq=10_000//num_cpu, figure_size=(8, 6))\n",
    "                         ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20f70cf1-e15e-4fed-ab75-12bdca996f2d",
   "metadata": {},
   "source": [
    "--- \n",
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "978d7768-b7bb-4c14-b620-28a278392b6b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "def init_weights(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.xavier_uniform_(m.weight)\n",
    "        if m.bias is not None:\n",
    "            nn.init.zeros_(m.bias)\n",
    "\n",
    "if train_critic_only:\n",
    "    #Freeze everything:\n",
    "    for name, param in model.policy.named_parameters():\n",
    "        if param.requires_grad:\n",
    "            param.requires_grad=False\n",
    "\n",
    "    if algorithm_name =='SAC':\n",
    "        # Unfreeze critic:\n",
    "        for param in model.policy.critic.parameters():\n",
    "            if param.requires_grad==False:\n",
    "                param.requires_grad=True\n",
    "\n",
    "        for param in model.policy.critic_target.parameters():\n",
    "            if param.requires_grad==False:\n",
    "                param.requires_grad=True\n",
    "\n",
    "\n",
    "    if algorithm_name == 'PPO':\n",
    "        for param in model.policy.mlp_extractor.value_net.parameters():\n",
    "            if param.requires_grad==False:\n",
    "                param.requires_grad=True\n",
    "            \n",
    "        for param in model.policy.value_net.parameters():\n",
    "            if param.requires_grad==False:\n",
    "                param.requires_grad=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "c876f635",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thomas/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/callbacks.py:414: UserWarning: Training and eval env are not of the same type<stable_baselines3.common.vec_env.vec_check_nan.VecCheckNan object at 0x7fca48ac2290> != <stable_baselines3.common.vec_env.subproc_vec_env.SubprocVecEnv object at 0x7fca511b7640>\n",
      "  warnings.warn(\"Training and eval env are not of the same type\" f\"{self.training_env} != {self.eval_env}\")\n",
      "/home/thomas/anaconda3/envs/SB3_update/lib/python3.10/site-packages/gymnasium/core.py:311: UserWarning: \u001b[33mWARN: env.reward to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.reward` for environment variables or `env.get_wrapper_attr('reward')` that will search the reminding wrappers.\u001b[0m\n",
      "  logger.warn(\n",
      "/home/thomas/anaconda3/envs/SB3_update/lib/python3.10/site-packages/gymnasium/core.py:311: UserWarning: \u001b[33mWARN: env.reward to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.reward` for environment variables or `env.get_wrapper_attr('reward')` that will search the reminding wrappers.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[74], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m25_000_000\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[43m           \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m           \u001b[49m\u001b[43mtb_log_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m           \u001b[49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrestart\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m           \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# save the model:\u001b[39;00m\n\u001b[1;32m      8\u001b[0m model\u001b[38;5;241m.\u001b[39msave(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./runs/\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m/checkpoints/final_model\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/ppo/ppo.py:315\u001b[0m, in \u001b[0;36mPPO.learn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    306\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlearn\u001b[39m(\n\u001b[1;32m    307\u001b[0m     \u001b[38;5;28mself\u001b[39m: SelfPPO,\n\u001b[1;32m    308\u001b[0m     total_timesteps: \u001b[38;5;28mint\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    313\u001b[0m     progress_bar: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    314\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SelfPPO:\n\u001b[0;32m--> 315\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    316\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtotal_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    317\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    318\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlog_interval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_interval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    319\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtb_log_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtb_log_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    320\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    321\u001b[0m \u001b[43m        \u001b[49m\u001b[43mprogress_bar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprogress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    322\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/on_policy_algorithm.py:277\u001b[0m, in \u001b[0;36mOnPolicyAlgorithm.learn\u001b[0;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps, progress_bar)\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    276\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps \u001b[38;5;241m<\u001b[39m total_timesteps:\n\u001b[0;32m--> 277\u001b[0m     continue_training \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect_rollouts\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrollout_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_rollout_steps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mn_steps\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    279\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m continue_training:\n\u001b[1;32m    280\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/on_policy_algorithm.py:194\u001b[0m, in \u001b[0;36mOnPolicyAlgorithm.collect_rollouts\u001b[0;34m(self, env, callback, rollout_buffer, n_rollout_steps)\u001b[0m\n\u001b[1;32m    189\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    190\u001b[0m         \u001b[38;5;66;03m# Otherwise, clip the actions to avoid out of bound error\u001b[39;00m\n\u001b[1;32m    191\u001b[0m         \u001b[38;5;66;03m# as we are sampling from an unbounded Gaussian distribution\u001b[39;00m\n\u001b[1;32m    192\u001b[0m         clipped_actions \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mclip(actions, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maction_space\u001b[38;5;241m.\u001b[39mlow, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maction_space\u001b[38;5;241m.\u001b[39mhigh)\n\u001b[0;32m--> 194\u001b[0m new_obs, rewards, dones, infos \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclipped_actions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    196\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m env\u001b[38;5;241m.\u001b[39mnum_envs\n\u001b[1;32m    198\u001b[0m \u001b[38;5;66;03m# Give access to local variables\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/vec_env/base_vec_env.py:206\u001b[0m, in \u001b[0;36mVecEnv.step\u001b[0;34m(self, actions)\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    200\u001b[0m \u001b[38;5;124;03mStep the environments with the given action\u001b[39;00m\n\u001b[1;32m    201\u001b[0m \n\u001b[1;32m    202\u001b[0m \u001b[38;5;124;03m:param actions: the action\u001b[39;00m\n\u001b[1;32m    203\u001b[0m \u001b[38;5;124;03m:return: observation, reward, done, information\u001b[39;00m\n\u001b[1;32m    204\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    205\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstep_async(actions)\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_wait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/vec_env/vec_check_nan.py:40\u001b[0m, in \u001b[0;36mVecCheckNan.step_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m VecEnvStepReturn:\n\u001b[0;32m---> 40\u001b[0m     observations, rewards, dones, infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvenv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_wait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     41\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_val(event\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep_wait\u001b[39m\u001b[38;5;124m\"\u001b[39m, observations\u001b[38;5;241m=\u001b[39mobservations, rewards\u001b[38;5;241m=\u001b[39mrewards, dones\u001b[38;5;241m=\u001b[39mdones)\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_observations \u001b[38;5;241m=\u001b[39m observations\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/vec_env/subproc_vec_env.py:129\u001b[0m, in \u001b[0;36mSubprocVecEnv.step_wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m VecEnvStepReturn:\n\u001b[0;32m--> 129\u001b[0m     results \u001b[38;5;241m=\u001b[39m [remote\u001b[38;5;241m.\u001b[39mrecv() \u001b[38;5;28;01mfor\u001b[39;00m remote \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mremotes]\n\u001b[1;32m    130\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwaiting \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m    131\u001b[0m     obs, rews, dones, infos, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreset_infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mresults)  \u001b[38;5;66;03m# type: ignore[assignment]\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/vec_env/subproc_vec_env.py:129\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m VecEnvStepReturn:\n\u001b[0;32m--> 129\u001b[0m     results \u001b[38;5;241m=\u001b[39m [\u001b[43mremote\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m remote \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mremotes]\n\u001b[1;32m    130\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwaiting \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m    131\u001b[0m     obs, rews, dones, infos, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreset_infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mresults)  \u001b[38;5;66;03m# type: ignore[assignment]\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/multiprocessing/connection.py:255\u001b[0m, in \u001b[0;36m_ConnectionBase.recv\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    253\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_closed()\n\u001b[1;32m    254\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_readable()\n\u001b[0;32m--> 255\u001b[0m buf \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_recv_bytes\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    256\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _ForkingPickler\u001b[38;5;241m.\u001b[39mloads(buf\u001b[38;5;241m.\u001b[39mgetbuffer())\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/multiprocessing/connection.py:419\u001b[0m, in \u001b[0;36mConnection._recv_bytes\u001b[0;34m(self, maxsize)\u001b[0m\n\u001b[1;32m    418\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_recv_bytes\u001b[39m(\u001b[38;5;28mself\u001b[39m, maxsize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m--> 419\u001b[0m     buf \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_recv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m4\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    420\u001b[0m     size, \u001b[38;5;241m=\u001b[39m struct\u001b[38;5;241m.\u001b[39munpack(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m!i\u001b[39m\u001b[38;5;124m\"\u001b[39m, buf\u001b[38;5;241m.\u001b[39mgetvalue())\n\u001b[1;32m    421\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m size \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:\n",
      "File \u001b[0;32m~/anaconda3/envs/SB3_update/lib/python3.10/multiprocessing/connection.py:384\u001b[0m, in \u001b[0;36mConnection._recv\u001b[0;34m(self, size, read)\u001b[0m\n\u001b[1;32m    382\u001b[0m remaining \u001b[38;5;241m=\u001b[39m size\n\u001b[1;32m    383\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m remaining \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 384\u001b[0m     chunk \u001b[38;5;241m=\u001b[39m \u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mremaining\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    385\u001b[0m     n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(chunk)\n\u001b[1;32m    386\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m n \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "model.learn(25_000_000,\n",
    "           callback=callback_list, \n",
    "           tb_log_name=log_name,\n",
    "           reset_num_timesteps=not restart\n",
    "           )\n",
    "\n",
    "# save the model:\n",
    "model.save('./runs/{}/checkpoints/final_model')\n",
    "if algorithm_name != 'PPO':\n",
    "    model.save_replay_buffer(\"./runs/{}/checkpoints/final_buffer\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2827c8cc-028d-4448-b109-d6542816df2a",
   "metadata": {},
   "source": [
    "---\n",
    "### Let's visualize the agent's performance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6728408e-71c2-4f7b-a8f1-f0cc6b43dcff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sogym.env.sogym at 0x7f98b10dbc40>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "viz_env=sogym(mode='test',observation_type='topopt_game',vol_constraint_type='hard' ,resolution = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5cbb153d-80e3-4d6c-ada3-f4d323360b74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.75501883 -1.          0.32291675  0.64145327  0.9568924   0.9920023 ]\n",
      "[-0.903395    0.6623455  -0.42170578 -0.5872051   0.7409822   0.8139564 ]\n",
      "[-0.94604313 -0.02086864  0.11097331  1.          0.8015071   0.8562474 ]\n",
      "[-1.          1.          0.08024888 -0.32997712  1.          1.        ]\n",
      "[ 0.07595539 -0.7214547   0.4731991   0.3694167   0.37574175  0.34123728]\n",
      "[-0.37390336 -0.01163775  0.40241554 -0.27844983  0.45322505  0.43657506]\n",
      "[-0.16364224 -0.55042976  0.31846595  1.          0.4676532   0.45424548]\n",
      "[-0.12401124  0.69860417  0.35660243 -0.06428144  0.7142122   0.7212964 ]\n",
      "Desired volume: 0.41 Obtained volume: 0.30805051633717917\n",
      "Env reward: 0.1713260659137914\n"
     ]
    }
   ],
   "source": [
    "obs,info=viz_env.reset()\n",
    "dones=False\n",
    "saved_conditions = viz_env.conditions\n",
    "saved_nelx, saved_nely, saved_dx, saved_dy = viz_env.nelx, viz_env.nely, viz_env.dx, viz_env.dy\n",
    "\n",
    "#use deepcopy to save \n",
    "while dones== False:\n",
    "    action, _states = model.predict(obs,deterministic=True)\n",
    "    print(action)\n",
    "    obs, rewards, dones,truncated, info = viz_env.step(action)\n",
    "print(\"Desired volume:\",saved_conditions['volfrac'],\"Obtained volume:\",viz_env.volume)\n",
    "print(\"Env reward:\",rewards)\n",
    "fig = viz_env.plot()\n",
    "fig.savefig('trained_agent.png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8b4d5238",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thomas/anaconda3/envs/SB3_update/lib/python3.10/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.07606540508568287\n"
     ]
    }
   ],
   "source": [
    "mean_reward, std_reward = evaluate_policy(model.policy, eval_env, n_eval_episodes=10, deterministic=True)\n",
    "print(mean_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0d2de1f-dd90-40b2-acc9-b3227fd2888b",
   "metadata": {},
   "outputs": [],
   "source": [
    "xval, f0val,it, H, Phimax, allPhi, den, N, cfg = run_mmc(saved_conditions,saved_nelx,saved_nely,saved_dx,saved_dy,plotting='contour')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 ('SB3_update')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "e21ef5adabae340b8408649b4e28a9d7d4d8eaab8fdd4faf01af585df564eed2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}