DistilBERT_SST-2_GPU.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "DistilBERT_SST-2_GPU.ipynb",
      "provenance": []
    },
    "environment": {
      "name": "tf2-2-2-gpu.2-2.m50",
      "type": "gcloud",
      "uri": "gcr.io/deeplearning-platform-release/tf2-2-2-gpu.2-2:m50"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EQhcSBOJvj54"
      },
      "source": [
        "*This notebook is adapted from [here](https://github.com/dipanjanS/deep_transfer_learning_nlp_dhs2019/blob/master/notebooks/6%20-%20Transformers%20-%20DistilBERT.ipynb). The purpose of this notebook is to show how to convert a custom DistilBERT-based text classification model to TensorFlow Lite.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9sl5ydGmtNkC"
      },
      "source": [
        "## Initial Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1KVYvYX-lgd3",
        "outputId": "9b3e3754-f510-441c-bb12-4ace33a94145"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Oct 12 01:43:22 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   37C    P0    39W / 300W |      0MiB / 16130MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tK6zJE1algeP"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bMUSaRgElgeT",
        "outputId": "83b2201f-ae6b-40fb-c240-39ddb6c26395"
      },
      "source": [
        "import os\n",
        "import time\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import transformers\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "# Fix random seed for reproducibility\n",
        "seed = 42\n",
        "np.random.seed(seed)\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "print(\"TF Version: \", tf.__version__)\n",
        "print(\"Eager mode: \", tf.executing_eagerly())\n",
        "print(\"GPU is\", \"available\" if tf.test.is_gpu_available() else \"NOT AVAILABLE\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TF Version:  2.4.0-dev20200810\n",
            "Eager mode:  True\n",
            "WARNING:tensorflow:From <ipython-input-2-fd8f900afcbb>:18: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.config.list_physical_devices('GPU')` instead.\n",
            "GPU is available\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UPgMZ7gktSIX"
      },
      "source": [
        "## SST-2 Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hb3sPepg7phK"
      },
      "source": [
        "data_dir = tf.keras.utils.get_file(\n",
        "      fname='SST-2.zip',\n",
        "      origin='https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',\n",
        "      extract=True)\n",
        "data_dir = os.path.join(os.path.dirname(data_dir), 'SST-2')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q6j_RXB47yXw",
        "outputId": "0c80225e-72d3-4bc1-b4b8-c5389a957723"
      },
      "source": [
        "!ls $data_dir"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dev.tsv  original  test.tsv  train.tsv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JdOaRRZ18X9x"
      },
      "source": [
        "train = os.path.join(data_dir, \"train.tsv\")\n",
        "valid = os.path.join(data_dir, \"dev.tsv\")\n",
        "test = os.path.join(data_dir, \"test.tsv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yVkvhuN08RR0"
      },
      "source": [
        "train_dataset = pd.read_csv(train, sep='\\t')\n",
        "valid_dataset = pd.read_csv(valid, sep='\\t')\n",
        "test_dataset = pd.read_csv(test, sep='\\t')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZU95IkmalgeX",
        "outputId": "ba861677-7161-47ba-fc24-4b11bf06d1bd"
      },
      "source": [
        "train_dataset.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 67349 entries, 0 to 67348\n",
            "Data columns (total 2 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   sentence  67349 non-null  object\n",
            " 1   label     67349 non-null  int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 1.0+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trRHOJb588rP",
        "outputId": "709c4686-90c9-4d4f-bebf-6ea88da56b24"
      },
      "source": [
        "valid_dataset.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 872 entries, 0 to 871\n",
            "Data columns (total 2 columns):\n",
            " #   Column    Non-Null Count  Dtype \n",
            "---  ------    --------------  ----- \n",
            " 0   sentence  872 non-null    object\n",
            " 1   label     872 non-null    int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 13.8+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y5PIDFJ1lgec",
        "outputId": "bd44546d-19b4-4bdc-a510-0592a964adb2"
      },
      "source": [
        "train_dataset.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>hide new secretions from the parental units</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>contains no wit , only labored gags</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>that loves its characters and communicates som...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>remains utterly satisfied to remain the same t...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>on the worst revenge-of-the-nerds clichés the ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            sentence  label\n",
              "0       hide new secretions from the parental units       0\n",
              "1               contains no wit , only labored gags       0\n",
              "2  that loves its characters and communicates som...      1\n",
              "3  remains utterly satisfied to remain the same t...      0\n",
              "4  on the worst revenge-of-the-nerds clichés the ...      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVNAbNntlgel",
        "outputId": "b8920c04-dd4e-4ef0-84a7-cde0f5ddc647"
      },
      "source": [
        "train_reviews = train_dataset['sentence'].values\n",
        "train_sentiments = train_dataset['label'].values\n",
        "\n",
        "valid_reviews = valid_dataset['sentence'].values\n",
        "valid_sentiments = valid_dataset['label'].values\n",
        "\n",
        "test_reviews = test_dataset['sentence'].values\n",
        "\n",
        "train_reviews.shape, valid_reviews.shape, test_reviews.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((67349,), (872,), (1821,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fAnON4yrtWYG"
      },
      "source": [
        "## Preprocess Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k68KgD7ulgep"
      },
      "source": [
        "tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-uncased')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHScGsj8lgez"
      },
      "source": [
        "import tqdm\n",
        "\n",
        "def create_bert_input_features(tokenizer, docs, max_seq_length):\n",
        "    \n",
        "    all_ids, all_masks = [], []\n",
        "    for doc in tqdm.tqdm(docs, desc=\"Converting docs to features\"):\n",
        "        tokens = tokenizer.tokenize(doc)\n",
        "        if len(tokens) > max_seq_length-2:\n",
        "            tokens = tokens[0 : (max_seq_length-2)]\n",
        "        tokens = ['[CLS]'] + tokens + ['[SEP]']\n",
        "        ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "        masks = [1] * len(ids)\n",
        "        # Zero-pad up to the sequence length.\n",
        "        while len(ids) < max_seq_length:\n",
        "            ids.append(0)\n",
        "            masks.append(0)\n",
        "        all_ids.append(ids)\n",
        "        all_masks.append(masks)\n",
        "    encoded = np.array([all_ids, all_masks])\n",
        "    return encoded"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5psX52tRtZyB"
      },
      "source": [
        "## Build the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gtqmuFyFlge6",
        "outputId": "1703cf09-8325-4e3e-fd67-a1069e58fc4f"
      },
      "source": [
        "MAX_SEQ_LENGTH = 128\n",
        "\n",
        "inp_id = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype='int32', name=\"bert_input_ids\")\n",
        "inp_mask = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype='int32', name=\"bert_input_masks\")\n",
        "inputs = [inp_id, inp_mask]\n",
        "\n",
        "hidden_state = transformers.TFDistilBertModel.from_pretrained('distilbert-base-uncased')(inputs)[0]\n",
        "pooled_output = hidden_state[:, 0]    \n",
        "dense1 = tf.keras.layers.Dense(256, activation='relu')(pooled_output)\n",
        "drop1 = tf.keras.layers.Dropout(0.25)(dense1)\n",
        "dense2 = tf.keras.layers.Dense(256, activation='relu')(drop1)\n",
        "drop2 = tf.keras.layers.Dropout(0.25)(dense2)\n",
        "output = tf.keras.layers.Dense(1, activation='sigmoid')(drop2)\n",
        "\n",
        "\n",
        "model = tf.keras.Model(inputs=inputs, outputs=output)\n",
        "model.compile(optimizer=tf.optimizers.Adam(learning_rate=3e-5), \n",
        "              loss='binary_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_projector', 'vocab_transform', 'vocab_layer_norm', 'activation_13']\n",
            "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the weights of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "bert_input_ids (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bert_input_masks (InputLayer)   [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_distil_bert_model (TFDistilB ((None, 128, 768),)  66362880    bert_input_ids[0][0]             \n",
            "                                                                 bert_input_masks[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem (Slici (None, 768)          0           tf_distil_bert_model[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 256)          196864      tf.__operators__.getitem[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_19 (Dropout)            (None, 256)          0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 256)          65792       dropout_19[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dropout_20 (Dropout)            (None, 256)          0           dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 1)            257         dropout_20[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 66,625,793\n",
            "Trainable params: 66,625,793\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dpDBzyDTs4HI",
        "outputId": "8057942f-7566-4f01-d2f9-8fec40e0baf6"
      },
      "source": [
        "tf.keras.utils.plot_model(model, \"model.png\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe0AAALlCAYAAADpD74LAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeVxU5f4H8M8wgIpbV8GFJHPH3DVgFDQtWTTMDZVNzA1bNKTMrUy8elNe1zS7WaalZuAClFgplpqWGwoEmltpVoqigNeVfeD7+6Mfcx3Zhm2GA5/363Vft3nO9j3PeZ7zdZ7znEElIgIiIiKq6SLNTB0BERERGYZJm4iISCGYtImIiBSCSZuIiEghzB8tOH78OFatWmWKWIjqjMjIyGrb96pVq3D8+PFq2z8RGUdx94ki37SvXr2KqKgoowREVNckJydXe/86fvw4YmNjq/UYRFR9SrtPFPmmXag6vwkQ1VURERGYMGFCtR9Ho9GwDxMpVGn3CT7TJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihWDSJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihWDSJiIiUogqS9r379+vql2RgR48eGDqEKgWYR9WvlOnTiEnJ8fUYVA1qnTSXrt2LQYOHAiNRlMV8VSZ6Oho2NnZ4fz586YOpVwMiXvDhg1wdXVF165dq+y4+/fvx7Rp06BSqaBSqeDu7o7w8PAq239FRUVFQaPR6OIKCgpCUlKSqcOqVdiHlW/r1q3o2LEjevfujTt37lTbcXifML1KJ+0ZM2bg7t27KCgoqIp4ipWSklLubRo2bIgWLVqgfv361RCRYaor7ilTpiA7OxtarbYy4ekZOnQoPv30U9jY2AAANm7cCD8/vyrbf3k8XG9eXl54//33AQC9e/fGmjVr0Lt3b5PEVVuxD5esInGbgq+vL8aMGVPtx+F9wvQqnbTNzc3x+OOPV0Usxbp9+zb8/f3LvZ2rqysSEhLQrl27aoiqbNUZt1qtRps2bSoTXomaNGkCAGjatGm17L8sxdXbY489BsB0MdV27MPFq2jcptK8eXOjHYv3CdOp0RPRcnNz4evri8uXL5s6lHJRatwAoFKp9P7fmEqqN1PGRJWj1L6g1LiNhfcJ06nSpB0XFwcPDw80a9YM7u7uepUqIli3bh1efvllODk5wc3NDRcvXgQA3LhxA6tWrULPnj2RkpICNzc3tG3bFuvWrcPZs2eRnp6O6dOnY+XKlQbFcfv2bXz22WdwdXVFdHQ0ACApKQlvvvkm2rdvj9u3b+PFF1+EtbU1HB0ddXGeO3cOb731Fp566ilcv34do0aNQrNmzeDo6IjY2FgAwLZt29CkSRPY2dkBAO7du4f3338f9evXR//+/QEAkZGRVRZ3oV27diEwMBDz5s3DrFmzigzbJSUlYfLkyQgNDcXIkSPh6uqqW3b06FHY2dkhJibGoDge3W9Nr7eH3bx5E4GBgVi6dCmmT5+O0aNH49atW7o6bNy4MVQqFd5//33k5uYCAI4fP47WrVvj3XffBVCxtlp4DKVjH65cWzx79iwWLlyILl264MqVK1i0aBHatm2Lbt264eDBg8jOzkZwcDA6dOgAOzs77N27V2/70tpvYR2U1M8f9c0330CtVsPHxwc7d+4sc3veJxRyn5BH7NixQ4opLpWHh4dYW1vLrFmzJCYmRt577z2xtLQUW1tbycjIEBGR5cuXy+bNm0VERKvVikajkVatWklGRobExMSIvb29qNVqCQkJkc8++0wcHR3l2rVr4unpKU8++WS54jl37pwEBwcLAImKihIRkZSUFBk6dKgAkBkzZsjZs2dl37590qRJE/H29hYRkfnz58tjjz0marVagoOD5eDBg/Lll1+KtbW1WFlZyfXr10VExM3NTdq0aaN3zKefflo0Go3uc1XFLSISHh4uTk5OkpWVJSIiaWlpYmNjI61atdKt06VLFzly5IiIiOTk5Iinp6du2e7du6VBgwYSHh5eZgwdO3YUAPLgwYMaU28XLlwQADJ48OAy4x88eLBMmDBB97lXr17i7++v+zx//nwBIHFxcbqynJwccXJy0n2uaFs1REX6V3l5eXmJl5dXubZhH66aPpyamioTJ04UADJ16lRJSEiQe/fuiYuLi7Rv315effVVOXfunNy/f1+GDBki7du319u+rPZbWj9fsWKFAJAbN27o6mLDhg16++d94m8Kvk9EVFnStrW11Stbvny5AJA1a9bItWvXpGXLlpKfn69bvnLlSgEg27dvFxGRqVOnCgC5ePGi3n4q0nFERA4dOlQk+S1YsEAASHp6uq7s+eefl06dOuk++/r6ioWFheTm5urKoqKiBIC88847IiIyatSoIo1Ko9FUusMXF3dGRoa0bt1atm7dqrfemDFjdEk7NzdXVCqVrFmzRrd87969eutrtVqDjv9oZxQxfb2VpzMOGTJE3n33Xd1nPz8/6dmzp+7z1atXxdzcXKZNm6Yr+/bbb2Xp0qUiIpVqq4aoyUmbfbhq+vDatWsFgJw+fVpXtmrVKgEgiYmJurLVq1cLAElNTdWVldZ+y+rnhUn7+vXrMn/+fPn666/14uJ94n8UfJ+IMK/8d/W/FU5MKBQQEIAFCxYgISEBtra2yMvLw4wZM/TWmTZtGho0aAAAsLCwgLm5OTp27Fgl8ZibFz01tVpdZFnjxo313k+1srKCWq2GhYWFrmzkyJGoV68efvnllyqJrTSPxn348GGkpKSgR48eeuWWlpa6/7awsICbmxtmz56NM2fOYMWKFXB3d9dbv/DcK0IJ9Vbohx9+AABkZGQgLCwMcXFxerOi27Rpg3HjxiEsLAzLly+HtbU1IiIisHjxYgDAsWPHjN5Wawr24apRGKOZ2f+ePjZu3BgA9GJq1KgRACA9PV03G7u09mtIPweAV199Fa1atcKIESP0ynmf+B8l3yeqLGk/ytbWFg0aNEBWVhbOnz+Phg0bYsOGDdV1uGplbm4OW1vbKn3FylAXLlwAoJ+ki7N9+3b4+Phgw4YN2LlzJyIiIjBkyBBjhFgiU9Rbfn4+QkNDcfHiRbz++us4cuSI7nlZoeDgYGzbtg3r16/HnDlzkJ6ejvbt2wOA4ttqVWIfrjrFTY4qLHs4WZTVfg3p51ZWVtiwYQMmTpyoew5cnu2NjfeJ8qnW2eMqlQrdu3eHlZUVkpOTkZycXGSd9PT06gyhymRmZsLe3t7oxy1M1n/99Vep61lZWSEmJgZhYWEwNzeHh4dHjfhRCmPV26VLl5CZmYnhw4fj/Pnz2LRpU5HRiUIODg5wdnbG2rVr8e233+p9I6kNbbUqsQ8bT0FBQZnt15B+/q9//Qv29vbw8fEp8kMrvE8o/z5RbUn7zz//RF5eHsaPH48ePXpARDBv3jy9dVJTU7Fp06bSAzQzQ15eXnWFaZCUlBSkpaXBy8sLwN//Mnzw4AHy8/N16zx48EDvX8xVFXfPnj0BADt27NArLygo0B0/JycH69evBwD4+fkhNjYWIoKDBw/qrW8IEdH7/8qoqnozJJY33ngDiYmJ+P777zF48GBdeV5eXrHbz507F9evX8cbb7yBcePG6cor01ZrG/Zh48Z98uTJUtuvIf0cAOrXr48vvvgCKSkpmD59uq6c94nacZ+okqStVqtx79493fCGiGDp0qVYvHgx7O3t4erqCgcHB2zduhVjx47FF198gcWLF8PPzw+TJ08GAGi1WuTn5xcZIrG1tcWNGzeQlJSEQ4cOITMz06CYsrKyAEDvd3gLL/LDx8jKyiqyz5ycHJw6dUr3edmyZZg0aRIcHR0B/H3B7ty5g+XLl+O3337DsmXLkJOTg19//RWJiYlVGrezszOGDBmCzZs34+OPP0ZmZibi4uJw5MgRpKWlYdu2bcjMzMTGjRt1jdzW1hZNmzZF3759Afz904P/+Mc/EBUVVebx7927BwC4e/dujam3wliK+3nGu3fvYtKkSbrnRwDw+eef45dffsHGjRtx9uxZ3Lx5E6dPn8bNmzd1240YMQLdu3dHr1699H6UojJtVcnYh6uuDxcXY2FZdna2rqxweeH5FQ6Xl9Z+S+vnheVarRa9e/fGkiVLEBUVheXLl+uOyftELbhPlGPWWolOnz4t3t7e4uHhIYGBgRIUFKQ341NE5NatW+Ln5yctWrQQGxsbCQgI0E1/DwsLk9atWwsACQoKkjNnzui2O3XqlNjZ2Unnzp0lMjLSoHiOHz8uw4cPFwAyaNAgOXr0qOzfv1834/GVV16R1NRU2bJlizRt2lQASEhIiGi1Wpk2bZpYWlpKcHCwjBs3TqZOnSpLly6VgoIC3f7v3r0rI0aMkEaNGolGo5G4uDh58cUXxd/fXzdjs6riLjzelClTpGXLlvLEE09ISEiIBAYGyuTJk2X//v2SmZkpDg4O4u7uLitWrJDAwED59NNPdfv94YcfpHXr1hIdHV3isQ8ePCivvPKKABAA4uHhIdu3bzd5vUVHR4uLi4suLo1GI+7u7uLq6ir29vZiaWkpAOSTTz4REZGXXnpJGjduLBqNRvbv3y979uwRa2tr8fLy0pvpKiISFBRU7LWpaFs1RE2dPc4+XDV9ODY2VjQajQAQPz8/uXTpkpw4cUKcnZ0FgEyYMEEuXLgg8fHxujJ/f3/5/fffRaT09nvr1q0S+/nOnTule/fuuleufvvtN4mNjRW1Wq17/ezMmTO8Tyj/PlE1r3zVJtOmTZP69eubOgzFUWK9Pffcc7p3342lpibt2kSJbbGuUOK1qWH3iap75ctYCl+NKM3GjRuLvO5gakqNu7Y6ePAg+vXrZ9I/RlFXKbUvKDVuqriaeJ9QXNJOS0ur1v0/ePBANymhKn/DtrrjNrXqqreqdOTIEcyYMQPdunXDmTNn8NNPP5k6pDqJfbju4n2i8mr0Hwwxti1btmDfvn3Iz8/HG2+8gZMnT5o6JEVQSr01b94c2dnZ+Pnnn/HJJ5/A2tra1CFRFVNKW6yLlHJtavp9QiWiP889IiICEyZMqJKp/ESkzxj9q/DVlMjIyGo7BhFVn1LuE5H8pk1ERKQQTNpEREQKwaRNRESkEEzaRERECsGkTUREpBBM2kRERArBpE1ERKQQTNpEREQKwaRNRESkEEzaRERECsGkTUREpBBM2kRERArBpE1ERKQQJf497cK/FERUmpycHNSrV8/UYShGcnKyUY4TGxvLPlzN2PapupR2nyjyTdvOzg5eXl7VGhDVDvfu3cOePXvw66+/mjoUxWjTpk2196/+/ftDo9FU6zHqul9//RV79uzBvXv3TB0K1UKl3SeK/D1tIkOJCD744APMnTsXgwYNQlhYGFq2bGnqsIiqza1btzBp0iR89913eOutt/DOO+/AzIxPGcloIpm0qdLi4uIwYcIE5OXlYdu2bXBxcTF1SERV7uTJk/D29oZWq8W2bdvg7Oxs6pCo7onkPxGp0hwcHJCYmAiNRoMhQ4YgJCQEBQUFpg6LqEqICNasWQMXFxd0794diYmJTNhkMkzaVCWaNm2KiIgIrFy5EsuXL4ebmxtu3Lhh6rCIKiU9PR2enp6YM2cOFi5ciOjoaDRv3tzUYVEdxuFxqnLx8fGYMGECMjIyEBYWhqFDh5o6JKJy++mnn+Dr6wszMzNs374dAwYMMHVIRBwep6r39NNPIzExEYMGDYKHhweHy0lRCofDhw4din79+iEpKYkJm2oMftOmarV+/XrMmjULLi4uCAsLQ+vWrU0dElGJ0tLSMGnSJOzbtw/Lli3D3LlzoVKpTB0WUSHOHqfql5CQgAkTJuD+/fsICwuDq6urqUMiKuLHH3+Er68vzM3NsX37dvTv39/UIRE9isPjVP369euHn3/+GUOGDMGwYcMQEhKC/Px8U4dFBOB/w+Gurq5wcHBAUlISEzbVWPymTUa1fv16vPbaaxgwYADCw8M5XE4mlZaWhokTJ+KHH37A0qVLORxONR2/aZNxBQYG4tixY7h69Sp69+6N7777ztQhUR116NAh9O7dGxcuXMBPP/2EefPmMWFTjcekTUbXt29fJCQk4Nlnn8WwYcMwf/58DpeT0YgIQkNDMXToUDg6Oup+GIhICTg8Tia1ZcsWvPTSS3ByckJ4eDhsbW1NHRLVYqmpqZg4cSJ+/PFHhIaGIigoyNQhEZUHh8fJtAICAnD06FFcu3YNvXv3xt69e00dEtVSBw8eRO/evfHbb7/hxx9/ZMImRWLSJpPr06cPEhISMHToUAwfPhxBQUHIy8szdVhUS+Tn5yMkJASurq7QaDRITEyEk5OTqcMiqhAOj1ONsmXLFrz88st4+umnsXXrVjz++OOmDokULDU1Ff7+/jh8+DBWrFjBb9ekdBwep5olICAAcXFxuHXrFnr37o2YmBhTh0QK9cMPP6BXr17466+/cPz4cSZsqhWYtKnGeeqpp3D8+HG4u7vj+eef53A5lcvDw+HOzs44ceIEevfubeqwiKoEh8epRiscLu/evTt27NiBJ5980tQhUQ127do1+Pj4IC4ujsPhVBtxeJxqtoCAAMTHxyMzMxMODg7Ys2ePqUOiGurAgQN4+umnkZqaitjYWCZsqpWYtKnG69q1K06ePIlRo0bB09OTw+WkR6vVIiQkBG5ubhg6dCji4+PRq1cvU4dFVC04PE6KsmXLFrzyyivo1q0btm/fjnbt2pk6JDKh5ORk+Pj4ID4+nsPhVBdweJyUpXC4PCsrCw4ODti9e7epQyIT2b9/P55++mmkp6fjxIkTTNhUJzBpk+LY29vjxIkTGDt2LEaMGIGgoCDk5uaaOiwyksLhcHd3d7i5uSE+Ph49e/Y0dVhERsHhcVK0wuHyp556Ctu3b0f79u1NHRJVo6tXr8LHxwc///wzli9fzm/XVNdweJyUrXC4PCcnB3379kVUVJSpQ6Jq8s0336B37964desWh8OpzmLSJsUrHC6fNGkSxo8fz+HyWqZwOHzUqFF4/vnnER8fjx49epg6LCKT4PA41SphYWF4+eWXYW9vjx07dnC4XOGuXLkCHx8fJCYm4oMPPsC0adNMHRKRKXF4nGoXf39/xMfHIy8vD3369EFERISpQ6IK+vrrr9GnTx/cvn0bJ0+eZMImAofHqRbq0qULYmNj8eKLL2LChAmYMWMGh8sVRKvVYv78+XrD4d27dzd1WEQ1AofHqVb78ssvMXXqVHTs2BE7duxAhw4dTB0SleLKlSvw9vbGL7/8gnXr1sHPz8/UIRHVJBwep9pt7NixOHnyJPLz89G3b1/s2LHD1CFRCXbt2oXevXvj7t27iI2NZcImKgaTNtV6nTt3xvHjx/Hiiy/C29sbAQEByMrKMnVY9P9ycnIQFBSE0aNHw9PTE3FxcejWrZupwyKqkTg8TnXKV199halTp6J9+/bYsWMHOnbsaOqQ6rS//voL3t7eOHv2LD755BP4+PiYOiSimozD41S3jBkzBomJibCwsEDfvn2xfft2U4dUZ+3cuRN9+vRBTk4OEhISmLCJDMCkTXXOk08+iR9//BGTJ0+Gj48PAgICkJmZWeL6sbGx0Gq1RoxQ2bRaLWJjY0tcXjgcPmbMGHh6euLIkSPo1KmTESMkUi4mbaqT6tWrhzVr1mDnzp349ttv4eLigosXLxZZ78qVKxg2bBhWrFhhgiiVacWKFRg2bBiuXLlSZNmff/6JQYMGYfPmzdi2bRu2bNkCKysrE0RJpExM2lSnjRo1ComJiahXrx769euHrVu36pbl5eVh3LhxuHfvHpYsWYJTp06ZMFJlOHXqFJYsWYK7d+/Cy8sLeXl5umVfffUV+vTpA61Wi4SEBHh7e5swUiJlYtKmOq9t27b46aef8Morr8Df3183XP7WW28hPj4eBQUFAABfX1/+SEsptFotAgICAAAigp9//hlvv/02srOzERQUhLFjx2L8+PE4duwYJwASVRBnjxM9pHB2ebNmzXD58mW9Zebm5li4cCGWLFliouhqtnfeeQfvvvsu8vPz9crbtWuH27dv47PPPsOYMWNMFB1RrRDJpE30iOPHj2PIkCHIy8vTfcsuZGZmhtjYWDg4OJgoupopMTERDg4ORRK2SqWChYUFDhw4ABcXFxNFR1Rr8JUvoodptVq8/vrrKCgoKJKwgb+Ttq+vL7Kzs00QXc2Uk5MDX19fqFSqIstEBCKC4OBgvefbRFQxTNpED1m0aBHi4uJKTDBarRZ//PEH/vnPfxo5sporJCQEFy9eLPG1uLy8PCQmJmLx4sVGjoyo9uHwONH/27t3L4YPHw5DuoSZmRmOHz8OR0dHI0RWc508eRL9+/cvdlTiUSqVCnv27IGHh4cRIiOqlTg8TlTo1KlTsLOzAwBYWFiUuq6ZmRn8/Pzq9DB5dnY2/Pz8YGZW+m2ksC7t7Oz42hxRJfGbNtEjzp49i2+//RZfffUV4uLiYGZmBhEp8m3S3Nwcs2fPxr///W8TRWpac+bMwZo1a4oMi6tUKqjVami1WnTq1En3y2fOzs7FPvcmIoNx9jhRaa5cuYK9e/fiq6++wv79+3XlhbOkVSoVfvzxRwwcONBUIZrE8ePH4eLiovuHjFqtBgAUFBTAwcEBY8aMwejRo9G5c2dThklU2zBpV8bx48dx9epVU4dBRnL//n0kJCTgxIkTOH36NPLz8yEiaNGiBVauXIl69eqZOkSjyMnJwZw5c5Camgrg7xGHHj16QKPRoF+/fmjcuLGJIyRjsbOzQ//+/U0dRl3CpF0Z48aNQ1RUlKnDICIyCS8vL0RGRpo6jLok0tzUESgdGy3l5eXh8OHDGDBgAOrXr2/qcKpVdnY2jh07hoEDB5Y5WY9qt3Hjxpk6hDqJSZuokiwsLPDss8+aOgyjqF+/fp05V6KaiK98ERERKQSTNhERkUIwaRMRESkEkzYREZFCMGkTEREpBJM2ERGRQjBpExERKQSTNhERkUIwaRMRESkEkzYREZFCMGkTEREpBJM2ERGRQjBp11IPHjwwdQilun//vqlDqBFq+nUiZWK7qr2YtGsIEcHq1auxYsUKdOrUCRMnToRWqy33fjZs2ABXV1d07dpVVxYdHQ07OzucP3++wvE9ug+tVovY2FiEhITg+++/N3g/a9euxcCBA6HRaCocS21Q3HUqTUXrGwBeeuklqFQqtGjRAr169UKXLl2gUqnQvHlz9OvXDx07doRarUaDBg0QGRkJR0dHqFQq1KtXD0OHDsWwYcPg4eGBZ555Bi1btoRKpcKaNWswYsQIqFQqqFQqDBgwAC4uLujTpw80Gg3mzZuH33//vUgsxmqL33//PSZNmqSLb/DgwXBzc4NGo4GTkxNWrVpVbf9w/PrrrzFmzBjdsc+cOVPq+r169YJKpYK1tTWWL1+OzMzMCh/bmO2KTESowry8vMTLy6vc212/fr1IWUhIiMyYMUNERA4fPiwjRoyQrKyscu9bq9WKi4uLtGrVSlf2/fffS9++feXy5csVjvHRfRw7dkwmT54sAOTTTz81eL95eXnSo0cPsbe3N3ib8iqufmua4q5TaSpa3yIikyZNknfeeUfy8/NFRGT//v0CQPz9/XXrnDlzRpo0aSIFBQVy9OhRASDOzs5F9pWXlyeDBg2Sc+fOSXJysgCQtm3b6q1z8uRJ8fDwELVaLQsXLtQdV8S4bbGgoECaNGkiAPRi2L59u6jVahk4cKDk5OQYHEd5YszKyhIAAkCmT59e4nZHjhwRtVotAGTOnDmVjsOY7aqi9z+qlAh+0zay27dvw9/fv0j5Rx99hCeffBIA4OLigq+//hr169cv9/7VajXatGmjV+bq6oqEhAS0a9euwjE+uo/+/ftj1qxZ5Y7P3Nwcjz/+eLm3M1RJ9VvTFHedSlPR+gYAlUqFhQsXwsys5O7erVs3eHt7IycnB82aNQPw998Jf5S5ubnum3vDhg0BAA0aNNBbx8HBAbt374a3tzfeffddhIaG6pYZsy2qVCo0btwYAPTOfcKECRg3bhwOHz6Mo0ePGhRHeWOsX78+2rVrh4YNGyIsLAy3bt0qdtuPPvoIo0aNAgA0bdq00rEYs12RaTBpG1Fubi58fX1x+fJlvfLs7GykpqZCpVKZKLL/KSnG4lhaWhohIsOVJ3Ylqmh9z507F/Xq1TNoPXNz8zLboY+PD+zt7Utdz8zMDGvXrkWLFi2wbNkyXLlypdxxV0VbLCnGDh06AAD++OOPcsf1sNJibNq0KQICApCVlYUNGzYUWZ6amopff/0VgwcPLjXW6lbT+jGVjknbiCIjI3H27Fmkp6dj+vTpWLlyJT7//HNMnz5dt3z69Ol630wMsWvXLgQGBmLevHmYNWsWUlJSdMtu376Nzz77DK6uroiOjtaVJyUlYfLkyQgNDcXIkSPh6upaYowl7aOy4uLi4OHhgWbNmsHd3V3vxiciWLduHV5++WU4OTnBzc0NFy9eBADcuHEDq1atQs+ePZGSkgI3Nze0bdsW69atKxK7Ic6ePYuFCxeiS5cuuHLlChYtWoS2bduiW7duOHjwILKzsxEcHIwOHTrAzs4Oe/fuLbKPL7/8EjNnzsScOXMwbNgwvP3228jJydFbp7TrVNY5l+To0aOws7NDTExMiesY+nyzQ4cOMDc3L3WdJUuWGLQv4O+kNX78eGRmZiIiIqJGtcWjR4/CzMwMTk5OurLqaHOvvfYaVCoV1q5dW2SOyqefforAwMBSk7Wp2hXVYKYdnle2ijzT8fT0lCeffFKvLD09XQDIsmXLyh1DeHi4ODk56Z5/p6WliY2Nje6Z1rlz5yQ4OFgASFRUlG67Ll26yJEjR0REJCcnRzw9PUuMsaR9nDlzpkLPwjw8PMTa2lpmzZolMTEx8t5774mlpaXY2tpKRkaGiIgsX75cNm/eLCJ/P6fTaDTSqlUrycjIkJiYGLG3txe1Wi0hISHy2WefiaOjo1y7dq3Y+i1LamqqTJw4UQDI1KlTJSEhQe7duycuLi7Svn17efXVV+XcuXNy//59GTJkiLRv315v+9WrV8uAAQMkNzdXRP6+np06dZJnnnlGCgoKRKUYtYcAACAASURBVKTs61TWOZdU37t375YGDRpIeHi4wedb3DPth124cEEAyODBg3Vl+fn5cu7cOb25CHfu3BEApc5PCAsLEwAyefJkk7TFNm3aCABJSkqSxMREiYmJEW9vb3nsscfkk08+0Vu3qttc7969RUTE3d1dAMiOHTt0y7RarfTs2VMePHggH374YbH935TtyhB8pm0SfKatZJmZmZgzZw6CgoJ0z7+tra0xcOBA3Tpdu3bFyJEj9bbLy8vDb7/9hoSEBAB/D4/NnDmzxOMUt4/KsrS0xAcffAAPDw+8/vrrWLJkCa5fv45PP/0U169fx/vvv4+JEycC+Ps5nZeXF27cuIFvvvkGHh4ecHZ2Rn5+Pvz8/DBlyhScOHECtra2FYrFxsZGN5s9KCgIffv2RePGjTFmzBhcvnwZ06ZNQ9euXdGoUSO88MILuHz5MtLS0gD8PcT59ttv46WXXtI9A27evDkWLlyIH3/8EeHh4QZdp7LOuSTDhw/H/fv34evrW6FzL83PP/+M/v37o3///nBycsIzzzyDGzdulGsfNjY2AIArV66YtC2uWbMGoaGhWLRoEaKiojB8+HD069dPt7w621xQUJAuhkK7d+/G0KFDdfMCHmXqdkU1V+ljYVSjHT58GCkpKejRo4de+aPPqB4d8rSwsICbmxtmz56NM2fOYMWKFXB3dy/1WGUNm5ZXkyZN9D4HBARgwYIFSEhIgK2tLfLy8jBjxgy9daZNm6ab9GRhYQFzc3N07NixSuJRq9UA9CcsFU5ienhCVqNGjQAA6enpsLGxQWxsLDIyMvDEE0/o7c/T0xMAcPDgQdjY2JR5nY4dO1bmOZcVe1Xr27cvDh48qPucl5enG7o21N27dwEAnTt3BmC6trhx40bdf//yyy8YNWoUNBoNvvrqK4wYMcKg+q9om/Pw8EDnzp1x7NgxxMfH4+mnn8bHH3+MDz/8sMRtakK7opqJSVvBLly4AKBiE0m2b98OHx8fbNiwATt37kRERASGDBlS1SEazNbWFg0aNEBWVhbOnz+Phg0bFjt5x5iKe9ZYWFZQUAAA+OuvvwAA//3vf/XWs7a2hpWVFa5fv27Qdaop51waCwsLzJ07t1zbFJ57r169SlzH2G2xR48eCA0Nxbhx4/DGG29gxIgR1Vr/KpUKr732GmbOnIk1a9Zg8eLFMDc3102GK05daldUPhweV7DCzlrYwcvDysoKMTExCAsLg7m5OTw8PCr1gxdVQaVSoXv37rCyskJycjKSk5OLrJOenm6CyEpW+NpRSTOc7e3tDbpOSjnn4cOHG7yuiCAyMhJNmjTRfUMsjinaYp8+fQAAly5dQl5eXrXX/6RJk9C0aVNERETgnXfeKfURAFD32hUZjknbyMzMzJCXl6dXJiIV2lfPnj0BADt27NArLygoQH5+fonb5eTkYP369QAAPz8/xMbGQkR0Q6HFxVjd/vzzT+Tl5WH8+PHo0aMHRATz5s3TWyc1NRWbNm0qdT/Gjr1///5o0qRJkZnMycnJyMzMxAsvvGDQdarMORd+6zdU4foltbvC8rLaZVnL33vvPfzyyy9YuXJlie/mV3dbLCnGX3/9FQDQqVMnWFhYVEube/iXzRo1aoSpU6ciNzcX8fHxcHNz0y0r7nrUhHZFNROTtpHZ2trixo0bSEpKwqFDh5CZmYmMjAwAKPfPFzo7O2PIkCHYvHkzPv74Y2RmZiIuLg5HjhxBWloatm3bhszMTGRlZQGA3qsiGzdu1HVsW1tbNG3aFH379i0xxuL2ce/ePQAo98+tqtVq3Lt3T7ediGDp0qVYvHgx7O3t4erqCgcHB2zduhVjx47FF198gcWLF8PPzw+TJ0/WHTM/P7/IsYuL3RCFN92H91dYlp2drSsrXF5YD82bN0doaCiOHj2KAwcO6Nb74IMPMGnSJAwZMsSg6+Ts7FzmORdX3/v378c//vEPREVFGXSeAHDnzh29/ZW0vKzfry5c/mgd//XXX3jttdcwd+5cBAUF6V5pBGDUtigiup8qLexjwN//QJw9ezYAYOnSpQBQ5W3u+vXruHbtml6MM2fOhJmZGWbOnKn36OX27dt65wCYvl1RDWb8Geu1R0VeeTh16pTY2dlJ586dJTIyUhISEnSvG7Vr107Cw8Plzp07Bu/v7t27MmXKFGnZsqU88cQTEhISIoGBgTJ58mTZv3+/HD16VIYPHy4AZNCgQXL06FHJzs4WBwcHcXd3lxUrVkhgYKDe6x6Pxnj8+PEi+0hMTJTRo0cLABk4cKAcPHjQ4JhPnz4t3t7e4uHhIYGBgRIUFKT3+o6IyK1bt8TPz09atGghNjY2EhAQINeuXRORv18jat26tQCQoKAgOXPmTImxGyI2NlY0Go0AED8/P7l06ZKcOHFCnJ2dBYBMmDBBLly4IPHx8boyf39/+f3333X72LVrl7i7u8usWbNk0aJFsnLlSt1rOYZcp/z8/FLPuaT6/uGHH6R169YSHR1d5nkWFBTIRx99JN27dxcAUq9ePQkJCZFz587p1omOjpZBgwYJAFGpVLJgwQI5e/ZskX199913MmLECN1Pdbq4uMhzzz0nw4cPl2HDhklwcLAkJSXpbVNcO6qutnjgwAGZNm2aLr6nnnpKPDw8xNHRUTp06CCenp5y+PBhvfiqqs19+eWXujocPXq0/PTTT7p1/f395e7duyIi8uDBA1m1apVuvzY2NrJy5UrJzMw0ebsyBF/5MokIlUgFx2YJ48aNA/D3j0AQEdUlvP+ZRCRnj9dQhe+3lmbjxo0YMWKEEaIxTE2MuSbGRERUUUzaNVThj3coSU2MuSbGRERUUZyIRkREpBBM2kRERArBpE1ERKQQTNpEREQKwaRNRESkEEzaRERECsGkTUREpBBM2kRERArBpE1ERKQQTNpEREQKwaRNRESkEEzaRERECsGkTUREpBBM2kRERArBpE1ERKQQ/HvalZScnIyIiAhTh0FEZFTJyclo06aNqcOoc5i0Kyk2NhYTJkwwdRhEREbn5eVl6hDqHJWIiKmDIKLyGz9+PABwpIeo7ojkM20iIiKFYNImIiJSCCZtIiIihWDSJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihWDSJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihWDSJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihWDSJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihWDSJiIiUggmbSIiIoVg0iYiIlIIJm0iIiKFYNImIiJSCCZtIiIihTA3dQBEVLYTJ07g1KlTemWXL18GAKxfv16vvGfPntBoNEaLjYiMh0mbSAFSU1MxY8YMqNVqmJn9PUAmIgCAmTNnAgAKCgqQn5+Pr7/+2mRxElH1UklhzyeiGisvLw/W1ta4d+9eqes1btwY6enpsLS0NFJkRGREkXymTaQAFhYW8Pb2LjUZW1hYwMfHhwmbqBZj0iZSCB8fH+Tm5pa4PC8vD76+vkaMiIiMjcPjRApRUFAAW1tb3Lx5s9jlNjY2uHHjhu6ZNxHVOhweJ1IKMzMz+Pv7Fzv8bWlpiUmTJjFhE9Vy7OFEClLSEHlubi58fHxMEBERGROHx4kUpmPHjvj999/1ytq2bYs///zTNAERkbFweJxIafz9/WFhYaH7bGlpicmTJ5swIiIyFn7TJlKYS5cuoVOnTnplv/76Kzp37myiiIjISPhNm0hpOnbsiJ49e0KlUkGlUqFnz55M2ER1BJM2kQIFBARArVZDrVYjICDA1OEQkZFweJxIga5fvw47OzuICK5cuYI2bdqYOiQiqn6R/IMhDxk3bhyioqJMHQZRudjZ2Zk6BCKDeHl5ITIy0tRhKBqT9iM0Gg2Cg4NNHQZRmfbv3w+VSoXnnnvO1KEQlWn16tWmDqFWYNJ+RJs2bTB+/HhTh0FUpsJk3bx5cxNHQlQ2fsOuGkzaRArFZE1U93D2OBERkUIwaRMRESkEkzYREZFCMGkTEREpBJM2ERGRQjBpExERKQSTNhERkUIwaRMRESkEkzYREZFCMGkTEREpBJM2ERGRQjBpExERKQSTNhFVidzcXKSmppo6DHrE/fv3K7TdgwcPqjgSqgpM2tVERLB69WqsWLECnTp1wsSJE6HVak0dFtVQ0dHRsLOzw/nz500dikGOHDkCFxcXdOzYEU899RR69uwJV1dXfPnllwAArVaL2NhYhISE4Pvvv9dtV5HzNEbd3Lp1C3PnzsXIkSMxcuRIDBo0CMOGDcOePXvKva+oqChoNBqoVCqoVCoEBQUhKSmpGqIu3dq1azFw4EBoNJpybbdhwwa4urqia9eu1RQZVQaTdhVISUkpUvbPf/4Tv/76K+bPn49Nmzbh7t27dT5pF1dPddWjddGwYUO0aNEC9evXL3W9muDMmTNwdXXFjBkz8Ntvv+HcuXNYsGABEhIScOPGDQBAXFwc1q9fjyVLluDq1au6bUs6z9JUZJvy2L9/P3r06IHMzExERERg165d+Omnn7BgwQLMmDEDQUFByMnJMXh/Xl5eeP/99wEAvXv3xpo1a9C7d+9qib00M2bMwN27d1FQUFCu7aZMmYLs7Ow6f7+qqZi0K+n27dvw9/cvUv7RRx/hySefBAC4uLjg66+/rrabjhKUVE91UXF14erqioSEBLRr167U9WqCzZs3Q0QwceJEmJn9fQvx8fHBxx9/rPtHRv/+/TFr1qwi2xZ3nmWpyDaGunr1KkaPHg1HR0d8+OGHqFevnm7ZoEGDsGPHDnzwwQeYN29eufb72GOPAQCaNm1apfGWh7m5OR5//PFyb6dWq9GmTZtqiIiqApN2JeTm5sLX1xeXL1/WK8/OzkZqaipUKpWJIqtZSqqnusjQuqjJdXbz5k3k5OTg0KFDeuV+fn66JA4AlpaWRo6s/GbPno0HDx7g9ddfL3b5gAED4OzsjLVr15ZreL6w7/MeQFWNSbsSIiMjcfbsWaSnp2P69OlYuXIlPv/8c0yfPl23fPr06QgNDTVKPF9++SVmzpyJOXPmYNiwYXj77bd1w3rnzp3DW2+9haeeegrXr1/HqFGj0KxZMzg6OiI2Nla3DxHBunXr8PLLL8PJyQlubm64ePEiAODGjRtYtWoVevbsiZSUFLi5uaFt27a4desWbt68icDAQCxduhTTp0/H6NGjcevWrRLryZCYSzteUlISJk+ejNDQUIwcORKurq5GqWMAiI+Px/Tp0+Hr6wtHR0d88sknekOJpdVhcXVx+/ZtfPbZZ3B1dUV0dHSpdVbavs+ePYuFCxeiS5cuuHLlChYtWoS2bduiW7duOHjwILKzsxEcHIwOHTrAzs4Oe/fu1Tuvo0ePws7ODjExMaWe/zPPPAMAGDt2rN7zajMzM3z88cclblfceRbas2cPXnnlFQQFBaF///7YsGFDidskJSXhzTffRPv27XH79m28+OKLsLa2hqOjY5F/5JR2rbKyshAdHQ2VSgUnJ6cS4+7bty+0Wi3Cw8MN7kflUVrfqew1LRQXFwcPDw80a9YM7u7uRepp165dCAwMxLx58zBr1qwij2VKi5GMTEjHy8tLvLy8yrWNp6enPPnkk3pl6enpAkCWLVtWleGVavXq1TJgwADJzc3VxdCpUyd55plnpKCgQObPny+PPfaYqNVqCQ4OloMHD8qXX34p1tbWYmVlJdevXxcRkeXLl8vmzZtFRESr1YpGo5FWrVpJRkaGxMTEiL29vajVagkJCZHPPvtMHB0d5dq1azJ48GCZMGGCLp5evXqJv7+/7nNx9VRWzKUdr0uXLnLkyBEREcnJyRFPT8/qq9yH/PXXX9KwYUP5448/REQkICBAAEi/fv1k9uzZIlJ6HYoUrYtz585JcHCwAJCoqChdeXF1Vtq+U1NTZeLEiQJApk6dKgkJCXLv3j1xcXGR9u3by6uvvirnzp2T+/fvy5AhQ6R9+/Z6+969e7c0aNBAwsPDS60DrVYro0aNEgACQCZOnCipqalF1jtz5owAkE8//bTU89yyZYt4e3tLfn6+iIj861//EgBy4MCBYrdJSUmRoUOHCgCZMWOGnD17Vvbt2ydNmjQRb29vg69VfHy8AJA2bdqUer5r1qwRAOLl5WVwP7pw4YIAkMGDB5e6bxEpte9U9pp6eHiItbW1zJo1S2JiYuS9994TS0tLsbW11bXH8PBwcXJykqysLBERSUtLExsbG2nVqpVBMRqqIvdXKiKCSfshSk3aN2/elIYNG8qWLVv0yjdt2iQA5IsvvhAREV9fX7GwsNAlSRGRqKgoASDvvPOOXLt2TVq2bKm7eYqIrFy5UgDI9u3bRURk6tSpAkAuXryod6whQ4bIu+++q/vs5+cnPXv21H1+tJ4Mjbm44+Xm5opKpZI1a9boyvbu3WtgbVXOm2++KXZ2drrPhTfnTz75RETEoDosrs0cOnSozKRtyL7Xrl0rAOT06dO6dVatWiUAJDExUVe2evVqAVAk2Wq1WoPqQavVysqVK6Vhw4YCQJo1ayZfffWV3jqPJu3izjM1NVWaNm0qly9f1q2TlpYmY8aMkXPnzpVYNwsWLBAAkp6erit7/vnnpVOnTrrPZV2riIgIASBOTk6lnuvOnTsFgPTt21dEyu5HDx/LkKRdVt+pzDX18PAQW1tbveMtX75cAMiaNWskIyNDWrduLVu3btVbZ8yYMXpJu6wYDcGkXSUizKvn+zsZU2xsLDIyMvDEE0/olXt6egIADh48CH9/f1hZWUGtVsPCwkK3zsiRI1GvXj388ssvOHbsGPLy8jBjxgy9/UybNg0NGjQAAFhYWMDc3BwdO3bUW+eHH34AAGRkZCAsLAxxcXGlzlo1NObijmdhYQE3NzfMnj0bZ86cwYoVK+Du7m5QXVXWtWvXkJmZqfvcpUsXNG/eXDdD2pA6LI65edld0ZB9q9VqANB7tty4cWMA0LvujRo1AgCkp6fDxsZGV164fVnUajXeeOMNeHl54aWXXsLevXvh5eWFHTt2wMvLq8TtHj3PI0eOoKCgQG+SmbW1te7VseK2eTjOh5c1btxY753ksq5V4UTRa9eulXqud+7cAQC0bdsWAMrsR+VVVt+p7DVt0qSJ3vECAgJ0s/27dOmClJQU9OjRQ2+dR+cjlLd/U/Vh0q4F/vrrLwDAf//7X71ya2trWFlZ4fr16yVua25uDltbW2i1Wpw/fx4NGzbUPU8sj/z8fISGhuLixYt4/fXXceTIkVKf8VUmZgDYvn07fHx8sGHDBuzcuRMREREYMmRIueMuLw8PD2zduhUHDhzAc889hzt37iAjIwMeHh4AUKk6LEtF913cZKjCssreeNu2bYuYmBi89tpr+M9//oOZM2di7NixBk/AOnPmDPLy8iAiVT5pq6xr1bVrV6hUKiQnJyMjIwMNGzYsdj+///47ABRJbA97uB+VV3n7DlC5a2pra4sGDRogKysLFy5cAFD2pMGKxEjVgxPRaoHCbyklzTS2t7cvdfvMzEzY29vDysoKycnJSE5OLrJOenp6idsXFBRg+PDhOH/+PDZt2lTqza2qYrayskJMTAzCwsJgbm4ODw8Po/wwycSJE7FixQoEBARg0aJFeP3117Ft2zY4Ozvr4qpIHRqiOvdtqN9++w3vvfdekfI1a9agTZs2uHnzZpn/4HpYkyZNkJ2djXPnzhVZlpubW6lYy7pWjRo1wogRIwAAiYmJJe5n3759UKvVGDVqVKnHK+xHhrp06RIyMzPL3XeqgkqlQvfu3XXJuvAf0cWpSP+m6sOkXUlmZmbIy8vTKxOREtevjiGl/v37o0mTJkVm5CYnJyMzMxMvvPBCidumpKQgLS0NXl5e6NGjB0SkyDupqamp2LRpU4n7OHnyJL7//nsMHjxYV1b47anQo/VUmZhzcnKwfv16AH+/ZhQbGwsRwcGDB0vcpqIevV55eXn473//i1OnTmHp0qXYuHGj3s3ckDosrs0U59H1Knp9yqOs9tmuXTu89957SEtL0ytXqVSwtbVF48aN0bp1a4OP5+DgAAB4++239Y596dIlREZGliPyosq6VgCwatUq1KtXD6tXry52H/v27cOJEyfw2muvoU+fPiUe6+F+BJR+Dyj0xhtvIDExscy+U9X+/PNP5OXlYfz48ejZsycAYMeOHXrrFBQUID8/H4Bh/ZuMh0m7kmxtbXHjxg0kJSXh0KFDyMzMREZGBgDoPU8DgNWrV8Pa2rrKvxE2b94coaGhOHr0KA4cOKAr/+CDDzBp0iS9YeOcnBycOnVK93nZsmWYNGkSHB0d4erqCgcHB2zduhVjx47FF198gcWLF8PPzw+TJ08G8PfPU+bn5+sNAxYOy33++ef45ZdfsHHjRpw9exY3b97E6dOncfPmzSL11KBBA4NiLu54ALBx40bdTcXW1hZNmzZF3759q6pKARR/vUJDQ/Hjjz9i3759OHToEOLj4/HHH3/olhtSh8W1maysLADQ++WtR9dzdnYuc9+FSf7h+iosy87O1pUVLn/4ePv378c//vEPREVFlVgnFhYWaNCgAUaOHKn3LPjw4cP4+eefsWTJEt2z13v37hWJ5dHzHDBgAIYNG4bo6Gg8++yz+PDDDzF37ly8+eabmDBhQrHblHSeWVlZen2urGsFAB06dEB0dDSOHTuG+fPn6327j42NxZQpU/Diiy9i2bJletuV1o8A4O7duwD+9zz8YXfv3sWkSZN08zWA0vtOZa6pWq3GvXv3dMtEBEuXLsXixYthb28PZ2dnDBkyBJs3b8bHH3+MzMxMxMXF4ciRI0hLS8O2bdt09V9ajGREJpoBVyNVZHbjqVOnxM7OTjp37iyRkZGSkJCge0WjXbt2Eh4eLnfu3BERkQ8++EDatGkjFy5cqI7wZdeuXeLu7i6zZs2SRYsWycqVK6WgoEC3fNq0aWJpaSnBwcEybtw4mTp1qixdulRvnVu3bomfn5+0aNFCbGxsJCAgQK5duyYiImFhYdK6dWsBIEFBQXLmzBnddi+99JI0btxYNBqN7N+/X/bs2SPW1tbi5eUlDx48KFJPhsRc0vGys7PFwcFB3N3dZcWKFRIYGKg3Q7mqFHe9vvnmG2ncuLHudafC/3Xr1k1XT6XVoUjRNnP8+HEZPny4AJBBgwbJ0aNHi12vrH3HxsaKRqMRAOLn5yeXLl2SEydOiLOzswCQCRMmyIULFyQ+Pl5X5u/vL7///ruIiPzwww/SunVriY6OLrVeXnjhBXFzc5Pu3bvLCy+8IB4eHuLo6ChhYWG6dRITE2X06NECQAYOHCgHDx4s8TwzMzPllVdekccff1xatmwpL7/8sq7PFLfN/v37pWPHjgJAXnnlFUlNTZUtW7ZI06ZNBYCEhISIVqs16FoVSktLk5kzZ4pGo5EXXnhBfHx8xM/PT/bs2VPk/MvqR9HR0eLi4qI7nkajEXd3d3F1dRV7e3uxtLTUm8VeWt85cOBApa7p6dOnxdvbWzw8PCQwMFCCgoL0ZuGLiNy9e1emTJkiLVu2lCeeeEJCQkIkMDBQJk+eLPv375f8/Pwy+7chOHu8SkSoRDjGUWjcuHEAUOlhuZpq+vTpCAsL0/3LmcovPDwcFhYWGDhwIFJSUpCRkYEHDx7g5MmTyM3Nxb/+9S9Th0j/r7quFftRxdT2+6uRRHL2uAk9/FpGSdLT02FtbV3qOhs3btRNqKnrqrNOT506hXnz5ukmgj387LZ///7YsmVLBSKm6sBrRbUVk7YJPTqZp7o9ePCg2l6vqSmqs05PnTqFa9euYfny5Zg4cSJatmyJO3fu4MSJE9i3bx+WL19ebcem8qnOa1UX+hHVXJyIVkds2bIF+/btQ35+Pt544w2cPHnS1CEpjp+fH9555x385z//gZ2dHZo1awZPT0+kp6dj9erVsLKyMnWI9P+q61qxH5Gp8Zn2Q/jMhQyVmZmJBg0a8JuWAvBa1Qy8v1YJPtMmqgh+q1YOXiuqTTg8TkREpBBM2kRERArBpE1ERKQQTNpEREQKwaRNRESkEEzaRERECsGkTUREpBBM2kRERArBpE1ERKQQTNpEREQKwaRNRESkEEzaRERECsE/GPKIqKgo/jUgIqJq4OXlZeoQFI9/mvMhx48fx9WrV00dBpFBVq9eDQAIDg42cSREhrGzs0P//v1NHYaSRTJpEynU+PHjAQAREREmjoSIjCSSz7SJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImIiBSCSZuIiEghmLSJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImIiBSCSZuIiEghmLSJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImIiBSCSZuIiEghmLSJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImIiBSCSZuIiEghmLSJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImIiBSCSZuIiEghmLSJiIgUgkmbiIhIIcxNHQARlS0zMxM5OTl6Zbm5uQCA27dv65XXq1cPVlZWRouNiIyHSZtIATZt2oSZM2cWu6xZs2Z6nz/88EO8+uqrxgiLiIxMJSJi6iCIqHRpaWlo3bo18vPzS11PrVYjJSUFNjY2RoqMiIwoks+0iRTAxsYGzz77LNRqdYnrqNVqPPfcc0zYRLUYkzaRQvj7+6O0gTERgb+/vxEjIiJj4/A4kULcv38fNjY2RSakFbK0tERaWhqaNGli5MiIyEg4PE6kFI0bN4anpycsLCyKLDM3N8cLL7zAhE1UyzFpEymIn58ftFptkfL8/Hz4+fmZICIiMiYOjxMpSG5uLqytrXH//n298kaNGiE9PR316tUzUWREZAQcHidSEktLS3h5ecHS0lJXZmFh3i6DCwAAIABJREFUgfHjxzNhE9UBTNpECuPr66v7NTQAyMvLg6+vrwkjIiJj4fA4kcIUFBSgZcuWSE9PBwA0b94cN2/eLPUdbiKqFTg8TqQ0ZmZm8PPzg6WlJSwsLODv78+ETVRHMGkTKZCPjw9yc3M5NE5Ux/APhlCtMm7cOFOHYDSFf8nr3//+t4kjMZ7IyEhTh0BkUkzaVKtERUVBo9GgTZs2pg6l2rVt29bUIRhNcnIyYmNjTR0GkckxaVOtExwcjPHjx5s6jGp39uxZAEC3bt1MHEn1i4iIwIQJE0wdBpHJMWkTKVRdSNZEpI8T0YiIiBSCSZuIiEghmLSJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImIiBSCSZuIiEghmLSJiIgUgkmbiIhIIZi0iYiIFIJJm4iISCGYtImK8eDBA1OHQERUBJM20UM2bNgAV1dXdO3a1dShlMuePXswYsQIqFQqqFQqDBgwAC4uLujTpw80Gg3mzZuH33//3dRhElElMWkTPWTKlCnIzs6GVqs1dSjlMnz4cKxbtw4A0LZtWxw7dgxHjhxBYmIi/vOf/+D06dPo0qUL3nrrLRQUFJg4WiKqKCZtooeo1Wq0adPG1GFUSMOGDQEADRo00Ct3cHDA7t274e3tjXfffRehoaGmCI+IqgCTNlEtoVKpSlxmZmaGtWvXokWLFli2bBmuXLlixMiIqKowaVOdt2vXLgQGBmLevHmYNWsWUlJS9JaLCNatW4eXX34ZTk5OcHNzw8WLFwEASUlJePPNN9G+fXvcvn0bL774IqytreHo6IjLly/r9pGUlITJkycjNDQUI0eOhKurq0H7B4CjR4/Czs4OMTExlTrPpk2bYvz48cjMzERERESNODciKichqkUAyI4dOwxePzw8XJycnCQrK0tERNLS0sTGxkZatWqlW2f58uWyefNmERHRarWi0WikVatWkpGRISkpKTJ06FABIDNmzJCzZ8/Kvn37pEmTJuLt7a3bR5cuXeTIkSMiIpKTkyOenp4G7V9EZPfu3dKgQQMJDw8v9Vzu3LkjAMTe3r7EdcLCwgSATJ48uUacm6F27NghvF0RSQR7AdUq5UnaGRkZ0rp1a9m6date+ZgxY3RJ+9q1a9KyZUvJz8/XLV+5cqUAkO3bt4uIyIIFCwSApKen69Z5/vnnpVOnTiIikpubKyqVStasWaNbvnfvXoP3L/J3wiuLIUn7u+++EwDy3HPP1ZhzMwSTNpGIiESYG/+7PVHNcPjwYaSkpKBHjx565ZaWlrr/PnbsGPLy8jBjxgy9daZNm6ab8KVWqwEA5ub/606NGzfG/fv3AQAWFhZwc3PD7NmzcebMGaxYsQLu7u4G7//hY1TW3bt3AQCdO3euMedGRIZj0qY668KFCwD0k/Sj/q+9ew+OqkrXP/50OhcJY2CEcA3CeILAeABRgYAIcrgJQ5DSEK4GgQHEgwKlM+jRn8XUQQEBmRlwVJCLDLd0oIRxAiJRtLiIaBUgMsiglGAgSmAETQK5vr8/LPpMi4EEknSv5Pupospee+2139473Y+99u7dhw8fVu3atbVkyZLr2ta6des0fPhwLVmyRG+++aZ8Pp969uxZYeOX1aXn3L59+2r33ICagAvRUGNdCuvjx4+X2ic6OlqZmZnKzMy8bNmZM2fKvK3o6Ght2bJFq1atUnh4uO677z4dPny4wsYvCzNTWlqaYmJiNHDgwGr13ICagtBGjdWuXTtJUmpqakB7SUmJiouLJUlt27aVmWn69OkBfU6fPq3ly5eXaTv5+flavHixJGnkyJHas2ePzEzbt28v8/hluSGKmV1x+fz583Xw4EHNmzdPTZs2DZnnBqDsmB5HjXX33XerZ8+eWrFihe68806NHj1ahw4d0s6dO5Wdna21a9dq0KBB6tixo9asWaOLFy9q8ODB+uKLL7R7926tXbtWklRYWChJAXdRu3DhgvLy8vyPly1bpkmTJsnr9apJkyaqU6eO7rjjDnXu3Pmq42dkZOjBBx/U0qVLlZSUVOrzuXS/9H/frvTjTML8+fO1aNEiTZkyRePHj5ck9enTJ+jPDUA5Be8iOKDiqZxf+Tp//ryNHTvWGjZsaDfffLPNmDHDJkyYYGPGjLGMjAwrLi62s2fP2siRI61BgwYWGxtrKSkpdvLkSTMzy8jIsPj4eJNkjz76qJ0+fdpWrlxpderUMUk2Y8YMy83NtY4dO1q/fv1s9uzZNmHCBHv99df9NVxpfDOz9957zxo3bmwbN24s9Xls3brVEhMTTZJJsm7dulmvXr1swIAB1r9/f5s2bZrt37//svWC/dzKiqvHATMz83nMrjKnBjjE4/EoNTVVycnJwS4FFcjn82no0KFXPQUAVHNpnNMGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBEeM7NgFwFUFI/Ho4SEBMXFxQW7FFSgzMxM7dmzR7xdoYZLCw92BUBFSkpKCnYJVebw4cOSpDZt2gS5ksoXFxdXo44tUBo+aQOOSk5OliT5fL4gVwKgiqRxThsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHhAe7AABXt3r1ai1dulQlJSX+tiNHjkiS7r33Xn9bWFiYxo0bp5EjR1Z1iQCqgMfMLNhFALiyAwcO6Pbbby9T3/3796t9+/aVXBGAIEhjehxwQPv27dWqVaur9ouPjyewgWqM0AYc8dBDDykiIqLU5RERERozZkwVVgSgqjE9Djji2LFjio+P15VeskePHlV8fHwVVgWgCjE9DrjilltuUYcOHeTxeC5b5vF4dOeddxLYQDVHaAMOSUlJkdfrvazd6/UqJSUlCBUBqEpMjwMOOX36tBo3bhzw1S/px696nTx5Uo0aNQpSZQCqANPjgEsaNGig7t27B3za9nq96tGjB4EN1ACENuCYhx56qExtAKofpscBx3z//feqX7++CgsLJf34Va/Tp0+rbt26Qa4MQCVjehxwTUxMjPr376/w8HCFh4drwIABBDZQQxDagINGjRql4uJiFRcXc59xoAbhB0NQqXw+X7BLqJYKCwsVGRkpM1N+fj77uZIkJycHuwQgAOe0Ual+7kYggCt4e0SI4Zw2Kl9qaqrMjH8V/G/Lli16++23g15HdfyXmpoa7JcN8LOYHgcc1bt372CXAKCKEdqAo8LDefkCNQ3T4wAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC04YScnJxglwAAQccvDiCkLVmyRD6fT59//rm+/vrrYJdTpc6dO6d58+apuLhYs2bNumz5hg0btGvXLklSVlaWHn30Ud1zzz3l2sbmzZv1yiuv6O9//7skqUuXLgoLC1Nubq6ioqLUo0cPTZgwQf/xH/9x/U8IwPUzoBJJstTU1Gtev6ioyLp162aNGjWqwKqqzqlTp65pvb/97W+WnJxskmzy5MmXLV+2bJm1bdvWiouLzczswIEDVrduXdu6dWu5t5WZmWmSrHnz5gHte/futfvuu8+8Xq/9z//8j39brrmWY5Cammq8PSIE+ZgeR0jzer2Ki4sLdhnX5LvvvtOoUaOuad3ExEQtWbLkZ5fl5ORo+vTpGjFihMLCfnwJt2vXTj179tQTTzwhMyvXtmrXri1JqlWrVkB7x44dlZ6ermHDhumFF17QnDlzruGZBNf1HAMgFBHaQCUoKCjQiBEjdOzYsWseIyoq6mfb9+7dq+zsbMXHxwe09+zZU5999pl27txZru14PJ5Sl4WFhenll19WgwYNNHPmTJ04caJcYwdTRRwDINQQ2gg5mzZt0oQJEzR9+nQ99thjysrK8i/75ptv9NJLL6ldu3bKyspS37591bx5c509e1bSj+d5J0+erCeffFL9+/fXs88+q/z8fEnSP/7xDz3zzDP69a9/rVOnTmnw4MG66aab1KlTJ+3ZsyeghiuNs3btWsXExKhZs2aSpO+//15//OMfdcMNN6hLly6SpLS0NB06dEhnzpzR+PHjNW/evArbP0ePHpUkRUZGBrQ3atRIknT48GFJ0q5du9SsWTNt2bLlurZXp04dJScnKy8vTz6fj2MABFOwJ+hRvamc57RXr15tnTt3tgsXLpiZWXZ2tsXGxvrPaW/ZssVat25tXq/XZsyYYUuXLrVOnTrZyZMnbcGCBda1a1crKCgwM7MzZ85Yy5YtrUePHlZSUmJPPfWU1a1b17xer02bNs22b99uGzZssPr161t0dLT/3OfVxjEz69u3r8XFxQXUftddd1lCQoL/8cCBA61FixbXuOfMLl68+LPntNeuXWuS7OWXXw5o37Ztm0myp59+2szM0tPTrVatWrZ69eorbufcuXMmyVq3bl1qn1WrVpkkGzNmTI04BpzTRojy8VeJSlWe0M7NzbXGjRvbmjVrAtofeOCBgAvRxo0bZ5Ls6NGj/rZvv/3WateubStXrgxYd/ny5SbJ/vrXv5qZ2YgRIywiIsIfBmZm69evN0n23HPPlXmcwYMHXxYYCQkJVRLaR44cMY/HY3369Alof+utt0ySzZo1y99WVFR01e2UJbS3bt1qkqxXr15mVv2PAaGNEMWFaAgdO3bsUFZWltq2bRvQ/tNp4IiICIWHhwec092zZ49yc3N18803B/QdOHCgJGn79u2SpOjoaHm9XkVERPj73H///YqKitLBgwfLPE4w3XrrrRo7dqy2bdumOXPm6Ny5c/r444/907/Nmzf39/V6vRWyzfPnz/u3LXEMgGDhe9oIGZ9//rmky0O6LI4fPy5J+te//hXQXr9+fUVHR+vUqVOlrhseHq4mTZqoqKjousapSosXL9Z//ud/asuWLfrggw/Ur18/tWzZUrt27VLfvn0rfHuXjk379u1L7VPTjgEQDIQ2QsalsD5+/Lj/E11Z/epXv5KkUq8Ubt269RXXz8vLU+vWra97nKoSFhamqVOnaurUqZJ+rL9p06Z64IEHVK9evQrdlpkpLS1NMTEx/k+7P6emHQMgGJgeR8ho166dJCk1NTWgvaSkRMXFxVdct0uXLoqJidHGjRsD2jMzM5WXl6dBgwaVum5WVpays7OVlJRU5nHCw8OVk5MTUFdOTo5KSkr8j8PCwlRYWHjFuivK448/LjPTSy+9FND+7/WUxq7yve758+fr4MGDmjdvnpo2bVpqv5p+DICqQGgjZNx9993q2bOnVqxYoVdeeUV5eXn6+OOPtXPnTmVnZ2vt2rXKy8tTUVGRiouLVVRU5F+3Xr16mjNnjnbt2qV3333X3/7nP/9Zo0ePVs+ePf1t+fn5OnDggP/xzJkzNXr0aHXq1KnM47Rt21bnzp3TrFmz9M9//lMzZ85Ufn6+jhw5on379kmSmjRpom+++Ub79+/X+++/r7y8vHLtj9zcXEnSxYsXr9hv7ty52rBhgzZv3hwQqhkZGfrlL3+p9evXX3H9S/d1/2l9x48f1+OPP67f//73mjJlisaPH+9fVlOOARBqmB5HSNm4caOmTZumP/zhD5o9e7bGjh2rgQMHqrCwUA0aNNCGDRuUnp4uM9OTTz6p8ePH67bbbpMkPfLII2rSpInmzp2rTZs2qW7dumrYsOFld/KKiIjQG2+8oczMTMXExKhFixZ65pln/MvLMs7UqVP1ySefaM6cOUpPT9fChQv15ZdfqqioSJmZmerQoYMmTZqk9PR0DR06VM8//7yio6PLvB927typFStWSJLeeustrVu3Tvfee6//u9iStG/fPv2///f/FB0drY8//viym614vV7Vrl074IKvn3rnnXe0aNEiSdKJEyd0zz33KCoqSlFRUTIztW7dWvv27Qs4l7169eoacQyAUOSxq82NAdfB4/EoNTVVycnJwS5FkjR+/HitWrVKFy5cCHYp12XhwoUKCwtT79691apVq2CXUy4uHAOfz6ehQ4eW+5awQCVL45M2UEViY2Ov2mfZsmVKTEy8ar/HHnusIkoC4BhCGzVKTk6OCgsLZWZXvOd2ZcjOzq7S7YWqYB4DwHVciIYaY+XKldq2bZuKi4v1xBNPaO/evcEuqcbhGADXh0/aqDFSUlKUkpIS7DJqNI4BcH34pA0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gl/5QqX78MMPg10CUC78zSJUeczMgl0Eqi+PxxPsEoBrxtsjQkwan7RRqXjTqzzJycmSJJ/PF+RKAFQVzmkDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgiPBgFwDg6j766CMdOHAgoO3YsWOSpMWLFwe0t2vXTgkJCVVWG4CqQ2gDDjh9+rQmTpwor9ersLAfJ8jMTJI0efJkSVJJSYmKi4v1t7/9LWh1AqhcHrv0ygcQsgoLC1W/fn19//33V+x344036syZM4qMjKyiygBUoTTOaQMOiIiI0LBhw64YxhERERo+fDiBDVRjhDbgiOHDh6ugoKDU5YWFhRoxYkQVVgSgqjE9DjiipKRETZo00bfffvuzy2NjY/XNN9/4z3kDqHaYHgdcERYWplGjRv3s9HdkZKRGjx5NYAPVHK9wwCGlTZEXFBRo+PDhQagIQFViehxwTHx8vL788suAtubNm+urr74KTkEAqgrT44BrRo0apYiICP/jyMhIjRkzJogVAagqfNIGHPPFF1+oZcuWAW1HjhzRrbfeGqSKAFQRPmkDromPj1e7du3k8Xjk8XjUrl07AhuoIQhtwEEpKSnyer3yer1KSUkJdjkAqgjT44CDTp06pWbNmsnMdOLECcXFxQW7JACVj+lxuGPIkCH+KeGa/q9p06YqKSmRmalZs2ZBrydU/g0ZMiTYf6ZApeJXvuCUhIQETZs2LdhlhISMjAx5PB716tUr2KWEhAULFgS7BKDSEdpwSlxcnJKTk4NdRki4FNb16tULciWhIS0tLdglAJWO0AYcRVgDNQ/ntAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbNVJOTk6wSwCAciO0UaMsWbJEffr0UZs2bYJdyjU5d+6cnn32WT399NPXtP7mzZuVmJgoj8cjj8ejrl27qlu3burQoYMSEhI0ffp0ffnllxVcNYCKQmijRhk7dqwuXryooqKiYJdSbm+99ZYmTpyo559//ppnCgYMGKBXX31VktS8eXPt3r1bO3fu1L59+7Rw4UJ9+umnatWqlZ555hmVlJRUZPkAKgChjRrF6/UqLi4u2GVck8TERC1ZsuS6x6ldu7YkqVatWgHtHTt2VHp6uoYNG6YXXnhBc+bMue5tAahYhDbgkKioqOsew+PxlLosLCxML7/8sho0aKCZM2fqxIkT1709ABWH0Ea1t2nTJk2YMEHTp0/XY489pqysrIDlZqZXX31VkyZNUufOndW3b18dPXpUkrR//3797ne/0y233KLvvvtODz/8sOrXr69OnTrp2LFj/jH279+vMWPGaM6cObr//vvVp0+fMo1fkXbt2qVmzZppy5Yt1zVOnTp1lJycrLy8PPl8PknVZx8BzjPAEUlJSZaUlFSudVavXm2dO3e2CxcumJlZdna2xcbGWqNGjfx9Zs2aZStWrDAzs6KiIktISLBGjRpZbm6uZWVlWe/evU2STZw40Q4dOmTbtm2zmJgYGzZsmH+MVq1a2c6dO83MLD8/3wYOHFim8cvr4sWLJskmT5582bL09HSrVauWrV69+opjnDt3ziRZ69atS+2zatUqk2Rjxoy56nMIlX10LX8fgGN8hDacUd435dzcXGvcuLGtWbMmoP2BBx7wh/bJkyetYcOGVlxc7F8+b948k2Tr1q0zM7Onn37aJNmZM2f8fX7zm99Yy5YtzcysoKDAPB6P/elPf/Ivf/vtt8s8fnlcKbTNfgy8qylLaG/dutUkWa9evZzZR4Q2agBfeDA+3QNVYceOHcrKylLbtm0D2iMjI/3/vXv3bhUWFmrixIkBfX7729/6L9Tyer2SpPDw/3u53Hjjjfrhhx8kSREREerbt6+mTp2qzz77TLNnz1a/fv3KPH5FulTr9Tp//rwk6dZbb612+whwGaGNauvzzz+XFBjSP3X48GHVrl37uq/KXrdunYYPH64lS5bozTfflM/nU8+ePSts/Kp2ad+1b9+efQSEEC5EQ7V1KayPHz9eap/o6GhlZmYqMzPzsmVnzpwp87aio6O1ZcsWrVq1SuHh4brvvvt0+PDhChu/KpmZ0tLSFBMTo4EDB7KPgBBCaKPaateunSQpNTU1oL2kpETFxcWSpLZt28rMNH369IA+p0+f1vLly8u0nfz8fC1evFiSNHLkSO3Zs0dmpu3bt1fI+OVRlhuimNkVl8+fP18HDx7UvHnz1LRp02q3jwCXMT2Oauvuu+9Wz549tWLFCt15550aPXq0Dh06pJ07dyo7O1tr167VoEGD1LFjR61Zs0YXL17U4MGD9cUXX2j37t1au3atJKmwsFCSAu6iduHCBeXl5fkfL1u2TJMmTZLX61WTJk1Up04d3XHHHercufNVxy+P3NxcSdLFixcvW5aRkaEHH3xQS5cuVVJSUqljXLqb2r/XL/04IzF//nwtWrRIU6ZM0fjx4yVJffr0cWofAdVa8C6CA8rnWq4OPn/+vI0dO9YaNmxoN998s82YMcMmTJhgY8aMsYyMDCsuLrazZ8/ayJEjrUGDBhYbG2spKSl28uRJMzPLyMiw+Ph4k2SPPvqonT592lauXGl16tQxSTZjxgzLzc21jh07Wr9+/Wz27Nk2YcIEe/311/01XGn88tixY4eNGzfOJFnDhg1t7dq1lpWV5V/+3nvvWePGjW3jxo2ljrF161ZLTEw0SSbJunXrZr169bIBAwZY//79bdq0abZ///7L1nNhH3H1OGoAn8fsKnNlQIgYMmSIJCktLS3IlSAU8feBGiCN6XEgiGJjY6/aZ9myZUpMTKyCagCEOkIbCKLs7OxglwDAIVw9DgCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjuD3tOGU9evXy+PxBLsMhKikpKRglwBUKo+ZWbCLAMriww8/1Ndffx3sMkLGggULJEnTpk0LciWho1mzZurSpUuwywAqSxqhDTgqOTlZkuTz+YJcCYAqksY5bQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjwoNdAICry8vLU35+fkBbQUGBJOm7774LaI+KilJ0dHSV1Qag6hDagAOWL1+uyZMn/+yym266KeDxokWL9N///d9VURaAKuYxMwt2EQCuLDs7W40bN1ZxcfEV+3m9XmVlZSk2NraKKgNQhdI4pw04IDY2Vv/1X/8lr9dbah+v16tevXoR2EA1RmgDjhg1apSuNDFmZho1alQVVgSgqjE9Djjihx9+UGxs7GUXpF0SGRmp7OxsxcTEVHFlAKoI0+OAK2688UYNHDhQERERly0LDw/XoEGDCGygmiO0AYeMHDlSRUVFl7UXFxdr5MiRQagIQFViehxwSEFBgerXr68ffvghoP0Xv/iFzpw5o6ioqCBVBqAKMD0OuCQyMlJJSUmKjIz0t0VERCg5OZnABmoAQhtwzIgRI/x3Q5OkwsJCjRgxIogVAagqTI8DjikpKVHDhg115swZSVK9evX07bffXvE73ACqBabHAdeEhYVp5MiRioyMVEREhEaNGkVgAzUEoQ04aPjw4SooKGBqHKhh+MEQVKohQ4YEu4Rq69Ivec2dOzfIlVRfaWlpwS4BCEBoo1KtX79eCQkJiouLC3Yp1U7z5s2DXUK1lZmZqT179gS7DOAyhDYq3bRp05ScnBzsMqqdQ4cOSZJuu+22IFdS/fh8Pg0dOjTYZQCXIbQBRxHWQM3DhWgAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMARhDYAAI4gtAEAcAShDSfk5OQEuwQACDpCGyFtyZIl6tOnj9q0aRPsUqrUmjVrdNdddykmJkadOnVSenr6ZX0++eQTPfjgg3ryySc1YcIEvfHGG+XezubNm5WYmCiPxyOPx6OuXbuqW7du6tChgxISEjR9+nR9+eUuarjvAAATBUlEQVSXFfGUAFQAfpoTIW3s2LFauXKlioqKgl3KNcnKylLjxo3Ltc6CBQu0bds2PfTQQ/rqq6+0ePFiJSYm6p133lHv3r0lSQcOHNC9996rd955R127dtXFixfVvn17XbhwQY888kiZtzVgwAC1b99ecXFxat68uXbv3u1f9vHHH+u5555Tq1atNH36dP3v//6vwsLc+//8azkGQKhy7xWIGsXr9SouLi7YZVyT7777TqNGjSrXOjk5Odq7d682b96sKVOmaMGCBXr33Xfl8Xg0d+5cf78nnnhCnTt3VteuXSVJN9xwg6ZMmaLf/e53+uGHH8q1zdq1a0uSatWqFdDesWNHpaena9iwYXrhhRc0Z86cco0bCq7lGAChjNAGKkFBQYFGjBihY8eOlWu9jz76SM8991xAW0JCgjp06KAvvvhC0o+fHN9991117949oF/37t2Vk5OjVatWlWubHo+n1GVhYWF6+eWX1aBBA82cOVMnTpwo19jBdK3HAAhlhDZCzqZNmzRhwgRNnz5djz32mLKysvzLvvnmG7300ktq166dsrKy1LdvXzVv3lxnz56VJG3YsEGTJ0/Wk08+qf79++vZZ59Vfn6+JOkf//iHnnnmGf3617/WqVOnNHjwYN10003q1KmT9uzZE1DDlcZZu3atYmJi1KxZM0nS999/rz/+8Y+64YYb1KVLF0lSWlqaDh06pDNnzmj8+PGaN29emZ57r169fvb8fZ06ddSiRQv/85Ckli1bBvS59PjSFPeuXbvUrFkzbdmypUzbLk2dOnWUnJysvLw8+Xy+an8MgJBmQCWSZKmpqWXuv3r1auvcubNduHDBzMyys7MtNjbWGjVqZGZmW7ZssdatW5vX67UZM2bY0qVLrVOnTnby5ElbsGCBde3a1QoKCszM7MyZM9ayZUvr0aOHlZSU2FNPPWV169Y1r9dr06ZNs+3bt9uGDRusfv36Fh0dbadOnTIzu+o4ZmZ9+/a1uLi4gNrvuusuS0hI8D8eOHCgtWjR4hr33P8pKiqy2NhYW7p0qZmZLVq0yCTZ3//+98v6RkVFWY8ePczMLD093WrVqmWrV6++4vjnzp0zSda6detS+6xatcok2ZgxY2rEMUhNTTXeHhGCfPxVolKVJ7Rzc3OtcePGtmbNmoD2Bx54wB/aZmbjxo0zSXb06FF/27fffmu1a9e2lStXBqy7fPlyk2R//etfzcxsxIgRFhER4Q8DM7P169ebJHvuuefKPM7gwYMvC4yEhIRKCe0333zTbr/9disqKjIzs6efftok2fvvv39Z33r16gWE76V1rqQsob1161aTZL169TKz6n8MCG2EKB/T4wgZO3bsUFZWltq2bRvQHhkZGfA4IiJC4eHhio+P97ft2bNHubm5uvnmmwP6Dhw4UJK0fft2SVJ0dLS8Xq8iIiL8fe6//35FRUXp4MGDZR6nqhQUFOjFF1+Uz+eT1+uVJP+UcF5e3mX98/LyAmq/tM71On/+vCTp1ltvlVSzjgEQSghthIzPP/9c0uUhXRbHjx+XJP3rX/8KaK9fv76io6N16tSpUtcNDw9XkyZNVFRUdF3jVIannnpKs2bNCjh/fSkoLwXpJQUFBbpw4YJatWpV4XVcOjbt27cvtU91PQZAKCG0ETIuhfWlN+3y+NWvfiVJpV4p3Lp16yuun5eXp9atW1/3OBXpL3/5i7p3764ePXoEtN92223yeDz66quvAtovPa7oGs1MaWlpiomJ8X/a/TnV8RgAoYbQRsho166dJCk1NTWgvaSkRMXFxVdct0uXLoqJidHGjRsD2jMzM5WXl6dBgwaVum5WVpays7OVlJRU5nHCw8OVk5MTUFdOTo5KSkr8j8PCwlRYWHjFukuzZs0a3XDDDRo8eHBA+44dO9SkSRN1795dH3zwQcCyDz74QJGRkXrwwQf9bf9eT2nM7IrL58+fr4MHD2revHlq2rRpqf2q2zEAQhGhjZBx9913q2fPnlqxYoVeeeUV5eXl6eOPP9bOnTuVnZ2ttWvXKi8vT0VFRSouLg64S1q9evU0Z84c7dq1S++++66//c9//rNGjx6tnj17+tvy8/N14MAB/+OZM2dq9OjR6tSpU5nHadu2rc6dO6dZs2bpn//8p2bOnKn8/HwdOXJE+/btkyQ1adJE33zzjfbv36/333//Z89B/5zNmzdr4cKFKiws1GuvvabXXntNr776qiZPnqxPP/1UkvTiiy9q586d/m0VFBRo4cKFevbZZ9WwYUNJUkZGhn75y19q/fr1V9zepfu6/7S+48eP6/HHH9fvf/97TZkyRePHj/cvq+7HAAhV3MYUIWXjxo2aNm2a/vCHP2j27NkaO3asBg4cqMLCQjVo0EAbNmxQenq6zExPPvmkxo8fr9tuu02S9Mgjj6hJkyaaO3euNm3apLp166phw4aX3ckrIiJCb7zxhjIzMxUTE6MWLVromWee8S8vyzhTp07VJ598ojlz5ig9PV0LFy7Ul19+qaKiImVmZqpDhw6aNGmS0tPTNXToUD3//POKjo6+6vP/+OOPlZSUpAsXLlz2veWoqCj/+dxOnTpp+/btmjNnjm655RZ99dVXmjhxoh599FF/f6/Xq9q1awdc8PVT77zzjhYtWiRJOnHihO655x5FRUUpKipKZqbWrVtr3759AeeyV69eXa2PARDKPHa1uTHgOng8HqWmpio5OTnYpUiSxo8fr1WrVunChQvBLqXGcuEY+Hw+DR069KqnDoAqlsYnbaCKxMbGXrXPsmXLlJiYWAXVAHARoY0aJScnR4WFhTKzK95zuzJkZ2dX6fZCVTCPAeA6LkRDjbFy5Upt27ZNxcXFeuKJJ7R3795gl1TjcAyA68MnbdQYKSkpSklJCXYZNRrHALg+fNIGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHOExMwt2Eai+PB6PEhISFBcXF+xSgDLLzMzUnj17xNsjQkwaP82JSpWUlBTsEqqtw4cPS5LatGkT5Eqqn7i4OP52EZL4pA04Kjk5WZLk8/mCXAmAKpLGOW0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcQWgDAOAIQhsAAEcQ2gAAOILQBgDAEYQ2AACOILQBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMARhDYAAI4gtAEAcAShDQCAIwhtAAAcER7sAgBc3erVq7V06VKVlJT4244cOSJJuvfee/1tYWFhGjdunEaOHFnVJQKoAh4zs2AXAeDKDhw4oNtvv71Mfffv36/27dtXckUAgiCN6XHAAe3bt1erVq2u2i8+Pp7ABqoxQhtwxEMPPaSIiIhSl0dERGjMmDFVWBGAqsb0OOCIY8eOKT4+Xld6yR49elTx8fFVWBWAKsT0OOCKW265RR06dJDH47lsmcfj0Z133klgA9UcoQ04JCUlRV6v97J2r9erlJSUIFQEoCoxPQ445PTp02rcuHHAV7+kH7/qdfLkSTVq1ChIlQGoAkyPAy5p0KCBunfvHvBp2+v1qkePHgQ2UAMQ2oBjHnrooTK1Aah+mB4HHPP999+rfv36KiwslPTjV71Onz6tunXrBrkyAJWM6XHANTExMerfv7/Cw8MVHh6uAQMGENhADUFoAw4aNWqUiouLVVxczH3GgRqEHwyBMz788EN9/fXXwS4jJBQWFioyMlJmpvz8fPl8vmCXFBKaNWumLl26BLsMoNJwThvOGDJkiNavXx/sMhDCkpKSlJaWFuwygMqSxidtOIU35f/z9ttvy+PxqF+/fsEuJSQMGTIk2CUAlY7QBhzVu3fvYJcAoIoR2oCjwsN5+QI1DVePAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtBGjZSTkxPsEgCg3Aht1ChLlixRnz591KZNm2CXUm5r1qzRXXfdpZiYGHXq1Enp6enlHmPz5s1KTEyUx+ORx+NR165d1a1bN3Xo0EEJCQmaPn26vvzyy0qoHkBFILRRo4wdO1YXL15UUVFRsEsplwULFmjVqlV66KGHNG7cOB06dEiJiYnKyMgo1zgDBgzQq6++Kklq3ry5du/erZ07d2rfvn1auHChPv30U7Vq1UrPPPOMSkpKKuOpALgO/LYfahSv16u4uDh98cUXwS6lzHJycrR3715t3rzZ3zZ06FDdfffdmjt3brl/V7t27dqSpFq1agW0d+zYUenp6UpJSdELL7ygX/ziF3r66aev/wkAqDB80gZC3EcffaTnnnsuoC0hIUEdOnS4pv/58Hg8pS4LCwvTyy+/rAYNGmjmzJk6ceJEuccHUHkIbVR7mzZt0oQJEzR9+nQ99thjysrKClhuZnr11Vc1adIkde7cWX379tXRo0clSfv379fvfvc73XLLLfruu+/08MMPq379+urUqZOOHTvmH2P//v0aM2aM5syZo/vvv199+vQp0/hl0atXr589B1+nTh21aNHC/3jXrl1q1qyZtmzZUuaxf06dOnWUnJysvLw8+Xy+qz6HUNhHQI1hgCOSkpIsKSmpXOusXr3aOnfubBcuXDAzs+zsbIuNjbVGjRr5+8yaNctWrFhhZmZFRUWWkJBgjRo1stzcXMvKyrLevXubJJs4caIdOnTItm3bZjExMTZs2DD/GK1atbKdO3eamVl+fr4NHDiwTONfq6KiIouNjbWlS5f629LT061WrVq2evXqK6577tw5k2StW7cutc+qVatMko0ZM+aqzyFU9tG1/H0AjvER2nBGed+Uc3NzrXHjxrZmzZqA9gceeMAf2idPnrSGDRtacXGxf/m8efNMkq1bt87MzJ5++mmTZGfOnPH3+c1vfmMtW7Y0M7OCggLzeDz2pz/9yb/87bffLvP41+LNN9+022+/3YqKigLaf/r455QltLdu3WqSrFevXs7sI0IbNYCPC9FQbe3YsUNZWVlq27ZtQHtkZKT/v3fv3q3CwkJNnDgxoM9vf/tb/4VaXq9XkhQe/n8vlxtvvFE//PCDJCkiIkJ9+/bV1KlT9dlnn2n27Nnq169fmccvr4KCAr344ovy+Xz+2i756eNrdf78eUnSrbfe6uQ+AqorQhvV1ueffy4pMKR/6vDhw6pdu7aWLFlyXdtat26dhg8friVLlujNN9+Uz+dTz549K2z8f/fUU09p1qxZatmyZYWN+VOX9l379u2d3EdAdcWFaKi2LoX18ePHS+0THR2tzMxMZWZmXrbszJkzZd5WdHS0tmzZolWrVik8PFz33XefDh8+XGHjX/KXv/xF3bt3V48ePcq9blmZmdLS0hQTE6OBAwc6t4+A6ozQRrXVrl07SVJqampAe0lJiYqLiyVJbdu2lZlp+vTpAX1Onz6t5cuXl2k7+fn5Wrx4sSRp5MiR2rNnj8xM27dvr5DxL1mzZo1uuOEGDR48OKB9x44dAc/taszsisvnz5+vgwcPat68eWratKlT+wio7pgeR7V19913q2fPnlqxYoXuvPNOjR49WocOHdLOnTuVnZ2ttWvXatCgQerYsaPWrFmjixcvavDgwfriiy+0e/durV27VpJUWFgoSQF3Ubtw4YLy8vL8j5ctW6ZJkybJ6/WqSZMmqlOnju644w517tz5quOXxebNm7Vw4UI9/PDDeu211yT9GL6fffaZ2rRpo3vuuUcZGRl68MEHtXTpUiUlJZU61qX7rv97/dKPMxLz58/XokWLNGXKFI0fP16S1KdPHyf2EVAjBO0aOKCcruXq4PPnz9vYsWOtYcOGdvPNN9uMGTNswoQJNmbMGMvIyLDi4mI7e/asjRw50ho0aGCxsbGWkpJiJ0+eNDOzjIwMi4+PN0n26KOP2unTp23lypVWp04dk2QzZsyw3Nxc69ixo/Xr189mz55tEyZMsNdff91fw5XGL4u9e/darVq1TNJl/6Kiouzs2bNmZvbee+9Z48aNbePGjaWOtXXrVktMTPSv361bN+vVq5cNGDDA+vfvb9OmTbP9+/dftl6o7yMzrh5HjeDzmF1lrgwIEUOGDJEkpaWlBbkShCL+PlADpDE9DgRRbGzsVfssW7ZMiYmJVVANgFBHaANBlJ2dHewSADiEq8cBAHAEoQ0AgCMIbQAAHEFoAwDgCEIbAABHENoAADiC0AYAwBGENgAAjiC0AQBwBKENAIAjCG0AABxBaAMA4AhCGwAARxDaAAA4gtAGAMAR/J42nJKZmSmfzxfsMhCCMjMzFRcXF+wygEpFaMMpe/bs0dChQ4NdBkJUUlJSsEsAKpXHzCzYRQAAgKtK45w2AACOILQBAHAEoQ0AgCMIbQAAHPH/AY9yEMObqrTQAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t5WqFsXIteSq"
      },
      "source": [
        "## Create TensorFlow Datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4j1gzPXPlge-",
        "outputId": "9cea563b-3529-4316-996f-b36ffe0acf01"
      },
      "source": [
        "train_features_ids, train_features_masks = create_bert_input_features(tokenizer, train_reviews, \n",
        "                                                                      max_seq_length=MAX_SEQ_LENGTH)\n",
        "val_features_ids, val_features_masks = create_bert_input_features(tokenizer, valid_reviews, \n",
        "                                                                  max_seq_length=MAX_SEQ_LENGTH)\n",
        "\n",
        "print('Train Features:', train_features_ids.shape, train_features_masks.shape)\n",
        "print('Val Features:', val_features_ids.shape, val_features_masks.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Converting docs to features: 100%|██████████| 67349/67349 [00:15<00:00, 4419.76it/s]\n",
            "Converting docs to features: 100%|██████████| 872/872 [00:00<00:00, 2785.91it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train Features: (67349, 128) (67349, 128)\n",
            "Val Features: (872, 128) (872, 128)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03gaQ-ems51N"
      },
      "source": [
        "# Create TensorFlow datasets for better performance\n",
        "train_ds = (\n",
        "    tf.data.Dataset\n",
        "    .from_tensor_slices(((train_features_ids, train_features_masks), train_sentiments))\n",
        "    .shuffle(2048)\n",
        "    .batch(24)\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        ")\n",
        "    \n",
        "valid_ds = (\n",
        "    tf.data.Dataset\n",
        "    .from_tensor_slices(((val_features_ids, val_features_masks), valid_sentiments))\n",
        "    .batch(24)\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xobIv2YJtgXf"
      },
      "source": [
        "## Train the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cnTMVebJlgfE",
        "outputId": "43b3c8a4-0e83-454b-8006-16ddf2a7c01f"
      },
      "source": [
        "start = time.time()\n",
        "model.fit(train_ds,\n",
        "    validation_data=valid_ds,\n",
        "    epochs=3)\n",
        "end = time.time()\n",
        "print(\"Training took {} seconds.\".format(end - start))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "2807/2807 [==============================] - 317s 113ms/step - loss: 0.2849 - accuracy: 0.8773 - val_loss: 0.2555 - val_accuracy: 0.8979\n",
            "Epoch 2/3\n",
            "2807/2807 [==============================] - 309s 110ms/step - loss: 0.1231 - accuracy: 0.9565 - val_loss: 0.2576 - val_accuracy: 0.9014\n",
            "Epoch 3/3\n",
            "2807/2807 [==============================] - 308s 110ms/step - loss: 0.0823 - accuracy: 0.9715 - val_loss: 0.3343 - val_accuracy: 0.9025\n",
            "Training took 934.4114644527435 seconds.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0tWZR3H0s51U",
        "outputId": "c2894129-d3dc-4acc-e22f-e3d4061d8a49"
      },
      "source": [
        "model.evaluate(valid_ds)[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "37/37 [==============================] - 1s 40ms/step - loss: 0.3343 - accuracy: 0.9025\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9025229215621948"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S5TdTdbdlgfH"
      },
      "source": [
        "model.save_weights('distillbert_ft_wts.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UmAPr3mTtjvm"
      },
      "source": [
        "## Model Conversion with TensorFlow Lite"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cRBedtQFq0Ha"
      },
      "source": [
        "Dynamic-range quantization is referred from [Hugging Face's official script](https://github.com/huggingface/tflite-android-transformers/blob/master/models_generation/distilbert.py)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qFfpNh7urdIC"
      },
      "source": [
        "### Dynamic-range"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KTofM0uXlgfT",
        "outputId": "9a69e158-d8da-41dd-ec93-e178de5fb350"
      },
      "source": [
        "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n",
        "converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, \n",
        "                                       tf.lite.OpsSet.SELECT_TF_OPS]\n",
        "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
        "tflite_model = converter.convert()\n",
        "open(\"distilbert_sst_seq_128.tflite\", \"wb\").write(tflite_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dec5c4d50>, because it is not built.\n",
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dec4a5a90>, because it is not built.\n",
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3b5710>, because it is not built.\n",
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3cc2d0>, because it is not built.\n",
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3c2210>, because it is not built.\n",
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3f2ad0>, because it is not built.\n",
            "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
            "WARNING:tensorflow:From /home/jupyter/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
            "INFO:tensorflow:Assets written to: /tmp/tmp4x_zcb2m/assets\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:absl:Using new converter: If you encounter a problem please file a bug. You can opt-out by setting experimental_new_converter=False\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "66977584"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gRCR-gLgrYHS",
        "outputId": "257d9fa1-2ff8-4cdc-b1df-62378605d6b3"
      },
      "source": [
        "!ls -lh distilbert_sst_seq_128.tflite"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 jupyter jupyter 64M Oct 12 02:00 distilbert_sst_seq_128.tflite\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wcw2xcRLrzTP"
      },
      "source": [
        "### Float16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E1MEg639rgWU",
        "outputId": "99714978-eef1-4f6e-a1b4-18c376907990"
      },
      "source": [
        "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n",
        "converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, \n",
        "                                       tf.lite.OpsSet.SELECT_TF_OPS]\n",
        "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
        "converter.target_spec.supported_types = [tf.float16]\n",
        "tflite_model = converter.convert()\n",
        "open(\"distilbert_sst_seq_128_fp16.tflite\", \"wb\").write(tflite_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dec5c4d50>, because it is not built.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dec5c4d50>, because it is not built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dec4a5a90>, because it is not built.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dec4a5a90>, because it is not built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3b5710>, because it is not built.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3b5710>, because it is not built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3cc2d0>, because it is not built.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3cc2d0>, because it is not built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3c2210>, because it is not built.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3c2210>, because it is not built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3f2ad0>, because it is not built.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f2dfc3f2ad0>, because it is not built.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /tmp/tmp8kp80euc/assets\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /tmp/tmp8kp80euc/assets\n",
            "INFO:absl:Using new converter: If you encounter a problem please file a bug. You can opt-out by setting experimental_new_converter=False\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "132845872"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ip8xKKPFrmUb",
        "outputId": "defed155-dd06-40fa-ee4f-1bc07327ea4e"
      },
      "source": [
        "!ls -lh distilbert_sst_seq_128_fp16.tflite"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 jupyter jupyter 127M Oct 12 02:01 distilbert_sst_seq_128_fp16.tflite\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-_LNbd7ksIss"
      },
      "source": [
        "### Integer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-d9RNsmsJ_j"
      },
      "source": [
        "# # ==============Representative dataset====================\n",
        "# train_features_ids = train_features_ids.astype(np.int32)\n",
        "# train_features_masks = train_features_masks.astype(np.int32)\n",
        "# train_tf_dataset = tf.data.Dataset.from_tensor_slices((train_features_ids, \n",
        "#     train_features_masks))\n",
        "\n",
        "# def representative_dataset_gen():\n",
        "#     for feature_id, feature_mask in train_tf_dataset.take(10):\n",
        "#         yield [feature_id, feature_mask]\n",
        "\n",
        "# # ==============Conversion====================\n",
        "# converter = tf.lite.TFLiteConverter.from_keras_model(model)\n",
        "# converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, \n",
        "#                                        tf.lite.OpsSet.SELECT_TF_OPS]\n",
        "# converter.representative_dataset = representative_dataset_gen\n",
        "# converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
        "# tflite_model = converter.convert()\n",
        "# open(\"distilbert_sst_seq_128_int.tflite\", \"wb\").write(tflite_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hWPYhR-UMybC"
      },
      "source": [
        "***Integer quantization isn't currently supported for this model.*** [Here](https://github.com/sayakpaul/BERT-for-Mobile/blob/master/Evaluation_SST_2_DistilBERT.ipynb) you can find the evaluation notebook. "
      ]
    }
  ]
}