diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 0f8934c57..44157ceb9 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -40,4 +40,4 @@ jobs:
uv pip install --system "numpy<2" ".[dev]"
- name: Tests
- run: nbdev_test --do_print --timing --n_workers 0 --flags polars
+ run: nbdev_test --do_print --timing --n_workers 0 --flags polars
\ No newline at end of file
diff --git a/action_files/test_models/src/evaluation.py b/action_files/test_models/src/evaluation.py
index e93d0d9e9..cda6e059b 100644
--- a/action_files/test_models/src/evaluation.py
+++ b/action_files/test_models/src/evaluation.py
@@ -41,9 +41,12 @@ def evaluate(model: str, dataset: str, group: str):
if __name__ == '__main__':
groups = ['Monthly']
- models = ['AutoDilatedRNN', 'RNN', 'TCN', 'DeepAR',
+ models = ['AutoDilatedRNN', 'RNN',
+ 'TCN',
+ 'DeepAR',
'NHITS', 'TFT', 'AutoMLP', 'DLinear', 'VanillaTransformer',
- 'BiTCN', 'TiDE', 'DeepNPTS', 'NBEATS', 'KAN']
+ 'BiTCN', 'TiDE', 'DeepNPTS', 'NBEATS', 'KAN'
+ ]
datasets = ['M3']
evaluation = [evaluate(model, dataset, group) for model, group in product(models, groups) for dataset in datasets]
evaluation = [eval_ for eval_ in evaluation if eval_ is not None]
diff --git a/action_files/test_models/src/models.py b/action_files/test_models/src/models.py
index ec32b5a82..96a1a0a3d 100644
--- a/action_files/test_models/src/models.py
+++ b/action_files/test_models/src/models.py
@@ -61,21 +61,22 @@ def main(dataset: str = 'M3', group: str = 'Monthly') -> None:
"random_seed": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
}
config_drnn = {'input_size': tune.choice([2 * horizon]),
- 'encoder_hidden_size': tune.choice([124]),
+ 'encoder_hidden_size': tune.choice([16]),
"max_steps": 300,
"val_check_steps": 100,
- "random_seed": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),}
+ "random_seed": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+ "scaler_type": "minmax1"}
models = [
AutoDilatedRNN(h=horizon, loss=MAE(), config=config_drnn, num_samples=2, cpus=1),
- RNN(h=horizon, input_size=2 * horizon, encoder_hidden_size=50, max_steps=300),
- TCN(h=horizon, input_size=2 * horizon, encoder_hidden_size=20, max_steps=300),
+ RNN(h=horizon, input_size=2 * horizon, encoder_hidden_size=64, max_steps=300),
+ TCN(h=horizon, input_size=2 * horizon, encoder_hidden_size=64, max_steps=300),
NHITS(h=horizon, input_size=2 * horizon, dropout_prob_theta=0.5, loss=MAE(), max_steps=1000, val_check_steps=500),
AutoMLP(h=horizon, loss=MAE(), config=config, num_samples=2, cpus=1),
DLinear(h=horizon, input_size=2 * horizon, loss=MAE(), max_steps=2000, val_check_steps=500),
TFT(h=horizon, input_size=2 * horizon, loss=SMAPE(), hidden_size=64, scaler_type='robust', windows_batch_size=512, max_steps=1500, val_check_steps=500),
VanillaTransformer(h=horizon, input_size=2 * horizon, loss=MAE(), hidden_size=64, scaler_type='minmax1', windows_batch_size=512, max_steps=1500, val_check_steps=500),
- DeepAR(h=horizon, input_size=2 * horizon, scaler_type='minmax1', max_steps=1000),
+ DeepAR(h=horizon, input_size=2 * horizon, scaler_type='minmax1', max_steps=500),
BiTCN(h=horizon, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=500),
TiDE(h=horizon, input_size=2 * horizon, loss=MAE(), max_steps=1000, val_check_steps=500),
DeepNPTS(h=horizon, input_size=2 * horizon, loss=MAE(), max_steps=1000, val_check_steps=500),
diff --git a/action_files/test_models/src/models2.py b/action_files/test_models/src/models2.py
index b309003fb..fe1fbfb6e 100644
--- a/action_files/test_models/src/models2.py
+++ b/action_files/test_models/src/models2.py
@@ -2,35 +2,39 @@
import time
import fire
-import numpy as np
+# import numpy as np
import pandas as pd
-import pytorch_lightning as pl
-import torch
+# import pytorch_lightning as pl
+# import torch
-import neuralforecast
+# import neuralforecast
from neuralforecast.core import NeuralForecast
from neuralforecast.models.gru import GRU
-from neuralforecast.models.rnn import RNN
-from neuralforecast.models.tcn import TCN
+# from neuralforecast.models.rnn import RNN
+# from neuralforecast.models.tcn import TCN
from neuralforecast.models.lstm import LSTM
from neuralforecast.models.dilated_rnn import DilatedRNN
-from neuralforecast.models.deepar import DeepAR
-from neuralforecast.models.mlp import MLP
-from neuralforecast.models.nhits import NHITS
-from neuralforecast.models.nbeats import NBEATS
+# from neuralforecast.models.deepar import DeepAR
+# from neuralforecast.models.mlp import MLP
+# from neuralforecast.models.nhits import NHITS
+# from neuralforecast.models.nbeats import NBEATS
from neuralforecast.models.nbeatsx import NBEATSx
-from neuralforecast.models.tft import TFT
-from neuralforecast.models.vanillatransformer import VanillaTransformer
-from neuralforecast.models.informer import Informer
-from neuralforecast.models.autoformer import Autoformer
-from neuralforecast.models.patchtst import PatchTST
+# from neuralforecast.models.tft import TFT
+# from neuralforecast.models.vanillatransformer import VanillaTransformer
+# from neuralforecast.models.informer import Informer
+# from neuralforecast.models.autoformer import Autoformer
+# from neuralforecast.models.patchtst import PatchTST
from neuralforecast.auto import (
- AutoMLP, AutoNHITS, AutoNBEATS, AutoDilatedRNN, AutoTFT
+ # AutoMLP,
+ AutoNHITS,
+ AutoNBEATS,
+ # AutoDilatedRNN,
+ # AutoTFT
)
-from neuralforecast.losses.pytorch import SMAPE, MAE
+from neuralforecast.losses.pytorch import MAE
from ray import tune
from src.data import get_data
@@ -49,32 +53,18 @@ def main(dataset: str = 'M3', group: str = 'Monthly') -> None:
"scaler_type": "minmax1",
"random_seed": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
}
- config = {
- "hidden_size": tune.choice([256, 512]),
- "num_layers": tune.choice([2, 4]),
- "input_size": tune.choice([2 * horizon]),
- "max_steps": 1000,
- "val_check_steps": 300,
- "scaler_type": "minmax1",
- "random_seed": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
- }
- config_drnn = {'input_size': tune.choice([2 * horizon]),
- 'encoder_hidden_size': tune.choice([124]),
- "max_steps": 300,
- "val_check_steps": 100,
- "random_seed": tune.choice([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),}
models = [
- LSTM(h=horizon, input_size=2 * horizon, encoder_hidden_size=50, max_steps=300),
- DilatedRNN(h=horizon, input_size=2 * horizon, encoder_hidden_size=50, max_steps=300),
- GRU(h=horizon, input_size=2 * horizon, encoder_hidden_size=50, max_steps=300),
+ LSTM(h=horizon, input_size=2 * horizon, encoder_hidden_size=64, max_steps=300),
+ DilatedRNN(h=horizon, input_size=2 * horizon, encoder_hidden_size=64, max_steps=300),
+ GRU(h=horizon, input_size=2 * horizon, encoder_hidden_size=64, max_steps=300),
AutoNBEATS(h=horizon, loss=MAE(), config=config_nbeats, num_samples=2, cpus=1),
AutoNHITS(h=horizon, loss=MAE(), config=config_nbeats, num_samples=2, cpus=1),
NBEATSx(h=horizon, input_size=2 * horizon, loss=MAE(), max_steps=1000),
- PatchTST(h=horizon, input_size=2 * horizon, patch_len=4, stride=4, loss=MAE(), scaler_type='minmax1', windows_batch_size=512, max_steps=1000, val_check_steps=500),
+ # PatchTST(h=horizon, input_size=2 * horizon, patch_len=4, stride=4, loss=MAE(), scaler_type='minmax1', windows_batch_size=512, max_steps=1000, val_check_steps=500),
]
# Models
- for model in models[:-1]:
+ for model in models:
model_name = type(model).__name__
print(50*'-', model_name, 50*'-')
start = time.time()
diff --git a/action_files/test_models/src/multivariate_models.py b/action_files/test_models/src/multivariate_models.py
index 1b1d9593b..8b1577a57 100644
--- a/action_files/test_models/src/multivariate_models.py
+++ b/action_files/test_models/src/multivariate_models.py
@@ -10,7 +10,7 @@
from neuralforecast.models.tsmixer import TSMixer
from neuralforecast.models.tsmixerx import TSMixerx
from neuralforecast.models.itransformer import iTransformer
-# from neuralforecast.models.stemgnn import StemGNN
+# # from neuralforecast.models.stemgnn import StemGNN
from neuralforecast.models.mlpmultivariate import MLPMultivariate
from neuralforecast.models.timemixer import TimeMixer
@@ -26,13 +26,13 @@ def main(dataset: str = 'multivariate', group: str = 'ETTm2') -> None:
train['ds'] = pd.to_datetime(train['ds'])
models = [
- SOFTS(h=horizon, n_series=7, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=500),
- TSMixer(h=horizon, n_series=7, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=500),
- TSMixerx(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=500),
- iTransformer(h=horizon, n_series=7, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=500),
- # StemGNN(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), dropout_rate=0.0, max_steps=1000, val_check_steps=500),
- MLPMultivariate(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), max_steps=1000, val_check_steps=500),
- TimeMixer(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=500)
+ SOFTS(h=horizon, n_series=7, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=500, val_check_steps=100, windows_batch_size=64, inference_windows_batch_size=64),
+ TSMixer(h=horizon, n_series=7, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=100, windows_batch_size=64, inference_windows_batch_size=64),
+ TSMixerx(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), dropout=0.0, max_steps=1000, val_check_steps=100, windows_batch_size=64, inference_windows_batch_size=64),
+ iTransformer(h=horizon, n_series=7, input_size=2 * horizon, loss=MAE(), dropout=0.0, max_steps=500, val_check_steps=100, windows_batch_size=64, inference_windows_batch_size=64),
+ # StemGNN(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), dropout_rate=0.0, max_steps=1000, val_check_steps=500, windows_batch_size=64, inference_windows_batch_size=64),
+ MLPMultivariate(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), max_steps=1000, val_check_steps=100, windows_batch_size=64, inference_windows_batch_size=64),
+ TimeMixer(h=horizon, n_series=7, input_size=2*horizon, loss=MAE(), dropout=0.0, max_steps=500, val_check_steps=100, windows_batch_size=64, inference_windows_batch_size=64)
]
# Models
diff --git a/nbs/common.base_auto.ipynb b/nbs/common.base_auto.ipynb
index e120c2f33..16db978b4 100644
--- a/nbs/common.base_auto.ipynb
+++ b/nbs/common.base_auto.ipynb
@@ -238,7 +238,11 @@
" self.callbacks = callbacks\n",
"\n",
" # Base Class attributes\n",
- " self.SAMPLING_TYPE = cls_model.SAMPLING_TYPE\n",
+ " self.EXOGENOUS_FUTR = cls_model.EXOGENOUS_FUTR\n",
+ " self.EXOGENOUS_HIST = cls_model.EXOGENOUS_HIST\n",
+ " self.EXOGENOUS_STAT = cls_model.EXOGENOUS_STAT\n",
+ " self.MULTIVARIATE = cls_model.MULTIVARIATE \n",
+ " self.RECURRENT = cls_model.RECURRENT \n",
"\n",
" def __repr__(self):\n",
" return type(self).__name__ if self.alias is None else self.alias\n",
diff --git a/nbs/common.base_model.ipynb b/nbs/common.base_model.ipynb
index 2ae169f8f..fae60e40c 100644
--- a/nbs/common.base_model.ipynb
+++ b/nbs/common.base_model.ipynb
@@ -36,19 +36,25 @@
"from contextlib import contextmanager\n",
"from copy import deepcopy\n",
"from dataclasses import dataclass\n",
+ "from typing import List, Dict, Union\n",
"\n",
"import fsspec\n",
"import numpy as np\n",
"import torch\n",
"import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
"import pytorch_lightning as pl\n",
+ "import neuralforecast.losses.pytorch as losses\n",
+ "\n",
+ "from neuralforecast.losses.pytorch import BasePointLoss, DistributionLoss\n",
"from pytorch_lightning.callbacks.early_stopping import EarlyStopping\n",
"from neuralforecast.tsdataset import (\n",
" TimeSeriesDataModule,\n",
" BaseTimeSeriesDataset,\n",
" _DistributedTimeSeriesDataModule,\n",
")\n",
- "from neuralforecast.losses.pytorch import IQLoss"
+ "from neuralforecast.common._scalers import TemporalNorm\n",
+ "from neuralforecast.utils import get_indexer_raise_missing"
]
},
{
@@ -112,27 +118,92 @@
"source": [
"#| export\n",
"class BaseModel(pl.LightningModule):\n",
- " EXOGENOUS_FUTR = True\n",
- " EXOGENOUS_HIST = True\n",
- " EXOGENOUS_STAT = True\n",
+ " EXOGENOUS_FUTR = True # If the model can handle future exogenous variables\n",
+ " EXOGENOUS_HIST = True # If the model can handle historical exogenous variables\n",
+ " EXOGENOUS_STAT = True # If the model can handle static exogenous variables\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(\n",
" self,\n",
- " random_seed,\n",
- " loss,\n",
- " valid_loss,\n",
- " optimizer,\n",
- " optimizer_kwargs,\n",
- " lr_scheduler,\n",
- " lr_scheduler_kwargs,\n",
- " futr_exog_list,\n",
- " hist_exog_list,\n",
- " stat_exog_list,\n",
- " max_steps,\n",
- " early_stop_patience_steps,\n",
+ " h: int,\n",
+ " input_size: int,\n",
+ " loss: Union[BasePointLoss, DistributionLoss, nn.Module],\n",
+ " valid_loss: Union[BasePointLoss, DistributionLoss, nn.Module],\n",
+ " learning_rate: float,\n",
+ " max_steps: int,\n",
+ " val_check_steps: int,\n",
+ " batch_size: int,\n",
+ " valid_batch_size: Union[int, None],\n",
+ " windows_batch_size: int,\n",
+ " inference_windows_batch_size: Union[int, None],\n",
+ " start_padding_enabled: bool,\n",
+ " n_series: Union[int, None] = None,\n",
+ " n_samples: Union[int, None] = 100,\n",
+ " h_train: int = 1,\n",
+ " inference_input_size: Union[int, None] = None,\n",
+ " step_size: int = 1,\n",
+ " num_lr_decays: int = 0,\n",
+ " early_stop_patience_steps: int = -1,\n",
+ " scaler_type: str = 'identity',\n",
+ " futr_exog_list: Union[List, None] = None,\n",
+ " hist_exog_list: Union[List, None] = None,\n",
+ " stat_exog_list: Union[List, None] = None,\n",
+ " exclude_insample_y: Union[bool, None] = False,\n",
+ " num_workers_loader: Union[int, None] = 0,\n",
+ " drop_last_loader: Union[bool, None] = False,\n",
+ " random_seed: Union[int, None] = 1,\n",
+ " alias: Union[str, None] = None,\n",
+ " optimizer: Union[torch.optim.Optimizer, None] = None,\n",
+ " optimizer_kwargs: Union[Dict, None] = None,\n",
+ " lr_scheduler: Union[torch.optim.lr_scheduler.LRScheduler, None] = None,\n",
+ " lr_scheduler_kwargs: Union[Dict, None] = None,\n",
+ " dataloader_kwargs=None,\n",
" **trainer_kwargs,\n",
" ):\n",
" super().__init__()\n",
+ "\n",
+ " # Multivarariate checks\n",
+ " if self.MULTIVARIATE and n_series is None:\n",
+ " raise Exception(f'{type(self).__name__} is a multivariate model. Please set n_series to the number of unique time series in your dataset.')\n",
+ " if not self.MULTIVARIATE:\n",
+ " if n_series is not None:\n",
+ " warnings.warn(\n",
+ " f'{type(self).__name__} is a univariate model. Parameter n_series is ignored.'\n",
+ " )\n",
+ " n_series = 1\n",
+ " self.n_series = n_series \n",
+ "\n",
+ " # Protections for previous recurrent models\n",
+ " if input_size < 1:\n",
+ " input_size = 3 * h\n",
+ " warnings.warn(\n",
+ " f'Input size too small. Automatically setting input size to 3 * horizon = {input_size}'\n",
+ " )\n",
+ "\n",
+ " if inference_input_size is None:\n",
+ " inference_input_size = input_size \n",
+ " elif inference_input_size is not None and inference_input_size < 1:\n",
+ " inference_input_size = input_size\n",
+ " warnings.warn(\n",
+ " f'Inference input size too small. Automatically setting inference input size to input_size = {input_size}'\n",
+ " )\n",
+ "\n",
+ " # For recurrent models we need one additional input as we need to shift insample_y to use it as input\n",
+ " if self.RECURRENT:\n",
+ " input_size += 1\n",
+ " inference_input_size += 1\n",
+ "\n",
+ " # Attributes needed for recurrent models\n",
+ " self.horizon_backup = h\n",
+ " self.input_size_backup = input_size\n",
+ " self.n_samples = n_samples\n",
+ " if self.RECURRENT:\n",
+ " self.h_train = h_train\n",
+ " self.inference_input_size = inference_input_size\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = False\n",
+ " \n",
" with warnings.catch_warnings(record=False):\n",
" warnings.filterwarnings('ignore')\n",
" # the following line issues a warning about the loss attribute being saved\n",
@@ -147,8 +218,8 @@
" self.valid_loss = loss\n",
" else:\n",
" self.valid_loss = valid_loss\n",
- " self.train_trajectories = []\n",
- " self.valid_trajectories = []\n",
+ " self.train_trajectories: List = []\n",
+ " self.valid_trajectories: List = []\n",
"\n",
" # Optimization\n",
" if optimizer is not None and not issubclass(optimizer, torch.optim.Optimizer):\n",
@@ -162,7 +233,6 @@
" self.lr_scheduler = lr_scheduler\n",
" self.lr_scheduler_kwargs = lr_scheduler_kwargs if lr_scheduler_kwargs is not None else {}\n",
"\n",
- "\n",
" # Variables\n",
" self.futr_exog_list = list(futr_exog_list) if futr_exog_list is not None else []\n",
" self.hist_exog_list = list(hist_exog_list) if hist_exog_list is not None else []\n",
@@ -181,12 +251,28 @@
" if not self.EXOGENOUS_STAT and self.stat_exog_size > 0:\n",
" raise Exception(f'{type(self).__name__} does not support static exogenous variables.')\n",
"\n",
- " # Implicit Quantile Loss\n",
- " if isinstance(self.loss, IQLoss):\n",
- " if not isinstance(self.valid_loss, IQLoss):\n",
- " raise Exception('Please set valid_loss to IQLoss() when training with IQLoss')\n",
- " if isinstance(self.valid_loss, IQLoss) and not isinstance(self.loss, IQLoss):\n",
- " raise Exception('Please set loss to IQLoss() when validating with IQLoss') \n",
+ " # Protections for loss functions\n",
+ " if isinstance(self.loss, (losses.IQLoss, losses.MQLoss, losses.HuberMQLoss)):\n",
+ " loss_type = type(self.loss)\n",
+ " if not isinstance(self.valid_loss, loss_type):\n",
+ " raise Exception(f'Please set valid_loss={type(self.loss).__name__}() when training with {type(self.loss).__name__}')\n",
+ " if isinstance(self.valid_loss, losses.IQLoss):\n",
+ " valid_loss_type = type(self.valid_loss)\n",
+ " if not isinstance(self.loss, valid_loss_type):\n",
+ " raise Exception(f'Please set loss={type(self.valid_loss).__name__}() when validating with {type(self.valid_loss).__name__}') \n",
+ "\n",
+ " # Deny impossible loss / valid_loss combinations\n",
+ " if isinstance(self.loss, losses.BasePointLoss) and self.valid_loss.is_distribution_output:\n",
+ " raise Exception(f'Validation with distribution loss {type(self.valid_loss).__name__} is not possible when using loss={type(self.loss).__name__}. Please use a point valid_loss (MAE, MSE, ...)')\n",
+ " elif self.valid_loss.is_distribution_output and self.valid_loss is not loss:\n",
+ " # Maybe we should raise a Warning or an Exception here, but meh for now.\n",
+ " self.valid_loss = loss\n",
+ " \n",
+ " if isinstance(self.loss, (losses.relMSE, losses.Accuracy, losses.sCRPS)):\n",
+ " raise Exception(f\"{type(self.loss).__name__} cannot be used for training. Please use another loss function (MAE, MSE, ...)\")\n",
+ " \n",
+ " if isinstance(self.valid_loss, (losses.relMSE)):\n",
+ " raise Exception(f\"{type(self.valid_loss).__name__} cannot be used for validation. Please use another valid_loss (MAE, MSE, ...)\")\n",
"\n",
" ## Trainer arguments ##\n",
" # Max steps, validation steps and check_val_every_n_epoch\n",
@@ -217,7 +303,73 @@
" if trainer_kwargs.get('enable_checkpointing', None) is None:\n",
" trainer_kwargs['enable_checkpointing'] = False\n",
"\n",
+ " # Set other attributes\n",
" self.trainer_kwargs = trainer_kwargs\n",
+ " self.h = h\n",
+ " self.input_size = input_size\n",
+ " self.windows_batch_size = windows_batch_size\n",
+ " self.start_padding_enabled = start_padding_enabled\n",
+ "\n",
+ " # Padder to complete train windows, \n",
+ " # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]\n",
+ " if start_padding_enabled:\n",
+ " self.padder_train = nn.ConstantPad1d(padding=(self.input_size-1, self.h), value=0.0)\n",
+ " else:\n",
+ " self.padder_train = nn.ConstantPad1d(padding=(0, self.h), value=0.0)\n",
+ "\n",
+ " # Batch sizes\n",
+ " if self.MULTIVARIATE and n_series is not None:\n",
+ " self.batch_size = max(batch_size, n_series)\n",
+ " else:\n",
+ " self.batch_size = batch_size\n",
+ " if valid_batch_size is None:\n",
+ " self.valid_batch_size = batch_size\n",
+ " else:\n",
+ " self.valid_batch_size = valid_batch_size\n",
+ " if inference_windows_batch_size is None:\n",
+ " self.inference_windows_batch_size = windows_batch_size\n",
+ " else:\n",
+ " self.inference_windows_batch_size = inference_windows_batch_size\n",
+ "\n",
+ " # Optimization \n",
+ " self.learning_rate = learning_rate\n",
+ " self.max_steps = max_steps\n",
+ " self.num_lr_decays = num_lr_decays\n",
+ " self.lr_decay_steps = (\n",
+ " max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7\n",
+ " )\n",
+ " self.early_stop_patience_steps = early_stop_patience_steps\n",
+ " self.val_check_steps = val_check_steps\n",
+ " self.windows_batch_size = windows_batch_size\n",
+ " self.step_size = step_size\n",
+ " \n",
+ " # If the model does not support exogenous, it can't support exclude_insample_y\n",
+ " if exclude_insample_y and not (self.EXOGENOUS_FUTR or self.EXOGENOUS_HIST or self.EXOGENOUS_STAT):\n",
+ " raise Exception(f'{type(self).__name__} does not support `exclude_insample_y=True`. Please set `exclude_insample_y=False`')\n",
+ "\n",
+ " self.exclude_insample_y = exclude_insample_y\n",
+ "\n",
+ " # Scaler\n",
+ " self.scaler = TemporalNorm(\n",
+ " scaler_type=scaler_type,\n",
+ " dim=1, # Time dimension is 1.\n",
+ " num_features= 1 + len(self.hist_exog_list) + len(self.futr_exog_list)\n",
+ " )\n",
+ "\n",
+ " # Fit arguments\n",
+ " self.val_size = 0\n",
+ " self.test_size = 0\n",
+ "\n",
+ " # Model state\n",
+ " self.decompose_forecast = False\n",
+ "\n",
+ " # DataModule arguments\n",
+ " self.num_workers_loader = num_workers_loader\n",
+ " self.dataloader_kwargs = dataloader_kwargs\n",
+ " self.drop_last_loader = drop_last_loader\n",
+ " # used by on_validation_epoch_end hook\n",
+ " self.validation_step_outputs: List = []\n",
+ " self.alias = alias\n",
"\n",
" def __repr__(self):\n",
" return type(self).__name__ if self.alias is None else self.alias\n",
@@ -246,21 +398,11 @@
" set(temporal_cols.tolist()) & set(self.hist_exog_list + self.futr_exog_list)\n",
" )\n",
" \n",
- " def _set_quantile_for_iqloss(self, **data_module_kwargs):\n",
- " if \"quantile\" in data_module_kwargs:\n",
- " if not isinstance(self.loss, IQLoss):\n",
- " raise Exception(\n",
- " \"Please train with loss=IQLoss() to make use of the quantile argument.\"\n",
- " )\n",
- " else:\n",
- " self.quantile = data_module_kwargs[\"quantile\"]\n",
- " data_module_kwargs.pop(\"quantile\")\n",
- " self.loss.update_quantile(q=self.quantile)\n",
- " elif isinstance(self.loss, IQLoss):\n",
- " self.quantile = 0.5\n",
- " self.loss.update_quantile(q=self.quantile)\n",
- "\n",
- " return data_module_kwargs\n",
+ " def _set_quantiles(self, quantiles=None):\n",
+ " if quantiles is None and isinstance(self.loss, losses.IQLoss):\n",
+ " self.loss.update_quantile(q=[0.5])\n",
+ " elif hasattr(self.loss, 'update_quantile') and callable(self.loss.update_quantile):\n",
+ " self.loss.update_quantile(q=quantiles)\n",
"\n",
" def _fit_distributed(\n",
" self,\n",
@@ -480,7 +622,792 @@
" model.load_state_dict(content[\"state_dict\"], strict=True, assign=True)\n",
" else: # pytorch<2.1\n",
" model.load_state_dict(content[\"state_dict\"], strict=True)\n",
- " return model"
+ " return model\n",
+ "\n",
+ " def _create_windows(self, batch, step, w_idxs=None):\n",
+ " # Parse common data\n",
+ " window_size = self.input_size + self.h\n",
+ " temporal_cols = batch['temporal_cols']\n",
+ " temporal = batch['temporal'] \n",
+ "\n",
+ " if step == 'train':\n",
+ " if self.val_size + self.test_size > 0:\n",
+ " cutoff = -self.val_size - self.test_size\n",
+ " temporal = temporal[:, :, :cutoff]\n",
+ "\n",
+ " temporal = self.padder_train(temporal)\n",
+ " \n",
+ " if temporal.shape[-1] < window_size:\n",
+ " raise Exception('Time series is too short for training, consider setting a smaller input size or set start_padding_enabled=True')\n",
+ " \n",
+ " windows = temporal.unfold(dimension=-1, \n",
+ " size=window_size, \n",
+ " step=self.step_size)\n",
+ "\n",
+ " if self.MULTIVARIATE:\n",
+ " # [n_series, C, Ws, L + h] -> [Ws, L + h, C, n_series]\n",
+ " windows = windows.permute(2, 3, 1, 0)\n",
+ " else:\n",
+ " # [n_series, C, Ws, L + h] -> [Ws * n_series, L + h, C, 1]\n",
+ " windows_per_serie = windows.shape[2]\n",
+ " windows = windows.permute(0, 2, 3, 1)\n",
+ " windows = windows.flatten(0, 1)\n",
+ " windows = windows.unsqueeze(-1)\n",
+ "\n",
+ " # Sample and Available conditions\n",
+ " available_idx = temporal_cols.get_loc('available_mask') \n",
+ " available_condition = windows[:, :self.input_size, available_idx]\n",
+ " available_condition = torch.sum(available_condition, axis=(1, -1)) # Sum over time & series dimension\n",
+ " final_condition = (available_condition > 0)\n",
+ " \n",
+ " if self.h > 0:\n",
+ " sample_condition = windows[:, self.input_size:, available_idx]\n",
+ " sample_condition = torch.sum(sample_condition, axis=(1, -1)) # Sum over time & series dimension\n",
+ " final_condition = (sample_condition > 0) & (available_condition > 0)\n",
+ " \n",
+ " windows = windows[final_condition]\n",
+ " \n",
+ " # Parse Static data to match windows\n",
+ " static = batch.get('static', None)\n",
+ " static_cols=batch.get('static_cols', None)\n",
+ "\n",
+ " # Repeat static if univariate: [n_series, S] -> [Ws * n_series, S]\n",
+ " if static is not None and not self.MULTIVARIATE:\n",
+ " static = torch.repeat_interleave(static, \n",
+ " repeats=windows_per_serie, dim=0)\n",
+ " static = static[final_condition] \n",
+ "\n",
+ " # Protection of empty windows\n",
+ " if final_condition.sum() == 0:\n",
+ " raise Exception('No windows available for training')\n",
+ "\n",
+ " # Sample windows\n",
+ " if self.windows_batch_size is not None:\n",
+ " n_windows = windows.shape[0]\n",
+ " w_idxs = np.random.choice(n_windows, \n",
+ " size=self.windows_batch_size,\n",
+ " replace=(n_windows < self.windows_batch_size))\n",
+ " windows = windows[w_idxs]\n",
+ " \n",
+ " if static is not None and not self.MULTIVARIATE:\n",
+ " static = static[w_idxs]\n",
+ "\n",
+ " windows_batch = dict(temporal=windows,\n",
+ " temporal_cols=temporal_cols,\n",
+ " static=static,\n",
+ " static_cols=static_cols)\n",
+ " return windows_batch\n",
+ "\n",
+ " elif step in ['predict', 'val']:\n",
+ "\n",
+ " if step == 'predict':\n",
+ " initial_input = temporal.shape[-1] - self.test_size\n",
+ " if initial_input <= self.input_size: # There is not enough data to predict first timestamp\n",
+ " temporal = F.pad(temporal, pad=(self.input_size-initial_input, 0), mode=\"constant\", value=0.0)\n",
+ " predict_step_size = self.predict_step_size\n",
+ " cutoff = - self.input_size - self.test_size\n",
+ " temporal = temporal[:, :, cutoff:]\n",
+ "\n",
+ " elif step == 'val':\n",
+ " predict_step_size = self.step_size\n",
+ " cutoff = -self.input_size - self.val_size - self.test_size\n",
+ " if self.test_size > 0:\n",
+ " temporal = batch['temporal'][:, :, cutoff:-self.test_size]\n",
+ " else:\n",
+ " temporal = batch['temporal'][:, :, cutoff:]\n",
+ " if temporal.shape[-1] < window_size:\n",
+ " initial_input = temporal.shape[-1] - self.val_size\n",
+ " temporal = F.pad(temporal, pad=(self.input_size-initial_input, 0), mode=\"constant\", value=0.0)\n",
+ "\n",
+ " if (step=='predict') and (self.test_size==0) and (len(self.futr_exog_list)==0):\n",
+ " temporal = F.pad(temporal, pad=(0, self.h), mode=\"constant\", value=0.0)\n",
+ "\n",
+ " windows = temporal.unfold(dimension=-1,\n",
+ " size=window_size,\n",
+ " step=predict_step_size)\n",
+ "\n",
+ " static = batch.get('static', None)\n",
+ " static_cols=batch.get('static_cols', None)\n",
+ "\n",
+ " if self.MULTIVARIATE:\n",
+ " # [n_series, C, Ws, L + h] -> [Ws, L + h, C, n_series]\n",
+ " windows = windows.permute(2, 3, 1, 0)\n",
+ " else:\n",
+ " # [n_series, C, Ws, L + h] -> [Ws * n_series, L + h, C, 1]\n",
+ " windows_per_serie = windows.shape[2]\n",
+ " windows = windows.permute(0, 2, 3, 1)\n",
+ " windows = windows.flatten(0, 1)\n",
+ " windows = windows.unsqueeze(-1)\n",
+ " if static is not None:\n",
+ " static = torch.repeat_interleave(static, \n",
+ " repeats=windows_per_serie, dim=0)\n",
+ "\n",
+ " # Sample windows for batched prediction\n",
+ " if w_idxs is not None:\n",
+ " windows = windows[w_idxs]\n",
+ " if static is not None and not self.MULTIVARIATE:\n",
+ " static = static[w_idxs]\n",
+ "\n",
+ " windows_batch = dict(temporal=windows,\n",
+ " temporal_cols=temporal_cols,\n",
+ " static=static,\n",
+ " static_cols=static_cols)\n",
+ " return windows_batch\n",
+ " else:\n",
+ " raise ValueError(f'Unknown step {step}') \n",
+ "\n",
+ " def _normalization(self, windows, y_idx):\n",
+ " # windows are already filtered by train/validation/test\n",
+ " # from the `create_windows_method` nor leakage risk\n",
+ " temporal = windows['temporal'] # [Ws, L + h, C, n_series]\n",
+ " temporal_cols = windows['temporal_cols'].copy() # [Ws, L + h, C, n_series]\n",
+ "\n",
+ " # To avoid leakage uses only the lags\n",
+ " temporal_data_cols = self._get_temporal_exogenous_cols(temporal_cols=temporal_cols)\n",
+ " temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)\n",
+ " temporal_idxs = np.append(y_idx, temporal_idxs)\n",
+ " temporal_data = temporal[:, :, temporal_idxs] \n",
+ " temporal_mask = temporal[:, :, temporal_cols.get_loc('available_mask')].clone()\n",
+ " if self.h > 0:\n",
+ " temporal_mask[:, -self.h:] = 0.0\n",
+ "\n",
+ " # Normalize. self.scaler stores the shift and scale for inverse transform\n",
+ " temporal_mask = temporal_mask.unsqueeze(2) # Add channel dimension for scaler.transform.\n",
+ " temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)\n",
+ "\n",
+ " # Replace values in windows dict\n",
+ " temporal[:, :, temporal_idxs] = temporal_data\n",
+ " windows['temporal'] = temporal\n",
+ "\n",
+ " return windows\n",
+ "\n",
+ " def _inv_normalization(self, y_hat, y_idx):\n",
+ " # Receives window predictions [Ws, h, output, n_series]\n",
+ " # Broadcasts scale if necessary and inverts normalization\n",
+ " add_channel_dim = y_hat.ndim > 3\n",
+ " y_loc, y_scale = self._get_loc_scale(y_idx, add_channel_dim=add_channel_dim)\n",
+ " y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)\n",
+ "\n",
+ " return y_hat\n",
+ "\n",
+ " def _parse_windows(self, batch, windows):\n",
+ " # windows: [Ws, L + h, C, n_series]\n",
+ "\n",
+ " # Filter insample lags from outsample horizon\n",
+ " y_idx = batch['y_idx']\n",
+ " mask_idx = batch['temporal_cols'].get_loc('available_mask')\n",
+ "\n",
+ " insample_y = windows['temporal'][:, :self.input_size, y_idx]\n",
+ " insample_mask = windows['temporal'][:, :self.input_size, mask_idx]\n",
+ "\n",
+ " # Declare additional information\n",
+ " outsample_y = None\n",
+ " outsample_mask = None\n",
+ " hist_exog = None\n",
+ " futr_exog = None\n",
+ " stat_exog = None\n",
+ "\n",
+ " if self.h > 0:\n",
+ " outsample_y = windows['temporal'][:, self.input_size:, y_idx]\n",
+ " outsample_mask = windows['temporal'][:, self.input_size:, mask_idx]\n",
+ "\n",
+ " # Recurrent models at t predict t+1, so we shift the input (insample_y) by one\n",
+ " if self.RECURRENT:\n",
+ " insample_y = torch.cat((insample_y, outsample_y[:, :-1]), dim=1)\n",
+ " insample_mask = torch.cat((insample_mask, outsample_mask[:, :-1]), dim=1)\n",
+ " self.maintain_state = False\n",
+ "\n",
+ " if len(self.hist_exog_list):\n",
+ " hist_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.hist_exog_list)\n",
+ " if self.RECURRENT:\n",
+ " hist_exog = windows['temporal'][:, :, hist_exog_idx]\n",
+ " hist_exog[:, self.input_size:] = 0.0\n",
+ " hist_exog = hist_exog[:, 1:]\n",
+ " else:\n",
+ " hist_exog = windows['temporal'][:, :self.input_size, hist_exog_idx]\n",
+ " if not self.MULTIVARIATE:\n",
+ " hist_exog = hist_exog.squeeze(-1)\n",
+ " else:\n",
+ " hist_exog = hist_exog.swapaxes(1, 2)\n",
+ "\n",
+ " if len(self.futr_exog_list):\n",
+ " futr_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.futr_exog_list)\n",
+ " futr_exog = windows['temporal'][:, :, futr_exog_idx]\n",
+ " if self.RECURRENT:\n",
+ " futr_exog = futr_exog[:, 1:]\n",
+ " if not self.MULTIVARIATE:\n",
+ " futr_exog = futr_exog.squeeze(-1)\n",
+ " else:\n",
+ " futr_exog = futr_exog.swapaxes(1, 2) \n",
+ "\n",
+ " if len(self.stat_exog_list):\n",
+ " static_idx = get_indexer_raise_missing(windows['static_cols'], self.stat_exog_list)\n",
+ " stat_exog = windows['static'][:, static_idx]\n",
+ "\n",
+ " # TODO: think a better way of removing insample_y features\n",
+ " if self.exclude_insample_y:\n",
+ " insample_y = insample_y * 0\n",
+ "\n",
+ " return insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
+ " hist_exog, futr_exog, stat_exog \n",
+ "\n",
+ " def _get_loc_scale(self, y_idx, add_channel_dim=False):\n",
+ " # [B, L, C, n_series] -> [B, L, n_series]\n",
+ " y_scale = self.scaler.x_scale[:, :, y_idx]\n",
+ " y_loc = self.scaler.x_shift[:, :, y_idx]\n",
+ " \n",
+ " # [B, L, n_series] -> [B, L, n_series, 1]\n",
+ " if add_channel_dim:\n",
+ " y_scale = y_scale.unsqueeze(-1)\n",
+ " y_loc = y_loc.unsqueeze(-1)\n",
+ "\n",
+ " return y_loc, y_scale\n",
+ "\n",
+ " def _compute_valid_loss(self, insample_y, outsample_y, output, outsample_mask, y_idx):\n",
+ " if self.loss.is_distribution_output:\n",
+ " y_loc, y_scale = self._get_loc_scale(y_idx)\n",
+ " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
+ " if isinstance(self.valid_loss, (losses.sCRPS, losses.MQLoss, losses.HuberMQLoss)):\n",
+ " _, _, quants = self.loss.sample(distr_args=distr_args) \n",
+ " output = quants\n",
+ " elif isinstance(self.valid_loss, losses.BasePointLoss):\n",
+ " distr = self.loss.get_distribution(distr_args=distr_args)\n",
+ " output = distr.mean\n",
+ "\n",
+ " # Validation Loss evaluation\n",
+ " if self.valid_loss.is_distribution_output:\n",
+ " valid_loss = self.valid_loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
+ " else:\n",
+ " output = self._inv_normalization(y_hat=output, y_idx=y_idx)\n",
+ " valid_loss = self.valid_loss(y=outsample_y, y_hat=output, y_insample=insample_y, mask=outsample_mask)\n",
+ " return valid_loss\n",
+ " \n",
+ " def _validate_step_recurrent_batch(self, insample_y, insample_mask, futr_exog, hist_exog, stat_exog, y_idx):\n",
+ " # Remember state in network and set horizon to 1\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = True\n",
+ " self.h = 1\n",
+ "\n",
+ " # Initialize results array\n",
+ " n_outputs = self.loss.outputsize_multiplier\n",
+ " y_hat = torch.zeros((insample_y.shape[0],\n",
+ " self.horizon_backup,\n",
+ " self.n_series * n_outputs),\n",
+ " device=insample_y.device,\n",
+ " dtype=insample_y.dtype)\n",
+ "\n",
+ " # First step prediction\n",
+ " tau = 0\n",
+ " \n",
+ " # Set exogenous\n",
+ " hist_exog_current = None\n",
+ " if self.hist_exog_size > 0:\n",
+ " hist_exog_current = hist_exog[:, :self.input_size + tau - 1]\n",
+ "\n",
+ " futr_exog_current = None\n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_current = futr_exog[:, :self.input_size + tau - 1]\n",
+ "\n",
+ " # First forecast step\n",
+ " y_hat[:, tau], insample_y = self._validate_step_recurrent_single(\n",
+ " insample_y=insample_y[:, :self.input_size + tau - 1],\n",
+ " insample_mask=insample_mask[:, :self.input_size + tau - 1],\n",
+ " hist_exog=hist_exog_current,\n",
+ " futr_exog=futr_exog_current,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx=y_idx,\n",
+ " )\n",
+ "\n",
+ " # Horizon prediction recursively\n",
+ " for tau in range(self.horizon_backup):\n",
+ " # Set exogenous\n",
+ " if self.hist_exog_size > 0:\n",
+ " hist_exog_current = hist_exog[:, self.input_size + tau - 1].unsqueeze(1)\n",
+ "\n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_current = futr_exog[:, self.input_size + tau - 1].unsqueeze(1)\n",
+ " \n",
+ " y_hat[:, tau], insample_y = self._validate_step_recurrent_single(\n",
+ " insample_y=insample_y,\n",
+ " insample_mask=None,\n",
+ " hist_exog=hist_exog_current,\n",
+ " futr_exog=futr_exog_current,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx = y_idx,\n",
+ " )\n",
+ " \n",
+ " # Reset state and horizon\n",
+ " self.maintain_state = False\n",
+ " self.rnn_state = None\n",
+ " self.h = self.horizon_backup\n",
+ "\n",
+ " return y_hat \n",
+ "\n",
+ " def _validate_step_recurrent_single(self, insample_y, insample_mask, hist_exog, futr_exog, stat_exog, y_idx):\n",
+ " # Input sequence\n",
+ " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
+ " insample_mask=insample_mask, # [Ws, L, n_series]\n",
+ " futr_exog=futr_exog, # univariate: [Ws, L, F]; multivariate: [Ws, F, L, n_series]\n",
+ " hist_exog=hist_exog, # univariate: [Ws, L, X]; multivariate: [Ws, X, L, n_series]\n",
+ " stat_exog=stat_exog) # univariate: [Ws, S]; multivariate: [n_series, S]\n",
+ "\n",
+ " # Model Predictions\n",
+ " output_batch_unmapped = self(windows_batch)\n",
+ " output_batch = self.loss.domain_map(output_batch_unmapped)\n",
+ " \n",
+ " # Inverse normalization and sampling\n",
+ " if self.loss.is_distribution_output:\n",
+ " # Sample distribution\n",
+ " y_loc, y_scale = self._get_loc_scale(y_idx)\n",
+ " distr_args = self.loss.scale_decouple(output=output_batch, loc=y_loc, scale=y_scale)\n",
+ " # When validating, the output is the mean of the distribution which is an attribute\n",
+ " distr = self.loss.get_distribution(distr_args=distr_args)\n",
+ "\n",
+ " # Scale back to feed back as input\n",
+ " insample_y = self.scaler.scaler(distr.mean, y_loc, y_scale)\n",
+ " else:\n",
+ " # Todo: for now, we assume that in case of a BasePointLoss with ndim==4, the last dimension\n",
+ " # contains a set of predictions for the target (e.g. MQLoss multiple quantiles), for which we use the \n",
+ " # mean as feedback signal for the recurrent predictions. A more precise way is to increase the\n",
+ " # insample input size of the recurrent network by the number of outputs so that each output\n",
+ " # can be fed back to a specific input channel. \n",
+ " if output_batch.ndim == 4:\n",
+ " output_batch = output_batch.mean(dim=-1)\n",
+ "\n",
+ " insample_y = output_batch\n",
+ "\n",
+ " # Remove horizon dim: [B, 1, N * n_outputs] -> [B, N * n_outputs]\n",
+ " y_hat = output_batch_unmapped.squeeze(1)\n",
+ " return y_hat, insample_y\n",
+ "\n",
+ " def _predict_step_recurrent_batch(self, insample_y, insample_mask, futr_exog, hist_exog, stat_exog, y_idx):\n",
+ " # Remember state in network and set horizon to 1\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = True\n",
+ " self.h = 1\n",
+ "\n",
+ " # Initialize results array\n",
+ " n_outputs = len(self.loss.output_names)\n",
+ " y_hat = torch.zeros((insample_y.shape[0],\n",
+ " self.horizon_backup,\n",
+ " self.n_series,\n",
+ " n_outputs),\n",
+ " device=insample_y.device,\n",
+ " dtype=insample_y.dtype)\n",
+ "\n",
+ " # First step prediction\n",
+ " tau = 0\n",
+ " \n",
+ " # Set exogenous\n",
+ " hist_exog_current = None\n",
+ " if self.hist_exog_size > 0:\n",
+ " hist_exog_current = hist_exog[:, :self.input_size + tau - 1]\n",
+ "\n",
+ " futr_exog_current = None\n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_current = futr_exog[:, :self.input_size + tau - 1]\n",
+ "\n",
+ " # First forecast step\n",
+ " y_hat[:, tau], insample_y = self._predict_step_recurrent_single(\n",
+ " insample_y=insample_y[:, :self.input_size + tau - 1],\n",
+ " insample_mask=insample_mask[:, :self.input_size + tau - 1],\n",
+ " hist_exog=hist_exog_current,\n",
+ " futr_exog=futr_exog_current,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx=y_idx,\n",
+ " )\n",
+ "\n",
+ " # Horizon prediction recursively\n",
+ " for tau in range(self.horizon_backup):\n",
+ " # Set exogenous\n",
+ " if self.hist_exog_size > 0:\n",
+ " hist_exog_current = hist_exog[:, self.input_size + tau - 1].unsqueeze(1)\n",
+ "\n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_current = futr_exog[:, self.input_size + tau - 1].unsqueeze(1)\n",
+ " \n",
+ " y_hat[:, tau], insample_y = self._predict_step_recurrent_single(\n",
+ " insample_y=insample_y,\n",
+ " insample_mask=None,\n",
+ " hist_exog=hist_exog_current,\n",
+ " futr_exog=futr_exog_current,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx = y_idx,\n",
+ " )\n",
+ " \n",
+ " # Reset state and horizon\n",
+ " self.maintain_state = False\n",
+ " self.rnn_state = None\n",
+ " self.h = self.horizon_backup\n",
+ "\n",
+ " # Squeeze for univariate case\n",
+ " if not self.MULTIVARIATE:\n",
+ " y_hat = y_hat.squeeze(2)\n",
+ "\n",
+ " return y_hat \n",
+ "\n",
+ " def _predict_step_recurrent_single(self, insample_y, insample_mask, hist_exog, futr_exog, stat_exog, y_idx):\n",
+ " # Input sequence\n",
+ " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
+ " insample_mask=insample_mask, # [Ws, L, n_series]\n",
+ " futr_exog=futr_exog, # univariate: [Ws, L, F]; multivariate: [Ws, F, L, n_series]\n",
+ " hist_exog=hist_exog, # univariate: [Ws, L, X]; multivariate: [Ws, X, L, n_series]\n",
+ " stat_exog=stat_exog) # univariate: [Ws, S]; multivariate: [n_series, S]\n",
+ "\n",
+ " # Model Predictions\n",
+ " output_batch_unmapped = self(windows_batch)\n",
+ " output_batch = self.loss.domain_map(output_batch_unmapped)\n",
+ " \n",
+ " # Inverse normalization and sampling\n",
+ " if self.loss.is_distribution_output:\n",
+ " # Sample distribution\n",
+ " y_loc, y_scale = self._get_loc_scale(y_idx)\n",
+ " distr_args = self.loss.scale_decouple(output=output_batch, loc=y_loc, scale=y_scale)\n",
+ " # When predicting, we need to sample to get the quantiles. The mean is an attribute.\n",
+ " _, _, quants = self.loss.sample(distr_args=distr_args, num_samples=self.n_samples)\n",
+ " mean = self.loss.distr_mean\n",
+ "\n",
+ " # Scale back to feed back as input\n",
+ " insample_y = self.scaler.scaler(mean, y_loc, y_scale)\n",
+ " \n",
+ " # Save predictions\n",
+ " y_hat = torch.concat((mean.unsqueeze(-1), quants), axis=-1)\n",
+ "\n",
+ " if self.loss.return_params:\n",
+ " distr_args = torch.stack(distr_args, dim=-1)\n",
+ " if distr_args.ndim > 4:\n",
+ " distr_args = distr_args.flatten(-2, -1)\n",
+ " y_hat = torch.concat((y_hat, distr_args), axis=-1)\n",
+ " else:\n",
+ " # Todo: for now, we assume that in case of a BasePointLoss with ndim==4, the last dimension\n",
+ " # contains a set of predictions for the target (e.g. MQLoss multiple quantiles), for which we use the \n",
+ " # mean as feedback signal for the recurrent predictions. A more precise way is to increase the\n",
+ " # insample input size of the recurrent network by the number of outputs so that each output\n",
+ " # can be fed back to a specific input channel. \n",
+ " if output_batch.ndim == 4:\n",
+ " output_batch = output_batch.mean(dim=-1)\n",
+ "\n",
+ " insample_y = output_batch\n",
+ " y_hat = self._inv_normalization(y_hat=output_batch, y_idx=y_idx)\n",
+ " y_hat = y_hat.unsqueeze(-1)\n",
+ "\n",
+ " # Remove horizon dim: [B, 1, N, n_outputs] -> [B, N, n_outputs]\n",
+ " y_hat = y_hat.squeeze(1)\n",
+ " return y_hat, insample_y\n",
+ "\n",
+ " def _predict_step_direct_batch(self, insample_y, insample_mask, hist_exog, futr_exog, stat_exog, y_idx):\n",
+ " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
+ " insample_mask=insample_mask, # [Ws, L, n_series]\n",
+ " futr_exog=futr_exog, # univariate: [Ws, L, F]; multivariate: [Ws, F, L, n_series]\n",
+ " hist_exog=hist_exog, # univariate: [Ws, L, X]; multivariate: [Ws, X, L, n_series]\n",
+ " stat_exog=stat_exog) # univariate: [Ws, S]; multivariate: [n_series, S]\n",
+ "\n",
+ " # Model Predictions\n",
+ " output_batch = self(windows_batch)\n",
+ " output_batch = self.loss.domain_map(output_batch)\n",
+ "\n",
+ " # Inverse normalization and sampling\n",
+ " if self.loss.is_distribution_output:\n",
+ " y_loc, y_scale = self._get_loc_scale(y_idx)\n",
+ " distr_args = self.loss.scale_decouple(output=output_batch, loc=y_loc, scale=y_scale)\n",
+ " _, sample_mean, quants = self.loss.sample(distr_args=distr_args)\n",
+ " y_hat = torch.concat((sample_mean, quants), axis=-1)\n",
+ "\n",
+ " if self.loss.return_params:\n",
+ " distr_args = torch.stack(distr_args, dim=-1)\n",
+ " if distr_args.ndim > 4:\n",
+ " distr_args = distr_args.flatten(-2, -1)\n",
+ " y_hat = torch.concat((y_hat, distr_args), axis=-1) \n",
+ " else:\n",
+ " y_hat = self._inv_normalization(y_hat=output_batch, \n",
+ " y_idx=y_idx)\n",
+ "\n",
+ " return y_hat\n",
+ " \n",
+ " def training_step(self, batch, batch_idx):\n",
+ " # Set horizon to h_train in case of recurrent model to speed up training\n",
+ " if self.RECURRENT:\n",
+ " self.h = self.h_train\n",
+ " \n",
+ " # windows: [Ws, L + h, C, n_series] or [Ws, L + h, C]\n",
+ " y_idx = batch['y_idx']\n",
+ "\n",
+ " windows = self._create_windows(batch, step='train')\n",
+ " original_outsample_y = torch.clone(windows['temporal'][:, self.input_size:, y_idx])\n",
+ " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
+ " \n",
+ " # Parse windows\n",
+ " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
+ " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
+ "\n",
+ " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
+ " insample_mask=insample_mask, # [Ws, L, n_series]\n",
+ " futr_exog=futr_exog, # univariate: [Ws, L, F]; multivariate: [Ws, F, L, n_series]\n",
+ " hist_exog=hist_exog, # univariate: [Ws, L, X]; multivariate: [Ws, X, L, n_series]\n",
+ " stat_exog=stat_exog) # univariate: [Ws, S]; multivariate: [n_series, S]\n",
+ "\n",
+ " # Model Predictions\n",
+ " output = self(windows_batch)\n",
+ " output = self.loss.domain_map(output)\n",
+ " \n",
+ " if self.loss.is_distribution_output:\n",
+ " y_loc, y_scale = self._get_loc_scale(y_idx)\n",
+ " outsample_y = original_outsample_y\n",
+ " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
+ " loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
+ " else:\n",
+ " loss = self.loss(y=outsample_y, y_hat=output, y_insample=insample_y, mask=outsample_mask)\n",
+ "\n",
+ " if torch.isnan(loss):\n",
+ " print('Model Parameters', self.hparams)\n",
+ " print('insample_y', torch.isnan(insample_y).sum())\n",
+ " print('outsample_y', torch.isnan(outsample_y).sum())\n",
+ " raise Exception('Loss is NaN, training stopped.')\n",
+ "\n",
+ " train_loss_log = loss.detach().item()\n",
+ " self.log(\n",
+ " 'train_loss',\n",
+ " train_loss_log,\n",
+ " batch_size=outsample_y.size(0),\n",
+ " prog_bar=True,\n",
+ " on_epoch=True,\n",
+ " )\n",
+ " self.train_trajectories.append((self.global_step, train_loss_log))\n",
+ "\n",
+ " self.h = self.horizon_backup\n",
+ "\n",
+ " return loss\n",
+ "\n",
+ "\n",
+ " def validation_step(self, batch, batch_idx):\n",
+ " if self.val_size == 0:\n",
+ " return np.nan\n",
+ "\n",
+ " # TODO: Hack to compute number of windows\n",
+ " windows = self._create_windows(batch, step='val')\n",
+ " n_windows = len(windows['temporal'])\n",
+ " y_idx = batch['y_idx']\n",
+ "\n",
+ " # Number of windows in batch\n",
+ " windows_batch_size = self.inference_windows_batch_size\n",
+ " if windows_batch_size < 0:\n",
+ " windows_batch_size = n_windows\n",
+ " n_batches = int(np.ceil(n_windows / windows_batch_size))\n",
+ "\n",
+ " valid_losses = []\n",
+ " batch_sizes = []\n",
+ " for i in range(n_batches):\n",
+ " # Create and normalize windows [Ws, L + h, C, n_series]\n",
+ " w_idxs = np.arange(i*windows_batch_size, \n",
+ " min((i+1)*windows_batch_size, n_windows))\n",
+ " windows = self._create_windows(batch, step='val', w_idxs=w_idxs)\n",
+ " original_outsample_y = torch.clone(windows['temporal'][:, self.input_size:, y_idx])\n",
+ "\n",
+ " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
+ "\n",
+ " # Parse windows\n",
+ " insample_y, insample_mask, _, outsample_mask, \\\n",
+ " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
+ "\n",
+ " if self.RECURRENT:\n",
+ " output_batch = self._validate_step_recurrent_batch(insample_y=insample_y,\n",
+ " insample_mask=insample_mask,\n",
+ " futr_exog=futr_exog,\n",
+ " hist_exog=hist_exog,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx=y_idx)\n",
+ " else: \n",
+ " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
+ " insample_mask=insample_mask, # [Ws, L, n_series]\n",
+ " futr_exog=futr_exog, # univariate: [Ws, L, F]; multivariate: [Ws, F, L, n_series]\n",
+ " hist_exog=hist_exog, # univariate: [Ws, L, X]; multivariate: [Ws, X, L, n_series]\n",
+ " stat_exog=stat_exog) # univariate: [Ws, S]; multivariate: [n_series, S]\n",
+ " \n",
+ " # Model Predictions\n",
+ " output_batch = self(windows_batch) \n",
+ "\n",
+ " output_batch = self.loss.domain_map(output_batch)\n",
+ " valid_loss_batch = self._compute_valid_loss(insample_y=insample_y,\n",
+ " outsample_y=original_outsample_y,\n",
+ " output=output_batch, \n",
+ " outsample_mask=outsample_mask,\n",
+ " y_idx=batch['y_idx'])\n",
+ " valid_losses.append(valid_loss_batch)\n",
+ " batch_sizes.append(len(output_batch))\n",
+ " \n",
+ " valid_loss = torch.stack(valid_losses)\n",
+ " batch_sizes = torch.tensor(batch_sizes, device=valid_loss.device)\n",
+ " batch_size = torch.sum(batch_sizes)\n",
+ " valid_loss = torch.sum(valid_loss * batch_sizes) / batch_size\n",
+ "\n",
+ " if torch.isnan(valid_loss):\n",
+ " raise Exception('Loss is NaN, training stopped.')\n",
+ "\n",
+ " valid_loss_log = valid_loss.detach()\n",
+ " self.log(\n",
+ " 'valid_loss',\n",
+ " valid_loss_log.item(),\n",
+ " batch_size=batch_size,\n",
+ " prog_bar=True,\n",
+ " on_epoch=True,\n",
+ " )\n",
+ " self.validation_step_outputs.append(valid_loss_log)\n",
+ " return valid_loss\n",
+ "\n",
+ " def predict_step(self, batch, batch_idx):\n",
+ " if self.RECURRENT:\n",
+ " self.input_size = self.inference_input_size\n",
+ "\n",
+ " # TODO: Hack to compute number of windows\n",
+ " windows = self._create_windows(batch, step='predict')\n",
+ " n_windows = len(windows['temporal'])\n",
+ " y_idx = batch['y_idx']\n",
+ "\n",
+ " # Number of windows in batch\n",
+ " windows_batch_size = self.inference_windows_batch_size\n",
+ " if windows_batch_size < 0:\n",
+ " windows_batch_size = n_windows\n",
+ " n_batches = int(np.ceil(n_windows / windows_batch_size))\n",
+ " y_hats = []\n",
+ " for i in range(n_batches):\n",
+ " # Create and normalize windows [Ws, L+H, C]\n",
+ " w_idxs = np.arange(i*windows_batch_size, \n",
+ " min((i+1)*windows_batch_size, n_windows))\n",
+ " windows = self._create_windows(batch, step='predict', w_idxs=w_idxs)\n",
+ " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
+ "\n",
+ " # Parse windows\n",
+ " insample_y, insample_mask, _, _, \\\n",
+ " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
+ "\n",
+ " if self.RECURRENT: \n",
+ " y_hat = self._predict_step_recurrent_batch(insample_y=insample_y,\n",
+ " insample_mask=insample_mask,\n",
+ " futr_exog=futr_exog,\n",
+ " hist_exog=hist_exog,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx=y_idx)\n",
+ " else:\n",
+ " y_hat = self._predict_step_direct_batch(insample_y=insample_y,\n",
+ " insample_mask=insample_mask,\n",
+ " futr_exog=futr_exog,\n",
+ " hist_exog=hist_exog,\n",
+ " stat_exog=stat_exog,\n",
+ " y_idx=y_idx) \n",
+ "\n",
+ "\n",
+ " y_hats.append(y_hat)\n",
+ " y_hat = torch.cat(y_hats, dim=0)\n",
+ " self.input_size = self.input_size_backup\n",
+ "\n",
+ " return y_hat\n",
+ " \n",
+ " def fit(self, dataset, val_size=0, test_size=0, random_seed=None, distributed_config=None):\n",
+ " \"\"\" Fit.\n",
+ "\n",
+ " The `fit` method, optimizes the neural network's weights using the\n",
+ " initialization parameters (`learning_rate`, `windows_batch_size`, ...)\n",
+ " and the `loss` function as defined during the initialization. \n",
+ " Within `fit` we use a PyTorch Lightning `Trainer` that\n",
+ " inherits the initialization's `self.trainer_kwargs`, to customize\n",
+ " its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).\n",
+ "\n",
+ " The method is designed to be compatible with SKLearn-like classes\n",
+ " and in particular to be compatible with the StatsForecast library.\n",
+ "\n",
+ " By default the `model` is not saving training checkpoints to protect \n",
+ " disk memory, to get them change `enable_checkpointing=True` in `__init__`.\n",
+ "\n",
+ " **Parameters:**
\n",
+ " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ " `val_size`: int, validation size for temporal cross-validation.
\n",
+ " `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
+ " `test_size`: int, test size for temporal cross-validation.
\n",
+ " \"\"\"\n",
+ " return self._fit(\n",
+ " dataset=dataset,\n",
+ " batch_size=self.batch_size,\n",
+ " valid_batch_size=self.valid_batch_size,\n",
+ " val_size=val_size,\n",
+ " test_size=test_size,\n",
+ " random_seed=random_seed,\n",
+ " distributed_config=distributed_config,\n",
+ " )\n",
+ "\n",
+ " def predict(self, dataset, test_size=None, step_size=1,\n",
+ " random_seed=None, quantiles=None, **data_module_kwargs):\n",
+ " \"\"\" Predict.\n",
+ "\n",
+ " Neural network prediction with PL's `Trainer` execution of `predict_step`.\n",
+ "\n",
+ " **Parameters:**
\n",
+ " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ " `test_size`: int=None, test size for temporal cross-validation.
\n",
+ " `step_size`: int=1, Step size between each window.
\n",
+ " `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
+ " `quantiles`: list of floats, optional (default=None), target quantiles to predict.
\n",
+ " `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).\n",
+ " \"\"\"\n",
+ " self._check_exog(dataset)\n",
+ " self._restart_seed(random_seed)\n",
+ " if \"quantile\" in data_module_kwargs:\n",
+ " warnings.warn(\"The 'quantile' argument will be deprecated, use 'quantiles' instead.\")\n",
+ " if quantiles is not None:\n",
+ " raise ValueError(\"You can't specify quantile and quantiles.\")\n",
+ " quantiles = [data_module_kwargs.pop(\"quantile\")]\n",
+ " self._set_quantiles(quantiles)\n",
+ "\n",
+ " self.predict_step_size = step_size\n",
+ " self.decompose_forecast = False\n",
+ " datamodule = TimeSeriesDataModule(dataset=dataset,\n",
+ " valid_batch_size=self.valid_batch_size,\n",
+ " **data_module_kwargs)\n",
+ "\n",
+ " # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.\n",
+ " pred_trainer_kwargs = self.trainer_kwargs.copy()\n",
+ " if (pred_trainer_kwargs.get('accelerator', None) == \"gpu\") and (torch.cuda.device_count() > 1):\n",
+ " pred_trainer_kwargs['devices'] = [0]\n",
+ "\n",
+ " trainer = pl.Trainer(**pred_trainer_kwargs)\n",
+ " fcsts = trainer.predict(self, datamodule=datamodule) \n",
+ " fcsts = torch.vstack(fcsts)\n",
+ "\n",
+ " if self.MULTIVARIATE:\n",
+ " # [B, h, n_series (, Q)] -> [n_series, B, h (, Q)]\n",
+ " fcsts = fcsts.swapaxes(0, 2)\n",
+ " fcsts = fcsts.swapaxes(1, 2)\n",
+ "\n",
+ " fcsts = fcsts.numpy().flatten()\n",
+ " fcsts = fcsts.reshape(-1, len(self.loss.output_names))\n",
+ " return fcsts\n",
+ "\n",
+ " def decompose(self, dataset, step_size=1, random_seed=None, quantiles=None, **data_module_kwargs):\n",
+ " \"\"\" Decompose Predictions.\n",
+ "\n",
+ " Decompose the predictions through the network's layers.\n",
+ " Available methods are `ESRNN`, `NHITS`, `NBEATS`, and `NBEATSx`.\n",
+ "\n",
+ " **Parameters:**
\n",
+ " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation here](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ " `step_size`: int=1, step size between each window of temporal data.
\n",
+ " `quantiles`: list of floats, optional (default=None), target quantiles to predict.
\n",
+ " `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).\n",
+ " \"\"\"\n",
+ " # Restart random seed\n",
+ " if random_seed is None:\n",
+ " random_seed = self.random_seed\n",
+ " torch.manual_seed(random_seed)\n",
+ " self._set_quantiles(quantiles)\n",
+ "\n",
+ " self.predict_step_size = step_size\n",
+ " self.decompose_forecast = True\n",
+ " datamodule = TimeSeriesDataModule(dataset=dataset,\n",
+ " valid_batch_size=self.valid_batch_size,\n",
+ " **data_module_kwargs)\n",
+ " trainer = pl.Trainer(**self.trainer_kwargs)\n",
+ " fcsts = trainer.predict(self, datamodule=datamodule)\n",
+ " self.decompose_forecast = False # Default decomposition back to false\n",
+ " return torch.vstack(fcsts).numpy() "
]
}
],
diff --git a/nbs/common.base_multivariate.ipynb b/nbs/common.base_multivariate.ipynb
deleted file mode 100644
index f1321600d..000000000
--- a/nbs/common.base_multivariate.ipynb
+++ /dev/null
@@ -1,625 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| default_exp common._base_multivariate"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "%load_ext autoreload\n",
- "%autoreload 2"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# BaseMultivariate\n",
- "\n",
- "> The `BaseWindows` class contains standard methods shared across window-based multivariate neural networks; in contrast to recurrent neural networks these models commit to a fixed sequence length input."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The standard methods include data preprocessing `_normalization`, optimization utilities like parameter initialization, `training_step`, `validation_step`, and shared `fit` and `predict` methods.These shared methods enable all the `neuralforecast.models` compatibility with the `core.NeuralForecast` wrapper class. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "import numpy as np\n",
- "import torch\n",
- "import torch.nn as nn\n",
- "import pytorch_lightning as pl\n",
- "import neuralforecast.losses.pytorch as losses\n",
- "\n",
- "from neuralforecast.common._base_model import BaseModel\n",
- "from neuralforecast.common._scalers import TemporalNorm\n",
- "from neuralforecast.tsdataset import TimeSeriesDataModule\n",
- "from neuralforecast.utils import get_indexer_raise_missing"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "class BaseMultivariate(BaseModel):\n",
- " \"\"\" Base Multivariate\n",
- " \n",
- " Base class for all multivariate models. The forecasts for all time-series are produced simultaneously \n",
- " within each window, which are randomly sampled during training.\n",
- " \n",
- " This class implements the basic functionality for all windows-based models, including:\n",
- " - PyTorch Lightning's methods training_step, validation_step, predict_step.
\n",
- " - fit and predict methods used by NeuralForecast.core class.
\n",
- " - sampling and wrangling methods to generate multivariate windows.\n",
- " \"\"\"\n",
- " def __init__(self, \n",
- " h,\n",
- " input_size,\n",
- " loss,\n",
- " valid_loss,\n",
- " learning_rate,\n",
- " max_steps,\n",
- " val_check_steps,\n",
- " n_series,\n",
- " batch_size,\n",
- " step_size=1,\n",
- " num_lr_decays=0,\n",
- " early_stop_patience_steps=-1,\n",
- " scaler_type='robust',\n",
- " futr_exog_list=None,\n",
- " hist_exog_list=None,\n",
- " stat_exog_list=None,\n",
- " num_workers_loader=0,\n",
- " drop_last_loader=False,\n",
- " random_seed=1, \n",
- " alias=None,\n",
- " optimizer=None,\n",
- " optimizer_kwargs=None,\n",
- " lr_scheduler=None,\n",
- " lr_scheduler_kwargs=None,\n",
- " dataloader_kwargs=None,\n",
- " **trainer_kwargs):\n",
- " super().__init__(\n",
- " random_seed=random_seed,\n",
- " loss=loss,\n",
- " valid_loss=valid_loss,\n",
- " optimizer=optimizer,\n",
- " optimizer_kwargs=optimizer_kwargs,\n",
- " lr_scheduler=lr_scheduler,\n",
- " lr_scheduler_kwargs=lr_scheduler_kwargs, \n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
- " max_steps=max_steps,\n",
- " early_stop_patience_steps=early_stop_patience_steps,\n",
- " **trainer_kwargs,\n",
- " )\n",
- "\n",
- " # Padder to complete train windows, \n",
- " # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]\n",
- " self.h = h\n",
- " self.input_size = input_size\n",
- " self.n_series = n_series\n",
- " self.padder = nn.ConstantPad1d(padding=(0, self.h), value=0.0)\n",
- "\n",
- " # Multivariate models do not support these loss functions yet.\n",
- " unsupported_losses = (\n",
- " losses.sCRPS,\n",
- " losses.MQLoss,\n",
- " losses.DistributionLoss,\n",
- " losses.PMM,\n",
- " losses.GMM,\n",
- " losses.HuberMQLoss,\n",
- " losses.MASE,\n",
- " losses.relMSE,\n",
- " losses.NBMM,\n",
- " )\n",
- " if isinstance(self.loss, unsupported_losses):\n",
- " raise Exception(f\"{self.loss} is not supported in a Multivariate model.\") \n",
- " if isinstance(self.valid_loss, unsupported_losses):\n",
- " raise Exception(f\"{self.valid_loss} is not supported in a Multivariate model.\") \n",
- "\n",
- " self.batch_size = batch_size\n",
- " \n",
- " # Optimization\n",
- " self.learning_rate = learning_rate\n",
- " self.max_steps = max_steps\n",
- " self.num_lr_decays = num_lr_decays\n",
- " self.lr_decay_steps = max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7\n",
- " self.early_stop_patience_steps = early_stop_patience_steps\n",
- " self.val_check_steps = val_check_steps\n",
- " self.step_size = step_size\n",
- "\n",
- " # Scaler\n",
- " self.scaler = TemporalNorm(scaler_type=scaler_type, dim=2) # Time dimension is in the second axis\n",
- "\n",
- " # Fit arguments\n",
- " self.val_size = 0\n",
- " self.test_size = 0\n",
- "\n",
- " # Model state\n",
- " self.decompose_forecast = False\n",
- "\n",
- " # DataModule arguments\n",
- " self.num_workers_loader = num_workers_loader\n",
- " self.dataloader_kwargs = dataloader_kwargs\n",
- " self.drop_last_loader = drop_last_loader\n",
- " # used by on_validation_epoch_end hook\n",
- " self.validation_step_outputs = []\n",
- " self.alias = alias\n",
- "\n",
- " def _create_windows(self, batch, step):\n",
- " # Parse common data\n",
- " window_size = self.input_size + self.h\n",
- " temporal_cols = batch['temporal_cols']\n",
- " temporal = batch['temporal']\n",
- "\n",
- " if step == 'train':\n",
- " if self.val_size + self.test_size > 0:\n",
- " cutoff = -self.val_size - self.test_size\n",
- " temporal = temporal[:, :, :cutoff]\n",
- "\n",
- " temporal = self.padder(temporal)\n",
- " windows = temporal.unfold(dimension=-1, \n",
- " size=window_size, \n",
- " step=self.step_size)\n",
- " # [n_series, C, Ws, L+H] 0, 1, 2, 3\n",
- "\n",
- " # Sample and Available conditions\n",
- " available_idx = temporal_cols.get_loc('available_mask')\n",
- " sample_condition = windows[:, available_idx, :, -self.h:]\n",
- " sample_condition = torch.sum(sample_condition, axis=2) # Sum over time\n",
- " sample_condition = torch.sum(sample_condition, axis=0) # Sum over time-series\n",
- " available_condition = windows[:, available_idx, :, :-self.h]\n",
- " available_condition = torch.sum(available_condition, axis=2) # Sum over time\n",
- " available_condition = torch.sum(available_condition, axis=0) # Sum over time-series\n",
- " final_condition = (sample_condition > 0) & (available_condition > 0) # Of shape [Ws]\n",
- " windows = windows[:, :, final_condition, :]\n",
- "\n",
- " # Get Static data\n",
- " static = batch.get('static', None)\n",
- " static_cols = batch.get('static_cols', None)\n",
- "\n",
- " # Protection of empty windows\n",
- " if final_condition.sum() == 0:\n",
- " raise Exception('No windows available for training')\n",
- "\n",
- " # Sample windows\n",
- " n_windows = windows.shape[2]\n",
- " if self.batch_size is not None:\n",
- " w_idxs = np.random.choice(n_windows, \n",
- " size=self.batch_size,\n",
- " replace=(n_windows < self.batch_size))\n",
- " windows = windows[:, :, w_idxs, :]\n",
- "\n",
- " windows = windows.permute(2, 1, 3, 0) # [Ws, C, L+H, n_series]\n",
- "\n",
- " windows_batch = dict(temporal=windows,\n",
- " temporal_cols=temporal_cols,\n",
- " static=static,\n",
- " static_cols=static_cols)\n",
- "\n",
- " return windows_batch\n",
- "\n",
- " elif step in ['predict', 'val']:\n",
- "\n",
- " if step == 'predict':\n",
- " predict_step_size = self.predict_step_size\n",
- " cutoff = - self.input_size - self.test_size\n",
- " temporal = batch['temporal'][:, :, cutoff:]\n",
- "\n",
- " elif step == 'val':\n",
- " predict_step_size = self.step_size\n",
- " cutoff = -self.input_size - self.val_size - self.test_size\n",
- " if self.test_size > 0:\n",
- " temporal = batch['temporal'][:, :, cutoff:-self.test_size]\n",
- " else:\n",
- " temporal = batch['temporal'][:, :, cutoff:]\n",
- "\n",
- " if (step=='predict') and (self.test_size==0) and (len(self.futr_exog_list)==0):\n",
- " temporal = self.padder(temporal)\n",
- "\n",
- " windows = temporal.unfold(dimension=-1,\n",
- " size=window_size,\n",
- " step=predict_step_size)\n",
- " # [n_series, C, Ws, L+H] -> [Ws, C, L+H, n_series]\n",
- " windows = windows.permute(2, 1, 3, 0)\n",
- "\n",
- " # Get Static data\n",
- " static = batch.get('static', None)\n",
- " static_cols=batch.get('static_cols', None)\n",
- "\n",
- " windows_batch = dict(temporal=windows,\n",
- " temporal_cols=temporal_cols,\n",
- " static=static,\n",
- " static_cols=static_cols)\n",
- "\n",
- "\n",
- " return windows_batch\n",
- " else:\n",
- " raise ValueError(f'Unknown step {step}') \n",
- "\n",
- " def _normalization(self, windows, y_idx):\n",
- " \n",
- " # windows are already filtered by train/validation/test\n",
- " # from the `create_windows_method` nor leakage risk\n",
- " temporal = windows['temporal'] # [Ws, C, L+H, n_series]\n",
- " temporal_cols = windows['temporal_cols'].copy() # [Ws, C, L+H, n_series]\n",
- "\n",
- " # To avoid leakage uses only the lags\n",
- " temporal_data_cols = self._get_temporal_exogenous_cols(temporal_cols=temporal_cols)\n",
- " temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)\n",
- " temporal_idxs = np.append(y_idx, temporal_idxs)\n",
- " temporal_data = temporal[:, temporal_idxs, :, :]\n",
- " temporal_mask = temporal[:, temporal_cols.get_loc('available_mask'), :, :].clone()\n",
- " temporal_mask[:, -self.h:, :] = 0.0\n",
- "\n",
- " # Normalize. self.scaler stores the shift and scale for inverse transform\n",
- " temporal_mask = temporal_mask.unsqueeze(1) # Add channel dimension for scaler.transform.\n",
- " temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)\n",
- " # Replace values in windows dict\n",
- " temporal[:, temporal_idxs, :, :] = temporal_data\n",
- " windows['temporal'] = temporal\n",
- "\n",
- " return windows\n",
- "\n",
- " def _inv_normalization(self, y_hat, temporal_cols, y_idx):\n",
- " # Receives window predictions [Ws, H, n_series]\n",
- " # Broadcasts outputs and inverts normalization\n",
- "\n",
- " # Add C dimension\n",
- " # if y_hat.ndim == 2:\n",
- " # remove_dimension = True\n",
- " # y_hat = y_hat.unsqueeze(-1)\n",
- " # else:\n",
- " # remove_dimension = False\n",
- " \n",
- " y_scale = self.scaler.x_scale[:, [y_idx], :].squeeze(1)\n",
- " y_loc = self.scaler.x_shift[:, [y_idx], :].squeeze(1)\n",
- "\n",
- " # y_scale = torch.repeat_interleave(y_scale, repeats=y_hat.shape[-1], dim=-1)\n",
- " # y_loc = torch.repeat_interleave(y_loc, repeats=y_hat.shape[-1], dim=-1)\n",
- "\n",
- " y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)\n",
- "\n",
- " # if remove_dimension:\n",
- " # y_hat = y_hat.squeeze(-1)\n",
- " # y_loc = y_loc.squeeze(-1)\n",
- " # y_scale = y_scale.squeeze(-1)\n",
- "\n",
- " return y_hat, y_loc, y_scale\n",
- "\n",
- " def _parse_windows(self, batch, windows):\n",
- " # Temporal: [Ws, C, L+H, n_series]\n",
- "\n",
- " # Filter insample lags from outsample horizon\n",
- " mask_idx = batch['temporal_cols'].get_loc('available_mask')\n",
- " y_idx = batch['y_idx'] \n",
- " insample_y = windows['temporal'][:, y_idx, :-self.h, :]\n",
- " insample_mask = windows['temporal'][:, mask_idx, :-self.h, :]\n",
- " outsample_y = windows['temporal'][:, y_idx, -self.h:, :]\n",
- " outsample_mask = windows['temporal'][:, mask_idx, -self.h:, :]\n",
- "\n",
- " # Filter historic exogenous variables\n",
- " if len(self.hist_exog_list):\n",
- " hist_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.hist_exog_list)\n",
- " hist_exog = windows['temporal'][:, hist_exog_idx, :-self.h, :]\n",
- " else:\n",
- " hist_exog = None\n",
- " \n",
- " # Filter future exogenous variables\n",
- " if len(self.futr_exog_list):\n",
- " futr_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.futr_exog_list)\n",
- " futr_exog = windows['temporal'][:, futr_exog_idx, :, :]\n",
- " else:\n",
- " futr_exog = None\n",
- "\n",
- " # Filter static variables\n",
- " if len(self.stat_exog_list):\n",
- " static_idx = get_indexer_raise_missing(windows['static_cols'], self.stat_exog_list)\n",
- " stat_exog = windows['static'][:, static_idx]\n",
- " else:\n",
- " stat_exog = None\n",
- "\n",
- " return insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog\n",
- "\n",
- " def training_step(self, batch, batch_idx): \n",
- " # Create and normalize windows [batch_size, n_series, C, L+H]\n",
- " windows = self._create_windows(batch, step='train')\n",
- " y_idx = batch['y_idx']\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
- " insample_mask=insample_mask, # [Ws, L, n_series]\n",
- " futr_exog=futr_exog, # [Ws, F, L + h, n_series]\n",
- " hist_exog=hist_exog, # [Ws, X, L, n_series]\n",
- " stat_exog=stat_exog) # [n_series, S]\n",
- "\n",
- " # Model Predictions\n",
- " output = self(windows_batch)\n",
- " if self.loss.is_distribution_output:\n",
- " outsample_y, y_loc, y_scale = self._inv_normalization(y_hat=outsample_y,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
- " else:\n",
- " loss = self.loss(y=outsample_y, y_hat=output, mask=outsample_mask)\n",
- "\n",
- " if torch.isnan(loss):\n",
- " print('Model Parameters', self.hparams)\n",
- " print('insample_y', torch.isnan(insample_y).sum())\n",
- " print('outsample_y', torch.isnan(outsample_y).sum())\n",
- " print('output', torch.isnan(output).sum())\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'train_loss',\n",
- " loss.detach().item(),\n",
- " batch_size=outsample_y.size(0),\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.train_trajectories.append((self.global_step, loss.detach().item()))\n",
- " return loss\n",
- "\n",
- " def validation_step(self, batch, batch_idx):\n",
- " if self.val_size == 0:\n",
- " return np.nan\n",
- " \n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " windows = self._create_windows(batch, step='val')\n",
- " y_idx = batch['y_idx']\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
- " insample_mask=insample_mask, # [Ws, L, n_series]\n",
- " futr_exog=futr_exog, # [Ws, F, L + h, n_series]\n",
- " hist_exog=hist_exog, # [Ws, X, L, n_series]\n",
- " stat_exog=stat_exog) # [n_series, S]\n",
- "\n",
- " # Model Predictions\n",
- " output = self(windows_batch)\n",
- " if self.loss.is_distribution_output:\n",
- " outsample_y, y_loc, y_scale = self._inv_normalization(y_hat=outsample_y,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- "\n",
- " if str(type(self.valid_loss)) in\\\n",
- " [\"\", \"\"]:\n",
- " _, output = self.loss.sample(distr_args=distr_args)\n",
- "\n",
- " # Validation Loss evaluation\n",
- " if self.valid_loss.is_distribution_output:\n",
- " valid_loss = self.valid_loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
- " else:\n",
- " valid_loss = self.valid_loss(y=outsample_y, y_hat=output, mask=outsample_mask)\n",
- "\n",
- " if torch.isnan(valid_loss):\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'valid_loss',\n",
- " valid_loss.detach().item(),\n",
- " batch_size=outsample_y.size(0),\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.validation_step_outputs.append(valid_loss)\n",
- " return valid_loss\n",
- "\n",
- " def predict_step(self, batch, batch_idx): \n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " windows = self._create_windows(batch, step='predict')\n",
- " y_idx = batch['y_idx'] \n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, _, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L, n_series]\n",
- " insample_mask=insample_mask, # [Ws, L, n_series]\n",
- " futr_exog=futr_exog, # [Ws, F, L + h, n_series]\n",
- " hist_exog=hist_exog, # [Ws, X, L, n_series]\n",
- " stat_exog=stat_exog) # [n_series, S]\n",
- "\n",
- " # Model Predictions\n",
- " output = self(windows_batch)\n",
- " if self.loss.is_distribution_output:\n",
- " _, y_loc, y_scale = self._inv_normalization(y_hat=torch.empty(size=(insample_y.shape[0], \n",
- " self.h, \n",
- " self.n_series),\n",
- " dtype=output[0].dtype,\n",
- " device=output[0].device),\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " _, y_hat = self.loss.sample(distr_args=distr_args)\n",
- "\n",
- " if self.loss.return_params:\n",
- " distr_args = torch.stack(distr_args, dim=-1)\n",
- " distr_args = torch.reshape(distr_args, (len(windows[\"temporal\"]), self.h, -1))\n",
- " y_hat = torch.concat((y_hat, distr_args), axis=2)\n",
- " else:\n",
- " y_hat, _, _ = self._inv_normalization(y_hat=output,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " return y_hat\n",
- " \n",
- " def fit(self, dataset, val_size=0, test_size=0, random_seed=None, distributed_config=None):\n",
- " \"\"\" Fit.\n",
- "\n",
- " The `fit` method, optimizes the neural network's weights using the\n",
- " initialization parameters (`learning_rate`, `windows_batch_size`, ...)\n",
- " and the `loss` function as defined during the initialization. \n",
- " Within `fit` we use a PyTorch Lightning `Trainer` that\n",
- " inherits the initialization's `self.trainer_kwargs`, to customize\n",
- " its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).\n",
- "\n",
- " The method is designed to be compatible with SKLearn-like classes\n",
- " and in particular to be compatible with the StatsForecast library.\n",
- "\n",
- " By default the `model` is not saving training checkpoints to protect \n",
- " disk memory, to get them change `enable_checkpointing=True` in `__init__`.\n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `val_size`: int, validation size for temporal cross-validation.
\n",
- " `test_size`: int, test size for temporal cross-validation.
\n",
- " \"\"\"\n",
- " if distributed_config is not None:\n",
- " raise ValueError(\"multivariate models cannot be trained using distributed data parallel.\")\n",
- " return self._fit(\n",
- " dataset=dataset,\n",
- " batch_size=self.n_series,\n",
- " valid_batch_size=self.n_series,\n",
- " val_size=val_size,\n",
- " test_size=test_size,\n",
- " random_seed=random_seed,\n",
- " shuffle_train=False,\n",
- " distributed_config=None,\n",
- " )\n",
- "\n",
- " def predict(self, dataset, test_size=None, step_size=1, random_seed=None, **data_module_kwargs):\n",
- " \"\"\" Predict.\n",
- "\n",
- " Neural network prediction with PL's `Trainer` execution of `predict_step`.\n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `test_size`: int=None, test size for temporal cross-validation.
\n",
- " `step_size`: int=1, Step size between each window.
\n",
- " `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).\n",
- " \"\"\"\n",
- " self._check_exog(dataset)\n",
- " self._restart_seed(random_seed)\n",
- " data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)\n",
- "\n",
- " self.predict_step_size = step_size\n",
- " self.decompose_forecast = False\n",
- " datamodule = TimeSeriesDataModule(dataset=dataset, \n",
- " valid_batch_size=self.n_series, \n",
- " batch_size=self.n_series,\n",
- " **data_module_kwargs)\n",
- "\n",
- " # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.\n",
- " pred_trainer_kwargs = self.trainer_kwargs.copy()\n",
- " if (pred_trainer_kwargs.get('accelerator', None) == \"gpu\") and (torch.cuda.device_count() > 1):\n",
- " pred_trainer_kwargs['devices'] = [0]\n",
- "\n",
- " trainer = pl.Trainer(**pred_trainer_kwargs)\n",
- " fcsts = trainer.predict(self, datamodule=datamodule)\n",
- " fcsts = torch.vstack(fcsts).numpy()\n",
- "\n",
- " fcsts = np.transpose(fcsts, (2,0,1))\n",
- " fcsts = fcsts.flatten()\n",
- " fcsts = fcsts.reshape(-1, len(self.loss.output_names))\n",
- " return fcsts\n",
- "\n",
- " def decompose(self, dataset, step_size=1, random_seed=None, **data_module_kwargs):\n",
- " raise NotImplementedError('decompose')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "from fastcore.test import test_fail"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# test unsupported losses\n",
- "test_fail(\n",
- " lambda: BaseMultivariate(\n",
- " h=1,\n",
- " input_size=1,\n",
- " loss=losses.MQLoss(),\n",
- " valid_loss=losses.RMSE(),\n",
- " learning_rate=1,\n",
- " max_steps=1,\n",
- " val_check_steps=1,\n",
- " n_series=1,\n",
- " batch_size=1,\n",
- " ),\n",
- " contains='MQLoss() is not supported'\n",
- ")\n",
- "\n",
- "test_fail(\n",
- " lambda: BaseMultivariate(\n",
- " h=1,\n",
- " input_size=1,\n",
- " loss=losses.RMSE(),\n",
- " valid_loss=losses.MASE(seasonality=1),\n",
- " learning_rate=1,\n",
- " max_steps=1,\n",
- " val_check_steps=1,\n",
- " n_series=1,\n",
- " batch_size=1,\n",
- " ),\n",
- " contains='MASE() is not supported'\n",
- ")"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "python3",
- "language": "python",
- "name": "python3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/nbs/common.base_recurrent.ipynb b/nbs/common.base_recurrent.ipynb
deleted file mode 100644
index 7b0ed5585..000000000
--- a/nbs/common.base_recurrent.ipynb
+++ /dev/null
@@ -1,663 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| default_exp common._base_recurrent"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "%load_ext autoreload\n",
- "%autoreload 2"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# BaseRecurrent"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "> The `BaseRecurrent` class contains standard methods shared across recurrent neural networks; these models possess the ability to process variable-length sequences of inputs through their internal memory states. The class is represented by `LSTM`, `GRU`, and `RNN`, along with other more sophisticated architectures like `MQCNN`."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The standard methods include `TemporalNorm` preprocessing, optimization utilities like parameter initialization, `training_step`, `validation_step`, and shared `fit` and `predict` methods.These shared methods enable all the `neuralforecast.models` compatibility with the `core.NeuralForecast` wrapper class."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "import numpy as np\n",
- "import torch\n",
- "import torch.nn as nn\n",
- "import pytorch_lightning as pl\n",
- "import neuralforecast.losses.pytorch as losses\n",
- "\n",
- "from neuralforecast.common._base_model import BaseModel\n",
- "from neuralforecast.common._scalers import TemporalNorm\n",
- "from neuralforecast.tsdataset import TimeSeriesDataModule\n",
- "from neuralforecast.utils import get_indexer_raise_missing"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "class BaseRecurrent(BaseModel):\n",
- " \"\"\" Base Recurrent\n",
- " \n",
- " Base class for all recurrent-based models. The forecasts are produced sequentially between \n",
- " windows.\n",
- " \n",
- " This class implements the basic functionality for all windows-based models, including:\n",
- " - PyTorch Lightning's methods training_step, validation_step, predict_step.
\n",
- " - fit and predict methods used by NeuralForecast.core class.
\n",
- " - sampling and wrangling methods to sequential windows.
\n",
- " \"\"\"\n",
- " def __init__(self,\n",
- " h,\n",
- " input_size,\n",
- " inference_input_size,\n",
- " loss,\n",
- " valid_loss,\n",
- " learning_rate,\n",
- " max_steps,\n",
- " val_check_steps,\n",
- " batch_size,\n",
- " valid_batch_size,\n",
- " scaler_type='robust',\n",
- " num_lr_decays=0,\n",
- " early_stop_patience_steps=-1,\n",
- " futr_exog_list=None,\n",
- " hist_exog_list=None,\n",
- " stat_exog_list=None,\n",
- " num_workers_loader=0,\n",
- " drop_last_loader=False,\n",
- " random_seed=1, \n",
- " alias=None,\n",
- " optimizer=None,\n",
- " optimizer_kwargs=None,\n",
- " lr_scheduler=None,\n",
- " lr_scheduler_kwargs=None,\n",
- " dataloader_kwargs=None,\n",
- " **trainer_kwargs):\n",
- " super().__init__(\n",
- " random_seed=random_seed,\n",
- " loss=loss,\n",
- " valid_loss=valid_loss,\n",
- " optimizer=optimizer,\n",
- " optimizer_kwargs=optimizer_kwargs,\n",
- " lr_scheduler=lr_scheduler,\n",
- " lr_scheduler_kwargs=lr_scheduler_kwargs,\n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
- " max_steps=max_steps,\n",
- " early_stop_patience_steps=early_stop_patience_steps, \n",
- " **trainer_kwargs,\n",
- " )\n",
- "\n",
- " # Padder to complete train windows, \n",
- " # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]\n",
- " self.h = h\n",
- " self.input_size = input_size\n",
- " self.inference_input_size = inference_input_size\n",
- " self.padder = nn.ConstantPad1d(padding=(0, self.h), value=0.0)\n",
- "\n",
- " unsupported_distributions = ['Bernoulli', 'ISQF']\n",
- " if isinstance(self.loss, losses.DistributionLoss) and\\\n",
- " self.loss.distribution in unsupported_distributions:\n",
- " raise Exception(f'Distribution {self.loss.distribution} not available for Recurrent-based models. Please choose another distribution.')\n",
- "\n",
- " # Valid batch_size\n",
- " self.batch_size = batch_size\n",
- " if valid_batch_size is None:\n",
- " self.valid_batch_size = batch_size\n",
- " else:\n",
- " self.valid_batch_size = valid_batch_size\n",
- "\n",
- " # Optimization\n",
- " self.learning_rate = learning_rate\n",
- " self.max_steps = max_steps\n",
- " self.num_lr_decays = num_lr_decays\n",
- " self.lr_decay_steps = max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7\n",
- " self.early_stop_patience_steps = early_stop_patience_steps\n",
- " self.val_check_steps = val_check_steps\n",
- "\n",
- " # Scaler\n",
- " self.scaler = TemporalNorm(\n",
- " scaler_type=scaler_type,\n",
- " dim=-1, # Time dimension is -1.\n",
- " num_features=1+len(self.hist_exog_list)+len(self.futr_exog_list)\n",
- " )\n",
- "\n",
- " # Fit arguments\n",
- " self.val_size = 0\n",
- " self.test_size = 0\n",
- "\n",
- " # DataModule arguments\n",
- " self.num_workers_loader = num_workers_loader\n",
- " self.dataloader_kwargs = dataloader_kwargs\n",
- " self.drop_last_loader = drop_last_loader\n",
- " # used by on_validation_epoch_end hook\n",
- " self.validation_step_outputs = []\n",
- " self.alias = alias\n",
- "\n",
- " def _normalization(self, batch, val_size=0, test_size=0):\n",
- " temporal = batch['temporal'] # B, C, T\n",
- " temporal_cols = batch['temporal_cols'].copy()\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Separate data and mask\n",
- " temporal_data_cols = self._get_temporal_exogenous_cols(temporal_cols=temporal_cols)\n",
- " temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)\n",
- " temporal_idxs = np.append(y_idx, temporal_idxs)\n",
- " temporal_data = temporal[:, temporal_idxs, :]\n",
- " temporal_mask = temporal[:, temporal_cols.get_loc('available_mask'), :].clone()\n",
- "\n",
- " # Remove validation and test set to prevent leakeage\n",
- " if val_size + test_size > 0:\n",
- " cutoff = val_size + test_size\n",
- " temporal_mask[:, -cutoff:] = 0\n",
- "\n",
- " # Normalize. self.scaler stores the shift and scale for inverse transform\n",
- " temporal_mask = temporal_mask.unsqueeze(1) # Add channel dimension for scaler.transform.\n",
- " temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)\n",
- "\n",
- " # Replace values in windows dict\n",
- " temporal[:, temporal_idxs, :] = temporal_data\n",
- " batch['temporal'] = temporal\n",
- "\n",
- " return batch\n",
- "\n",
- " def _inv_normalization(self, y_hat, temporal_cols, y_idx):\n",
- " # Receives window predictions [B, seq_len, H, output]\n",
- " # Broadcasts outputs and inverts normalization\n",
- "\n",
- " # Get 'y' scale and shift, and add W dimension\n",
- " y_loc = self.scaler.x_shift[:, [y_idx], 0].flatten() #[B,C,T] -> [B] \n",
- " y_scale = self.scaler.x_scale[:, [y_idx], 0].flatten() #[B,C,T] -> [B]\n",
- "\n",
- " # Expand scale and shift to y_hat dimensions\n",
- " y_loc = y_loc.view(*y_loc.shape, *(1,)*(y_hat.ndim-1))#.expand(y_hat) \n",
- " y_scale = y_scale.view(*y_scale.shape, *(1,)*(y_hat.ndim-1))#.expand(y_hat)\n",
- "\n",
- " y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)\n",
- "\n",
- " return y_hat, y_loc, y_scale\n",
- "\n",
- " def _create_windows(self, batch, step):\n",
- " temporal = batch['temporal']\n",
- " temporal_cols = batch['temporal_cols']\n",
- "\n",
- " if step == 'train':\n",
- " if self.val_size + self.test_size > 0:\n",
- " cutoff = -self.val_size - self.test_size\n",
- " temporal = temporal[:, :, :cutoff]\n",
- " temporal = self.padder(temporal)\n",
- "\n",
- " # Truncate batch to shorter time-series \n",
- " av_condition = torch.nonzero(torch.min(temporal[:, temporal_cols.get_loc('available_mask')], axis=0).values)\n",
- " min_time_stamp = int(av_condition.min())\n",
- " \n",
- " available_ts = temporal.shape[-1] - min_time_stamp\n",
- " if available_ts < 1 + self.h:\n",
- " raise Exception(\n",
- " 'Time series too short for given input and output size. \\n'\n",
- " f'Available timestamps: {available_ts}'\n",
- " )\n",
- "\n",
- " temporal = temporal[:, :, min_time_stamp:]\n",
- "\n",
- " if step == 'val':\n",
- " if self.test_size > 0:\n",
- " temporal = temporal[:, :, :-self.test_size]\n",
- " temporal = self.padder(temporal)\n",
- "\n",
- " if step == 'predict':\n",
- " if (self.test_size == 0) and (len(self.futr_exog_list)==0):\n",
- " temporal = self.padder(temporal)\n",
- "\n",
- " # Test size covers all data, pad left one timestep with zeros\n",
- " if temporal.shape[-1] == self.test_size:\n",
- " padder_left = nn.ConstantPad1d(padding=(1, 0), value=0.0)\n",
- " temporal = padder_left(temporal)\n",
- "\n",
- " # Parse batch\n",
- " window_size = 1 + self.h # 1 for current t and h for future\n",
- " windows = temporal.unfold(dimension=-1,\n",
- " size=window_size,\n",
- " step=1)\n",
- "\n",
- " # Truncated backprogatation/inference (shorten sequence where RNNs unroll)\n",
- " n_windows = windows.shape[2]\n",
- " input_size = -1\n",
- " if (step == 'train') and (self.input_size>0):\n",
- " input_size = self.input_size\n",
- " if (input_size > 0) and (n_windows > input_size):\n",
- " max_sampleable_time = n_windows-self.input_size+1\n",
- " start = np.random.choice(max_sampleable_time)\n",
- " windows = windows[:, :, start:(start+input_size), :]\n",
- "\n",
- " if (step == 'val') and (self.inference_input_size>0):\n",
- " cutoff = self.inference_input_size + self.val_size\n",
- " windows = windows[:, :, -cutoff:, :]\n",
- "\n",
- " if (step == 'predict') and (self.inference_input_size>0):\n",
- " cutoff = self.inference_input_size + self.test_size\n",
- " windows = windows[:, :, -cutoff:, :]\n",
- " \n",
- " # [B, C, input_size, 1+H]\n",
- " windows_batch = dict(temporal=windows,\n",
- " temporal_cols=temporal_cols,\n",
- " static=batch.get('static', None),\n",
- " static_cols=batch.get('static_cols', None))\n",
- "\n",
- " return windows_batch\n",
- "\n",
- " def _parse_windows(self, batch, windows):\n",
- " # [B, C, seq_len, 1+H]\n",
- " # Filter insample lags from outsample horizon\n",
- " mask_idx = batch['temporal_cols'].get_loc('available_mask')\n",
- " y_idx = batch['y_idx'] \n",
- " insample_y = windows['temporal'][:, y_idx, :, :-self.h]\n",
- " insample_mask = windows['temporal'][:, mask_idx, :, :-self.h]\n",
- " outsample_y = windows['temporal'][:, y_idx, :, -self.h:].contiguous()\n",
- " outsample_mask = windows['temporal'][:, mask_idx, :, -self.h:].contiguous()\n",
- "\n",
- " # Filter historic exogenous variables\n",
- " if len(self.hist_exog_list):\n",
- " hist_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.hist_exog_list)\n",
- " hist_exog = windows['temporal'][:, hist_exog_idx, :, :-self.h]\n",
- " else:\n",
- " hist_exog = None\n",
- " \n",
- " # Filter future exogenous variables\n",
- " if len(self.futr_exog_list):\n",
- " futr_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.futr_exog_list)\n",
- " futr_exog = windows['temporal'][:, futr_exog_idx, :, :]\n",
- " else:\n",
- " futr_exog = None\n",
- " # Filter static variables\n",
- " if len(self.stat_exog_list):\n",
- " static_idx = get_indexer_raise_missing(windows['static_cols'], self.stat_exog_list)\n",
- " stat_exog = windows['static'][:, static_idx]\n",
- " else:\n",
- " stat_exog = None\n",
- "\n",
- " return insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog\n",
- "\n",
- " def training_step(self, batch, batch_idx):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " batch = self._normalization(batch, val_size=self.val_size, test_size=self.test_size)\n",
- " windows = self._create_windows(batch, step='train')\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [B, seq_len, 1]\n",
- " insample_mask=insample_mask, # [B, seq_len, 1]\n",
- " futr_exog=futr_exog, # [B, F, seq_len, 1+H]\n",
- " hist_exog=hist_exog, # [B, C, seq_len]\n",
- " stat_exog=stat_exog) # [B, S]\n",
- "\n",
- " # Model predictions\n",
- " output = self(windows_batch) # tuple([B, seq_len, H, output])\n",
- " if self.loss.is_distribution_output:\n",
- " outsample_y, y_loc, y_scale = self._inv_normalization(y_hat=outsample_y,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=batch['y_idx'])\n",
- " B = output[0].size()[0]\n",
- " T = output[0].size()[1]\n",
- " H = output[0].size()[2]\n",
- " output = [arg.view(-1, *(arg.size()[2:])) for arg in output]\n",
- " outsample_y = outsample_y.view(B*T,H)\n",
- " outsample_mask = outsample_mask.view(B*T,H)\n",
- " y_loc = y_loc.repeat_interleave(repeats=T, dim=0).squeeze(-1)\n",
- " y_scale = y_scale.repeat_interleave(repeats=T, dim=0).squeeze(-1)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
- " else:\n",
- " loss = self.loss(y=outsample_y, y_hat=output, mask=outsample_mask)\n",
- "\n",
- " if torch.isnan(loss):\n",
- " print('Model Parameters', self.hparams)\n",
- " print('insample_y', torch.isnan(insample_y).sum())\n",
- " print('outsample_y', torch.isnan(outsample_y).sum())\n",
- " print('output', torch.isnan(output).sum())\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'train_loss',\n",
- " loss.detach().item(),\n",
- " batch_size=outsample_y.size(0),\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.train_trajectories.append((self.global_step, loss.detach().item()))\n",
- " return loss\n",
- "\n",
- " def validation_step(self, batch, batch_idx):\n",
- " if self.val_size == 0:\n",
- " return np.nan\n",
- "\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " batch = self._normalization(batch, val_size=self.val_size, test_size=self.test_size)\n",
- " windows = self._create_windows(batch, step='val')\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [B, seq_len, 1]\n",
- " insample_mask=insample_mask, # [B, seq_len, 1]\n",
- " futr_exog=futr_exog, # [B, F, seq_len, 1+H]\n",
- " hist_exog=hist_exog, # [B, C, seq_len]\n",
- " stat_exog=stat_exog) # [B, S]\n",
- "\n",
- " # Remove train y_hat (+1 and -1 for padded last window with zeros)\n",
- " # tuple([B, seq_len, H, output]) -> tuple([B, validation_size, H, output])\n",
- " val_windows = (self.val_size) + 1\n",
- " outsample_y = outsample_y[:, -val_windows:-1, :]\n",
- " outsample_mask = outsample_mask[:, -val_windows:-1, :] \n",
- "\n",
- " # Model predictions\n",
- " output = self(windows_batch) # tuple([B, seq_len, H, output])\n",
- " if self.loss.is_distribution_output:\n",
- " output = [arg[:, -val_windows:-1] for arg in output]\n",
- " outsample_y, y_loc, y_scale = self._inv_normalization(y_hat=outsample_y,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " B = output[0].size()[0]\n",
- " T = output[0].size()[1]\n",
- " H = output[0].size()[2]\n",
- " output = [arg.reshape(-1, *(arg.size()[2:])) for arg in output]\n",
- " outsample_y = outsample_y.reshape(B*T,H)\n",
- " outsample_mask = outsample_mask.reshape(B*T,H)\n",
- " y_loc = y_loc.repeat_interleave(repeats=T, dim=0).squeeze(-1)\n",
- " y_scale = y_scale.repeat_interleave(repeats=T, dim=0).squeeze(-1)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " _, sample_mean, quants = self.loss.sample(distr_args=distr_args)\n",
- "\n",
- " if str(type(self.valid_loss)) in\\\n",
- " [\"\", \"\"]:\n",
- " output = quants\n",
- " elif str(type(self.valid_loss)) in [\"\"]:\n",
- " output = torch.unsqueeze(sample_mean, dim=-1) # [N,H,1] -> [N,H]\n",
- " \n",
- " else:\n",
- " output = output[:, -val_windows:-1, :]\n",
- "\n",
- " # Validation Loss evaluation\n",
- " if self.valid_loss.is_distribution_output:\n",
- " valid_loss = self.valid_loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
- " else:\n",
- " outsample_y, _, _ = self._inv_normalization(y_hat=outsample_y, temporal_cols=batch['temporal_cols'], y_idx=y_idx)\n",
- " output, _, _ = self._inv_normalization(y_hat=output, temporal_cols=batch['temporal_cols'], y_idx=y_idx)\n",
- " valid_loss = self.valid_loss(y=outsample_y, y_hat=output, mask=outsample_mask)\n",
- "\n",
- " if torch.isnan(valid_loss):\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'valid_loss',\n",
- " valid_loss.detach().item(),\n",
- " batch_size=outsample_y.size(0),\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.validation_step_outputs.append(valid_loss)\n",
- " return valid_loss\n",
- "\n",
- " def predict_step(self, batch, batch_idx):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " batch = self._normalization(batch, val_size=0, test_size=self.test_size)\n",
- " windows = self._create_windows(batch, step='predict')\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, _, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [B, seq_len, 1]\n",
- " insample_mask=insample_mask, # [B, seq_len, 1]\n",
- " futr_exog=futr_exog, # [B, F, seq_len, 1+H]\n",
- " hist_exog=hist_exog, # [B, C, seq_len]\n",
- " stat_exog=stat_exog) # [B, S]\n",
- "\n",
- " # Model Predictions\n",
- " output = self(windows_batch) # tuple([B, seq_len, H], ...)\n",
- " if self.loss.is_distribution_output:\n",
- " _, y_loc, y_scale = self._inv_normalization(y_hat=output[0],\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " B = output[0].size()[0]\n",
- " T = output[0].size()[1]\n",
- " H = output[0].size()[2]\n",
- " output = [arg.reshape(-1, *(arg.size()[2:])) for arg in output]\n",
- " y_loc = y_loc.repeat_interleave(repeats=T, dim=0).squeeze(-1)\n",
- " y_scale = y_scale.repeat_interleave(repeats=T, dim=0).squeeze(-1)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " _, sample_mean, quants = self.loss.sample(distr_args=distr_args)\n",
- " y_hat = torch.concat((sample_mean, quants), axis=2)\n",
- " y_hat = y_hat.view(B, T, H, -1)\n",
- "\n",
- " if self.loss.return_params:\n",
- " distr_args = torch.stack(distr_args, dim=-1)\n",
- " distr_args = torch.reshape(distr_args, (B, T, H, -1))\n",
- " y_hat = torch.concat((y_hat, distr_args), axis=3)\n",
- " else:\n",
- " y_hat, _, _ = self._inv_normalization(y_hat=output,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " return y_hat\n",
- "\n",
- " def fit(self, dataset, val_size=0, test_size=0, random_seed=None, distributed_config=None):\n",
- " \"\"\" Fit.\n",
- "\n",
- " The `fit` method, optimizes the neural network's weights using the\n",
- " initialization parameters (`learning_rate`, `batch_size`, ...)\n",
- " and the `loss` function as defined during the initialization. \n",
- " Within `fit` we use a PyTorch Lightning `Trainer` that\n",
- " inherits the initialization's `self.trainer_kwargs`, to customize\n",
- " its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).\n",
- "\n",
- " The method is designed to be compatible with SKLearn-like classes\n",
- " and in particular to be compatible with the StatsForecast library.\n",
- "\n",
- " By default the `model` is not saving training checkpoints to protect \n",
- " disk memory, to get them change `enable_checkpointing=True` in `__init__`. \n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `val_size`: int, validation size for temporal cross-validation.
\n",
- " `test_size`: int, test size for temporal cross-validation.
\n",
- " `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
- " \"\"\"\n",
- " return self._fit(\n",
- " dataset=dataset,\n",
- " batch_size=self.batch_size,\n",
- " valid_batch_size=self.valid_batch_size,\n",
- " val_size=val_size,\n",
- " test_size=test_size,\n",
- " random_seed=random_seed,\n",
- " distributed_config=distributed_config,\n",
- " )\n",
- "\n",
- " def predict(self, dataset, step_size=1,\n",
- " random_seed=None, **data_module_kwargs):\n",
- " \"\"\" Predict.\n",
- "\n",
- " Neural network prediction with PL's `Trainer` execution of `predict_step`.\n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `step_size`: int=1, Step size between each window.
\n",
- " `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
- " `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).\n",
- " \"\"\"\n",
- " self._check_exog(dataset)\n",
- " self._restart_seed(random_seed)\n",
- " data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)\n",
- " \n",
- " if step_size > 1:\n",
- " raise Exception('Recurrent models do not support step_size > 1')\n",
- "\n",
- " # fcsts (window, batch, h)\n",
- " # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.\n",
- " pred_trainer_kwargs = self.trainer_kwargs.copy()\n",
- " if (pred_trainer_kwargs.get('accelerator', None) == \"gpu\") and (torch.cuda.device_count() > 1):\n",
- " pred_trainer_kwargs['devices'] = [0]\n",
- "\n",
- " trainer = pl.Trainer(**pred_trainer_kwargs)\n",
- "\n",
- " datamodule = TimeSeriesDataModule(\n",
- " dataset=dataset,\n",
- " valid_batch_size=self.valid_batch_size,\n",
- " num_workers=self.num_workers_loader,\n",
- " **data_module_kwargs\n",
- " )\n",
- " fcsts = trainer.predict(self, datamodule=datamodule)\n",
- " if self.test_size > 0:\n",
- " # Remove warmup windows (from train and validation)\n",
- " # [N,T,H,output], avoid indexing last dim for univariate output compatibility\n",
- " fcsts = torch.vstack([fcst[:, -(1+self.test_size-self.h):,:] for fcst in fcsts])\n",
- " fcsts = fcsts.numpy().flatten()\n",
- " fcsts = fcsts.reshape(-1, len(self.loss.output_names))\n",
- " else:\n",
- " fcsts = torch.vstack([fcst[:,-1:,:] for fcst in fcsts]).numpy().flatten()\n",
- " fcsts = fcsts.reshape(-1, len(self.loss.output_names))\n",
- " return fcsts"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseRecurrent, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseRecurrent.fit, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseRecurrent.predict, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.utils import AirPassengersDF\n",
- "from neuralforecast.tsdataset import TimeSeriesDataset, TimeSeriesDataModule"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# add h=0,1 unit test for _parse_windows \n",
- "# Declare batch\n",
- "AirPassengersDF['x'] = np.array(len(AirPassengersDF))\n",
- "AirPassengersDF['x2'] = np.array(len(AirPassengersDF)) * 2\n",
- "dataset, indices, dates, ds = TimeSeriesDataset.from_df(df=AirPassengersDF)\n",
- "data = TimeSeriesDataModule(dataset=dataset, batch_size=1, drop_last=True)\n",
- "\n",
- "train_loader = data.train_dataloader()\n",
- "batch = next(iter(train_loader))\n",
- "\n",
- "# Test that hist_exog_list and futr_exog_list correctly filter data that is sent to scaler.\n",
- "baserecurrent = BaseRecurrent(h=12,\n",
- " input_size=117,\n",
- " hist_exog_list=['x', 'x2'],\n",
- " futr_exog_list=['x'],\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " learning_rate=0.001,\n",
- " max_steps=1,\n",
- " val_check_steps=0,\n",
- " batch_size=1,\n",
- " valid_batch_size=1,\n",
- " windows_batch_size=10,\n",
- " inference_input_size=2,\n",
- " start_padding_enabled=True)\n",
- "\n",
- "windows = baserecurrent._create_windows(batch, step='train')\n",
- "\n",
- "temporal_cols = windows['temporal_cols'].copy() # B, L+H, C\n",
- "temporal_data_cols = baserecurrent._get_temporal_exogenous_cols(temporal_cols=temporal_cols)\n",
- "\n",
- "test_eq(set(temporal_data_cols), set(['x', 'x2']))\n",
- "test_eq(windows['temporal'].shape, torch.Size([1,len(['y', 'x', 'x2', 'available_mask']),117,12+1]))"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "python3",
- "language": "python",
- "name": "python3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/nbs/common.base_windows.ipynb b/nbs/common.base_windows.ipynb
deleted file mode 100644
index 80f12e5f5..000000000
--- a/nbs/common.base_windows.ipynb
+++ /dev/null
@@ -1,897 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "524620c1",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| default_exp common._base_windows"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "15392f6f",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "%load_ext autoreload\n",
- "%autoreload 2"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "1e0f9607-d12d-44e5-b2be-91a57a0bca79",
- "metadata": {},
- "source": [
- "# BaseWindows\n",
- "\n",
- "> The `BaseWindows` class contains standard methods shared across window-based neural networks; in contrast to recurrent neural networks these models commit to a fixed sequence length input. The class is represented by `MLP`, and other more sophisticated architectures like `NBEATS`, and `NHITS`."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "1730a556-1574-40ad-92a2-23b924ceb398",
- "metadata": {},
- "source": [
- "The standard methods include data preprocessing `_normalization`, optimization utilities like parameter initialization, `training_step`, `validation_step`, and shared `fit` and `predict` methods.These shared methods enable all the `neuralforecast.models` compatibility with the `core.NeuralForecast` wrapper class. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "2508f7a9-1433-4ad8-8f2f-0078c6ed6c3c",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "44065066-e72a-431f-938f-1528adef9fe8",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "import numpy as np\n",
- "import torch\n",
- "import torch.nn as nn\n",
- "import pytorch_lightning as pl\n",
- "\n",
- "from neuralforecast.common._base_model import BaseModel\n",
- "from neuralforecast.common._scalers import TemporalNorm\n",
- "from neuralforecast.tsdataset import TimeSeriesDataModule\n",
- "from neuralforecast.utils import get_indexer_raise_missing"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ce70cd14-ecb1-4205-8511-fecbd26c8408",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "class BaseWindows(BaseModel):\n",
- " \"\"\" Base Windows\n",
- " \n",
- " Base class for all windows-based models. The forecasts are produced separately \n",
- " for each window, which are randomly sampled during training.\n",
- " \n",
- " This class implements the basic functionality for all windows-based models, including:\n",
- " - PyTorch Lightning's methods training_step, validation_step, predict_step.
\n",
- " - fit and predict methods used by NeuralForecast.core class.
\n",
- " - sampling and wrangling methods to generate windows.\n",
- " \"\"\"\n",
- " def __init__(self,\n",
- " h,\n",
- " input_size,\n",
- " loss,\n",
- " valid_loss,\n",
- " learning_rate,\n",
- " max_steps,\n",
- " val_check_steps,\n",
- " batch_size,\n",
- " valid_batch_size,\n",
- " windows_batch_size,\n",
- " inference_windows_batch_size,\n",
- " start_padding_enabled,\n",
- " step_size=1,\n",
- " num_lr_decays=0,\n",
- " early_stop_patience_steps=-1,\n",
- " scaler_type='identity',\n",
- " futr_exog_list=None,\n",
- " hist_exog_list=None,\n",
- " stat_exog_list=None,\n",
- " exclude_insample_y=False,\n",
- " num_workers_loader=0,\n",
- " drop_last_loader=False,\n",
- " random_seed=1,\n",
- " alias=None,\n",
- " optimizer=None,\n",
- " optimizer_kwargs=None,\n",
- " lr_scheduler=None,\n",
- " lr_scheduler_kwargs=None,\n",
- " dataloader_kwargs=None,\n",
- " **trainer_kwargs):\n",
- " super().__init__(\n",
- " random_seed=random_seed,\n",
- " loss=loss,\n",
- " valid_loss=valid_loss,\n",
- " optimizer=optimizer,\n",
- " optimizer_kwargs=optimizer_kwargs,\n",
- " lr_scheduler=lr_scheduler,\n",
- " lr_scheduler_kwargs=lr_scheduler_kwargs,\n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
- " max_steps=max_steps,\n",
- " early_stop_patience_steps=early_stop_patience_steps, \n",
- " **trainer_kwargs,\n",
- " )\n",
- "\n",
- " # Padder to complete train windows, \n",
- " # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]\n",
- " self.h = h\n",
- " self.input_size = input_size\n",
- " self.windows_batch_size = windows_batch_size\n",
- " self.start_padding_enabled = start_padding_enabled\n",
- " if start_padding_enabled:\n",
- " self.padder_train = nn.ConstantPad1d(padding=(self.input_size-1, self.h), value=0.0)\n",
- " else:\n",
- " self.padder_train = nn.ConstantPad1d(padding=(0, self.h), value=0.0)\n",
- "\n",
- " # Batch sizes\n",
- " self.batch_size = batch_size\n",
- " if valid_batch_size is None:\n",
- " self.valid_batch_size = batch_size\n",
- " else:\n",
- " self.valid_batch_size = valid_batch_size\n",
- " if inference_windows_batch_size is None:\n",
- " self.inference_windows_batch_size = windows_batch_size\n",
- " else:\n",
- " self.inference_windows_batch_size = inference_windows_batch_size\n",
- "\n",
- " # Optimization \n",
- " self.learning_rate = learning_rate\n",
- " self.max_steps = max_steps\n",
- " self.num_lr_decays = num_lr_decays\n",
- " self.lr_decay_steps = (\n",
- " max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7\n",
- " )\n",
- " self.early_stop_patience_steps = early_stop_patience_steps\n",
- " self.val_check_steps = val_check_steps\n",
- " self.windows_batch_size = windows_batch_size\n",
- " self.step_size = step_size\n",
- " \n",
- " self.exclude_insample_y = exclude_insample_y\n",
- "\n",
- " # Scaler\n",
- " self.scaler = TemporalNorm(\n",
- " scaler_type=scaler_type,\n",
- " dim=1, # Time dimension is 1.\n",
- " num_features=1+len(self.hist_exog_list)+len(self.futr_exog_list)\n",
- " )\n",
- "\n",
- " # Fit arguments\n",
- " self.val_size = 0\n",
- " self.test_size = 0\n",
- "\n",
- " # Model state\n",
- " self.decompose_forecast = False\n",
- "\n",
- " # DataModule arguments\n",
- " self.num_workers_loader = num_workers_loader\n",
- " self.dataloader_kwargs = dataloader_kwargs\n",
- " self.drop_last_loader = drop_last_loader\n",
- " # used by on_validation_epoch_end hook\n",
- " self.validation_step_outputs = []\n",
- " self.alias = alias\n",
- "\n",
- " def _create_windows(self, batch, step, w_idxs=None):\n",
- " # Parse common data\n",
- " window_size = self.input_size + self.h\n",
- " temporal_cols = batch['temporal_cols']\n",
- " temporal = batch['temporal']\n",
- "\n",
- " if step == 'train':\n",
- " if self.val_size + self.test_size > 0:\n",
- " cutoff = -self.val_size - self.test_size\n",
- " temporal = temporal[:, :, :cutoff]\n",
- "\n",
- " temporal = self.padder_train(temporal)\n",
- " if temporal.shape[-1] < window_size:\n",
- " raise Exception('Time series is too short for training, consider setting a smaller input size or set start_padding_enabled=True')\n",
- " windows = temporal.unfold(dimension=-1, \n",
- " size=window_size, \n",
- " step=self.step_size)\n",
- "\n",
- " # [B, C, Ws, L+H] 0, 1, 2, 3\n",
- " # -> [B * Ws, L+H, C] 0, 2, 3, 1\n",
- " windows_per_serie = windows.shape[2]\n",
- " windows = windows.permute(0, 2, 3, 1).contiguous()\n",
- " windows = windows.reshape(-1, window_size, len(temporal_cols))\n",
- "\n",
- " # Sample and Available conditions\n",
- " available_idx = temporal_cols.get_loc('available_mask')\n",
- " available_condition = windows[:, :self.input_size, available_idx]\n",
- " available_condition = torch.sum(available_condition, axis=1)\n",
- " final_condition = (available_condition > 0)\n",
- " if self.h > 0:\n",
- " sample_condition = windows[:, self.input_size:, available_idx]\n",
- " sample_condition = torch.sum(sample_condition, axis=1)\n",
- " final_condition = (sample_condition > 0) & (available_condition > 0)\n",
- " windows = windows[final_condition]\n",
- "\n",
- " # Parse Static data to match windows\n",
- " # [B, S_in] -> [B, Ws, S_in] -> [B*Ws, S_in]\n",
- " static = batch.get('static', None)\n",
- " static_cols=batch.get('static_cols', None)\n",
- " if static is not None:\n",
- " static = torch.repeat_interleave(static, \n",
- " repeats=windows_per_serie, dim=0)\n",
- " static = static[final_condition]\n",
- "\n",
- " # Protection of empty windows\n",
- " if final_condition.sum() == 0:\n",
- " raise Exception('No windows available for training')\n",
- "\n",
- " # Sample windows\n",
- " n_windows = len(windows)\n",
- " if self.windows_batch_size is not None:\n",
- " w_idxs = np.random.choice(n_windows, \n",
- " size=self.windows_batch_size,\n",
- " replace=(n_windows < self.windows_batch_size))\n",
- " windows = windows[w_idxs]\n",
- " \n",
- " if static is not None:\n",
- " static = static[w_idxs]\n",
- "\n",
- " # think about interaction available * sample mask\n",
- " # [B, C, Ws, L+H]\n",
- " windows_batch = dict(temporal=windows,\n",
- " temporal_cols=temporal_cols,\n",
- " static=static,\n",
- " static_cols=static_cols)\n",
- " return windows_batch\n",
- "\n",
- " elif step in ['predict', 'val']:\n",
- "\n",
- " if step == 'predict':\n",
- " initial_input = temporal.shape[-1] - self.test_size\n",
- " if initial_input <= self.input_size: # There is not enough data to predict first timestamp\n",
- " padder_left = nn.ConstantPad1d(padding=(self.input_size-initial_input, 0), value=0.0)\n",
- " temporal = padder_left(temporal)\n",
- " predict_step_size = self.predict_step_size\n",
- " cutoff = - self.input_size - self.test_size\n",
- " temporal = temporal[:, :, cutoff:]\n",
- "\n",
- " elif step == 'val':\n",
- " predict_step_size = self.step_size\n",
- " cutoff = -self.input_size - self.val_size - self.test_size\n",
- " if self.test_size > 0:\n",
- " temporal = batch['temporal'][:, :, cutoff:-self.test_size]\n",
- " else:\n",
- " temporal = batch['temporal'][:, :, cutoff:]\n",
- " if temporal.shape[-1] < window_size:\n",
- " initial_input = temporal.shape[-1] - self.val_size\n",
- " padder_left = nn.ConstantPad1d(padding=(self.input_size-initial_input, 0), value=0.0)\n",
- " temporal = padder_left(temporal)\n",
- "\n",
- " if (step=='predict') and (self.test_size==0) and (len(self.futr_exog_list)==0):\n",
- " padder_right = nn.ConstantPad1d(padding=(0, self.h), value=0.0)\n",
- " temporal = padder_right(temporal)\n",
- "\n",
- " windows = temporal.unfold(dimension=-1,\n",
- " size=window_size,\n",
- " step=predict_step_size)\n",
- "\n",
- " # [batch, channels, windows, window_size] 0, 1, 2, 3\n",
- " # -> [batch * windows, window_size, channels] 0, 2, 3, 1\n",
- " windows_per_serie = windows.shape[2]\n",
- " windows = windows.permute(0, 2, 3, 1).contiguous()\n",
- " windows = windows.reshape(-1, window_size, len(temporal_cols))\n",
- "\n",
- " static = batch.get('static', None)\n",
- " static_cols=batch.get('static_cols', None)\n",
- " if static is not None:\n",
- " static = torch.repeat_interleave(static, \n",
- " repeats=windows_per_serie, dim=0)\n",
- " \n",
- " # Sample windows for batched prediction\n",
- " if w_idxs is not None:\n",
- " windows = windows[w_idxs]\n",
- " if static is not None:\n",
- " static = static[w_idxs]\n",
- " \n",
- " windows_batch = dict(temporal=windows,\n",
- " temporal_cols=temporal_cols,\n",
- " static=static,\n",
- " static_cols=static_cols)\n",
- " return windows_batch\n",
- " else:\n",
- " raise ValueError(f'Unknown step {step}')\n",
- "\n",
- " def _normalization(self, windows, y_idx):\n",
- " # windows are already filtered by train/validation/test\n",
- " # from the `create_windows_method` nor leakage risk\n",
- " temporal = windows['temporal'] # B, L+H, C\n",
- " temporal_cols = windows['temporal_cols'].copy() # B, L+H, C\n",
- "\n",
- " # To avoid leakage uses only the lags\n",
- " #temporal_data_cols = temporal_cols.drop('available_mask').tolist()\n",
- " temporal_data_cols = self._get_temporal_exogenous_cols(temporal_cols=temporal_cols)\n",
- " temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)\n",
- " temporal_idxs = np.append(y_idx, temporal_idxs)\n",
- " temporal_data = temporal[:, :, temporal_idxs]\n",
- " temporal_mask = temporal[:, :, temporal_cols.get_loc('available_mask')].clone()\n",
- " if self.h > 0:\n",
- " temporal_mask[:, -self.h:] = 0.0\n",
- "\n",
- " # Normalize. self.scaler stores the shift and scale for inverse transform\n",
- " temporal_mask = temporal_mask.unsqueeze(-1) # Add channel dimension for scaler.transform.\n",
- " temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)\n",
- "\n",
- " # Replace values in windows dict\n",
- " temporal[:, :, temporal_idxs] = temporal_data\n",
- " windows['temporal'] = temporal\n",
- "\n",
- " return windows\n",
- "\n",
- " def _inv_normalization(self, y_hat, temporal_cols, y_idx):\n",
- " # Receives window predictions [B, H, output]\n",
- " # Broadcasts outputs and inverts normalization\n",
- "\n",
- " # Add C dimension\n",
- " if y_hat.ndim == 2:\n",
- " remove_dimension = True\n",
- " y_hat = y_hat.unsqueeze(-1)\n",
- " else:\n",
- " remove_dimension = False\n",
- "\n",
- " y_scale = self.scaler.x_scale[:, :, [y_idx]]\n",
- " y_loc = self.scaler.x_shift[:, :, [y_idx]]\n",
- "\n",
- " y_scale = torch.repeat_interleave(y_scale, repeats=y_hat.shape[-1], dim=-1).to(y_hat.device)\n",
- " y_loc = torch.repeat_interleave(y_loc, repeats=y_hat.shape[-1], dim=-1).to(y_hat.device)\n",
- "\n",
- " y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)\n",
- " y_loc = y_loc.to(y_hat.device)\n",
- " y_scale = y_scale.to(y_hat.device)\n",
- " \n",
- " if remove_dimension:\n",
- " y_hat = y_hat.squeeze(-1)\n",
- " y_loc = y_loc.squeeze(-1)\n",
- " y_scale = y_scale.squeeze(-1)\n",
- "\n",
- " return y_hat, y_loc, y_scale\n",
- "\n",
- " def _parse_windows(self, batch, windows):\n",
- " # Filter insample lags from outsample horizon\n",
- " y_idx = batch['y_idx']\n",
- " mask_idx = batch['temporal_cols'].get_loc('available_mask')\n",
- "\n",
- " insample_y = windows['temporal'][:, :self.input_size, y_idx]\n",
- " insample_mask = windows['temporal'][:, :self.input_size, mask_idx]\n",
- "\n",
- " # Declare additional information\n",
- " outsample_y = None\n",
- " outsample_mask = None\n",
- " hist_exog = None\n",
- " futr_exog = None\n",
- " stat_exog = None\n",
- "\n",
- " if self.h > 0:\n",
- " outsample_y = windows['temporal'][:, self.input_size:, y_idx]\n",
- " outsample_mask = windows['temporal'][:, self.input_size:, mask_idx]\n",
- "\n",
- " if len(self.hist_exog_list):\n",
- " hist_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.hist_exog_list)\n",
- " hist_exog = windows['temporal'][:, :self.input_size, hist_exog_idx]\n",
- "\n",
- " if len(self.futr_exog_list):\n",
- " futr_exog_idx = get_indexer_raise_missing(windows['temporal_cols'], self.futr_exog_list)\n",
- " futr_exog = windows['temporal'][:, :, futr_exog_idx]\n",
- "\n",
- " if len(self.stat_exog_list):\n",
- " static_idx = get_indexer_raise_missing(windows['static_cols'], self.stat_exog_list)\n",
- " stat_exog = windows['static'][:, static_idx]\n",
- "\n",
- " # TODO: think a better way of removing insample_y features\n",
- " if self.exclude_insample_y:\n",
- " insample_y = insample_y * 0\n",
- "\n",
- " return insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog\n",
- "\n",
- " def training_step(self, batch, batch_idx):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " windows = self._create_windows(batch, step='train')\n",
- " y_idx = batch['y_idx']\n",
- " original_outsample_y = torch.clone(windows['temporal'][:,-self.h:,y_idx])\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L]\n",
- " insample_mask=insample_mask, # [Ws, L]\n",
- " futr_exog=futr_exog, # [Ws, L + h, F]\n",
- " hist_exog=hist_exog, # [Ws, L, X]\n",
- " stat_exog=stat_exog) # [Ws, S]\n",
- "\n",
- " # Model Predictions\n",
- " output = self(windows_batch)\n",
- " if self.loss.is_distribution_output:\n",
- " _, y_loc, y_scale = self._inv_normalization(y_hat=outsample_y,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " outsample_y = original_outsample_y\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
- " else:\n",
- " loss = self.loss(y=outsample_y, y_hat=output, mask=outsample_mask)\n",
- "\n",
- " if torch.isnan(loss):\n",
- " print('Model Parameters', self.hparams)\n",
- " print('insample_y', torch.isnan(insample_y).sum())\n",
- " print('outsample_y', torch.isnan(outsample_y).sum())\n",
- " print('output', torch.isnan(output).sum())\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'train_loss',\n",
- " loss.detach().item(),\n",
- " batch_size=outsample_y.size(0),\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.train_trajectories.append((self.global_step, loss.detach().item()))\n",
- " return loss\n",
- "\n",
- " def _compute_valid_loss(self, outsample_y, output, outsample_mask, temporal_cols, y_idx):\n",
- " if self.loss.is_distribution_output:\n",
- " _, y_loc, y_scale = self._inv_normalization(y_hat=outsample_y,\n",
- " temporal_cols=temporal_cols,\n",
- " y_idx=y_idx)\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " _, sample_mean, quants = self.loss.sample(distr_args=distr_args)\n",
- "\n",
- " if str(type(self.valid_loss)) in\\\n",
- " [\"\", \"\"]:\n",
- " output = quants\n",
- " elif str(type(self.valid_loss)) in [\"\"]:\n",
- " output = torch.unsqueeze(sample_mean, dim=-1) # [N,H,1] -> [N,H]\n",
- "\n",
- " # Validation Loss evaluation\n",
- " if self.valid_loss.is_distribution_output:\n",
- " valid_loss = self.valid_loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)\n",
- " else:\n",
- " output, _, _ = self._inv_normalization(y_hat=output,\n",
- " temporal_cols=temporal_cols,\n",
- " y_idx=y_idx)\n",
- " valid_loss = self.valid_loss(y=outsample_y, y_hat=output, mask=outsample_mask)\n",
- " return valid_loss\n",
- " \n",
- " def validation_step(self, batch, batch_idx):\n",
- " if self.val_size == 0:\n",
- " return np.nan\n",
- "\n",
- " # TODO: Hack to compute number of windows\n",
- " windows = self._create_windows(batch, step='val')\n",
- " n_windows = len(windows['temporal'])\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Number of windows in batch\n",
- " windows_batch_size = self.inference_windows_batch_size\n",
- " if windows_batch_size < 0:\n",
- " windows_batch_size = n_windows\n",
- " n_batches = int(np.ceil(n_windows/windows_batch_size))\n",
- "\n",
- " valid_losses = []\n",
- " batch_sizes = []\n",
- " for i in range(n_batches):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " w_idxs = np.arange(i*windows_batch_size, \n",
- " min((i+1)*windows_batch_size, n_windows))\n",
- " windows = self._create_windows(batch, step='val', w_idxs=w_idxs)\n",
- " original_outsample_y = torch.clone(windows['temporal'][:,-self.h:,y_idx])\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L]\n",
- " insample_mask=insample_mask, # [Ws, L]\n",
- " futr_exog=futr_exog, # [Ws, L + h, F]\n",
- " hist_exog=hist_exog, # [Ws, L, X]\n",
- " stat_exog=stat_exog) # [Ws, S]\n",
- " \n",
- " # Model Predictions\n",
- " output_batch = self(windows_batch)\n",
- " valid_loss_batch = self._compute_valid_loss(outsample_y=original_outsample_y,\n",
- " output=output_batch, outsample_mask=outsample_mask,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=batch['y_idx'])\n",
- " valid_losses.append(valid_loss_batch)\n",
- " batch_sizes.append(len(output_batch))\n",
- " \n",
- " valid_loss = torch.stack(valid_losses)\n",
- " batch_sizes = torch.tensor(batch_sizes, device=valid_loss.device)\n",
- " batch_size = torch.sum(batch_sizes)\n",
- " valid_loss = torch.sum(valid_loss * batch_sizes) / batch_size\n",
- "\n",
- " if torch.isnan(valid_loss):\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'valid_loss',\n",
- " valid_loss.detach().item(),\n",
- " batch_size=batch_size,\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.validation_step_outputs.append(valid_loss)\n",
- " return valid_loss\n",
- "\n",
- " def predict_step(self, batch, batch_idx):\n",
- "\n",
- " # TODO: Hack to compute number of windows\n",
- " windows = self._create_windows(batch, step='predict')\n",
- " n_windows = len(windows['temporal'])\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Number of windows in batch\n",
- " windows_batch_size = self.inference_windows_batch_size\n",
- " if windows_batch_size < 0:\n",
- " windows_batch_size = n_windows\n",
- " n_batches = int(np.ceil(n_windows/windows_batch_size))\n",
- "\n",
- " y_hats = []\n",
- " for i in range(n_batches):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " w_idxs = np.arange(i*windows_batch_size, \n",
- " min((i+1)*windows_batch_size, n_windows))\n",
- " windows = self._create_windows(batch, step='predict', w_idxs=w_idxs)\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, _, \\\n",
- " hist_exog, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L]\n",
- " insample_mask=insample_mask, # [Ws, L]\n",
- " futr_exog=futr_exog, # [Ws, L + h, F]\n",
- " hist_exog=hist_exog, # [Ws, L, X]\n",
- " stat_exog=stat_exog) # [Ws, S] \n",
- "\n",
- " # Model Predictions\n",
- " output_batch = self(windows_batch)\n",
- " # Inverse normalization and sampling\n",
- " if self.loss.is_distribution_output:\n",
- " _, y_loc, y_scale = self._inv_normalization(y_hat=torch.empty(size=(insample_y.shape[0], self.h),\n",
- " dtype=output_batch[0].dtype,\n",
- " device=output_batch[0].device),\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " distr_args = self.loss.scale_decouple(output=output_batch, loc=y_loc, scale=y_scale)\n",
- " _, sample_mean, quants = self.loss.sample(distr_args=distr_args)\n",
- " y_hat = torch.concat((sample_mean, quants), axis=2)\n",
- "\n",
- " if self.loss.return_params:\n",
- " distr_args = torch.stack(distr_args, dim=-1)\n",
- " distr_args = torch.reshape(distr_args, (len(windows[\"temporal\"]), self.h, -1))\n",
- " y_hat = torch.concat((y_hat, distr_args), axis=2)\n",
- " else:\n",
- " y_hat, _, _ = self._inv_normalization(y_hat=output_batch,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " y_hats.append(y_hat)\n",
- " y_hat = torch.cat(y_hats, dim=0)\n",
- " return y_hat\n",
- " \n",
- " def fit(self, dataset, val_size=0, test_size=0, random_seed=None, distributed_config=None):\n",
- " \"\"\" Fit.\n",
- "\n",
- " The `fit` method, optimizes the neural network's weights using the\n",
- " initialization parameters (`learning_rate`, `windows_batch_size`, ...)\n",
- " and the `loss` function as defined during the initialization. \n",
- " Within `fit` we use a PyTorch Lightning `Trainer` that\n",
- " inherits the initialization's `self.trainer_kwargs`, to customize\n",
- " its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).\n",
- "\n",
- " The method is designed to be compatible with SKLearn-like classes\n",
- " and in particular to be compatible with the StatsForecast library.\n",
- "\n",
- " By default the `model` is not saving training checkpoints to protect \n",
- " disk memory, to get them change `enable_checkpointing=True` in `__init__`.\n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `val_size`: int, validation size for temporal cross-validation.
\n",
- " `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
- " `test_size`: int, test size for temporal cross-validation.
\n",
- " \"\"\"\n",
- " return self._fit(\n",
- " dataset=dataset,\n",
- " batch_size=self.batch_size,\n",
- " valid_batch_size=self.valid_batch_size,\n",
- " val_size=val_size,\n",
- " test_size=test_size,\n",
- " random_seed=random_seed,\n",
- " distributed_config=distributed_config,\n",
- " )\n",
- "\n",
- " def predict(self, dataset, test_size=None, step_size=1,\n",
- " random_seed=None, **data_module_kwargs):\n",
- " \"\"\" Predict.\n",
- "\n",
- " Neural network prediction with PL's `Trainer` execution of `predict_step`.\n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `test_size`: int=None, test size for temporal cross-validation.
\n",
- " `step_size`: int=1, Step size between each window.
\n",
- " `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
- " `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).\n",
- " \"\"\"\n",
- " self._check_exog(dataset)\n",
- " self._restart_seed(random_seed)\n",
- " data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)\n",
- "\n",
- " self.predict_step_size = step_size\n",
- " self.decompose_forecast = False\n",
- " datamodule = TimeSeriesDataModule(dataset=dataset,\n",
- " valid_batch_size=self.valid_batch_size,\n",
- " **data_module_kwargs)\n",
- "\n",
- " # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.\n",
- " pred_trainer_kwargs = self.trainer_kwargs.copy()\n",
- " if (pred_trainer_kwargs.get('accelerator', None) == \"gpu\") and (torch.cuda.device_count() > 1):\n",
- " pred_trainer_kwargs['devices'] = [0]\n",
- "\n",
- " trainer = pl.Trainer(**pred_trainer_kwargs)\n",
- " fcsts = trainer.predict(self, datamodule=datamodule) \n",
- " fcsts = torch.vstack(fcsts).numpy().flatten()\n",
- " fcsts = fcsts.reshape(-1, len(self.loss.output_names))\n",
- " return fcsts\n",
- "\n",
- " def decompose(self, dataset, step_size=1, random_seed=None, **data_module_kwargs):\n",
- " \"\"\" Decompose Predictions.\n",
- "\n",
- " Decompose the predictions through the network's layers.\n",
- " Available methods are `ESRNN`, `NHITS`, `NBEATS`, and `NBEATSx`.\n",
- "\n",
- " **Parameters:**
\n",
- " `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation here](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
- " `step_size`: int=1, step size between each window of temporal data.
\n",
- " `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).\n",
- " \"\"\"\n",
- " # Restart random seed\n",
- " if random_seed is None:\n",
- " random_seed = self.random_seed\n",
- " torch.manual_seed(random_seed)\n",
- " data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)\n",
- "\n",
- " self.predict_step_size = step_size\n",
- " self.decompose_forecast = True\n",
- " datamodule = TimeSeriesDataModule(dataset=dataset,\n",
- " valid_batch_size=self.valid_batch_size,\n",
- " **data_module_kwargs)\n",
- " trainer = pl.Trainer(**self.trainer_kwargs)\n",
- " fcsts = trainer.predict(self, datamodule=datamodule)\n",
- " self.decompose_forecast = False # Default decomposition back to false\n",
- " return torch.vstack(fcsts).numpy()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1712ea15",
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseWindows, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "48063f70",
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseWindows.fit, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "75529be6",
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseWindows.predict, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a1f8315d",
- "metadata": {},
- "outputs": [],
- "source": [
- "show_doc(BaseWindows.decompose, title_level=3)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8927f2e5-f376-4c99-bb8f-8cbb73efe01e",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.utils import AirPassengersDF\n",
- "from neuralforecast.tsdataset import TimeSeriesDataset, TimeSeriesDataModule"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "61490e69-f014-4087-83c5-540d5bd7d458",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# add h=0,1 unit test for _parse_windows \n",
- "# Declare batch\n",
- "AirPassengersDF['x'] = np.array(len(AirPassengersDF))\n",
- "AirPassengersDF['x2'] = np.array(len(AirPassengersDF)) * 2\n",
- "dataset, indices, dates, ds = TimeSeriesDataset.from_df(df=AirPassengersDF)\n",
- "data = TimeSeriesDataModule(dataset=dataset, batch_size=1, drop_last=True)\n",
- "\n",
- "train_loader = data.train_dataloader()\n",
- "batch = next(iter(train_loader))\n",
- "\n",
- "# Instantiate BaseWindows to test _parse_windows method h in [0,1]\n",
- "for h in [0, 1]:\n",
- " basewindows = BaseWindows(h=h,\n",
- " input_size=len(AirPassengersDF)-h,\n",
- " hist_exog_list=['x'],\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " learning_rate=0.001,\n",
- " max_steps=1,\n",
- " val_check_steps=0,\n",
- " batch_size=1,\n",
- " valid_batch_size=1,\n",
- " windows_batch_size=1,\n",
- " inference_windows_batch_size=1,\n",
- " start_padding_enabled=False)\n",
- "\n",
- " windows = basewindows._create_windows(batch, step='train')\n",
- " original_outsample_y = torch.clone(windows['temporal'][:,-basewindows.h:,0])\n",
- " windows = basewindows._normalization(windows=windows, y_idx=0)\n",
- "\n",
- " insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = basewindows._parse_windows(batch, windows)\n",
- "\n",
- " # Check equality of parsed and original insample_y\n",
- " parsed_insample_y = insample_y.numpy().flatten()\n",
- " original_insample_y = AirPassengersDF.y.values\n",
- " test_eq(parsed_insample_y, original_insample_y[:basewindows.input_size])\n",
- "\n",
- " # Check equality of parsed and original hist_exog\n",
- " parsed_hist_exog = hist_exog.numpy().flatten()\n",
- " original_hist_exog = AirPassengersDF.x.values\n",
- " test_eq(parsed_hist_exog, original_hist_exog[:basewindows.input_size])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "86ab58a9",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# Test that start_padding_enabled=True solves the problem of short series\n",
- "h = 12\n",
- "basewindows = BaseWindows(h=h,\n",
- " input_size=500,\n",
- " hist_exog_list=['x'],\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " learning_rate=0.001,\n",
- " max_steps=1,\n",
- " val_check_steps=0,\n",
- " batch_size=1,\n",
- " valid_batch_size=1,\n",
- " windows_batch_size=10,\n",
- " inference_windows_batch_size=2,\n",
- " start_padding_enabled=True)\n",
- "\n",
- "windows = basewindows._create_windows(batch, step='train')\n",
- "windows = basewindows._normalization(windows=windows, y_idx=0)\n",
- "insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = basewindows._parse_windows(batch, windows)\n",
- "\n",
- "basewindows.val_size = 12\n",
- "windows = basewindows._create_windows(batch, step='val')\n",
- "windows = basewindows._normalization(windows=windows, y_idx=0)\n",
- "insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = basewindows._parse_windows(batch, windows)\n",
- "\n",
- "basewindows.test_size = 12\n",
- "basewindows.predict_step_size = 1\n",
- "windows = basewindows._create_windows(batch, step='predict')\n",
- "windows = basewindows._normalization(windows=windows, y_idx=0)\n",
- "insample_y, insample_mask, outsample_y, outsample_mask, \\\n",
- " hist_exog, futr_exog, stat_exog = basewindows._parse_windows(batch, windows)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "54d2e850",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "\n",
- "# Test that hist_exog_list and futr_exog_list correctly filter data.\n",
- "# that is sent to scaler.\n",
- "basewindows = BaseWindows(h=12,\n",
- " input_size=500,\n",
- " hist_exog_list=['x', 'x2'],\n",
- " futr_exog_list=['x'],\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " learning_rate=0.001,\n",
- " max_steps=1,\n",
- " val_check_steps=0,\n",
- " batch_size=1,\n",
- " valid_batch_size=1,\n",
- " windows_batch_size=10,\n",
- " inference_windows_batch_size=2,\n",
- " start_padding_enabled=True)\n",
- "\n",
- "windows = basewindows._create_windows(batch, step='train')\n",
- "\n",
- "temporal_cols = windows['temporal_cols'].copy() # B, L+H, C\n",
- "temporal_data_cols = basewindows._get_temporal_exogenous_cols(temporal_cols=temporal_cols)\n",
- "\n",
- "test_eq(set(temporal_data_cols), set(['x', 'x2']))\n",
- "test_eq(windows['temporal'].shape, torch.Size([10,500+12,len(['y', 'x', 'x2', 'available_mask'])]))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "bf493ff9",
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "python3",
- "language": "python",
- "name": "python3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
diff --git a/nbs/common.model_checks.ipynb b/nbs/common.model_checks.ipynb
new file mode 100644
index 000000000..d618c5c33
--- /dev/null
+++ b/nbs/common.model_checks.ipynb
@@ -0,0 +1,248 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| default_exp common._model_checks"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 1. Checks for models"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This file provides a set of unit tests for all models"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "import pandas as pd\n",
+ "import neuralforecast.losses.pytorch as losses\n",
+ "\n",
+ "from neuralforecast import NeuralForecast\n",
+ "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic, generate_series"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "seed = 0\n",
+ "test_size = 14\n",
+ "FREQ = \"D\"\n",
+ "\n",
+ "# 1 series, no exogenous\n",
+ "N_SERIES_1 = 1\n",
+ "df = generate_series(n_series=N_SERIES_1, seed=seed, freq=FREQ, equal_ends=True)\n",
+ "max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)\n",
+ "Y_TRAIN_DF_1 = df[df.ds < max_ds]\n",
+ "Y_TEST_DF_1 = df[df.ds >= max_ds]\n",
+ "\n",
+ "# 5 series, no exogenous\n",
+ "N_SERIES_2 = 5\n",
+ "df = generate_series(n_series=N_SERIES_2, seed=seed, freq=FREQ, equal_ends=True)\n",
+ "max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)\n",
+ "Y_TRAIN_DF_2 = df[df.ds < max_ds]\n",
+ "Y_TEST_DF_2 = df[df.ds >= max_ds]\n",
+ "\n",
+ "# 1 series, with static and temporal exogenous\n",
+ "N_SERIES_3 = 1\n",
+ "df, STATIC_3 = generate_series(n_series=N_SERIES_3, n_static_features=2, \n",
+ " n_temporal_features=2, seed=seed, freq=FREQ, equal_ends=True)\n",
+ "max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)\n",
+ "Y_TRAIN_DF_3 = df[df.ds < max_ds]\n",
+ "Y_TEST_DF_3 = df[df.ds >= max_ds]\n",
+ "\n",
+ "# 5 series, with static and temporal exogenous\n",
+ "N_SERIES_4 = 5\n",
+ "df, STATIC_4 = generate_series(n_series=N_SERIES_4, n_static_features=2, \n",
+ " n_temporal_features=2, seed=seed, freq=FREQ, equal_ends=True)\n",
+ "max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)\n",
+ "Y_TRAIN_DF_4 = df[df.ds < max_ds]\n",
+ "Y_TEST_DF_4 = df[df.ds >= max_ds]\n",
+ "\n",
+ "# Generic test for a given config for a model\n",
+ "def _run_model_tests(model_class, config):\n",
+ " if model_class.RECURRENT:\n",
+ " config[\"inference_input_size\"] = config[\"input_size\"]\n",
+ "\n",
+ " # DF_1\n",
+ " if model_class.MULTIVARIATE:\n",
+ " config[\"n_series\"] = N_SERIES_1\n",
+ " if isinstance(config[\"loss\"], losses.relMSE):\n",
+ " config[\"loss\"].y_train = Y_TRAIN_DF_1[\"y\"].values \n",
+ " if isinstance(config[\"valid_loss\"], losses.relMSE):\n",
+ " config[\"valid_loss\"].y_train = Y_TRAIN_DF_1[\"y\"].values \n",
+ "\n",
+ " model = model_class(**config)\n",
+ " fcst = NeuralForecast(models=[model], freq=FREQ)\n",
+ " fcst.fit(df=Y_TRAIN_DF_1, val_size=24)\n",
+ " _ = fcst.predict(futr_df=Y_TEST_DF_1)\n",
+ " # DF_2\n",
+ " if model_class.MULTIVARIATE:\n",
+ " config[\"n_series\"] = N_SERIES_2\n",
+ " if isinstance(config[\"loss\"], losses.relMSE):\n",
+ " config[\"loss\"].y_train = Y_TRAIN_DF_2[\"y\"].values \n",
+ " if isinstance(config[\"valid_loss\"], losses.relMSE):\n",
+ " config[\"valid_loss\"].y_train = Y_TRAIN_DF_2[\"y\"].values\n",
+ " model = model_class(**config)\n",
+ " fcst = NeuralForecast(models=[model], freq=FREQ)\n",
+ " fcst.fit(df=Y_TRAIN_DF_2, val_size=24)\n",
+ " _ = fcst.predict(futr_df=Y_TEST_DF_2)\n",
+ "\n",
+ " if model.EXOGENOUS_STAT and model.EXOGENOUS_FUTR:\n",
+ " # DF_3\n",
+ " if model_class.MULTIVARIATE:\n",
+ " config[\"n_series\"] = N_SERIES_3\n",
+ " if isinstance(config[\"loss\"], losses.relMSE):\n",
+ " config[\"loss\"].y_train = Y_TRAIN_DF_3[\"y\"].values \n",
+ " if isinstance(config[\"valid_loss\"], losses.relMSE):\n",
+ " config[\"valid_loss\"].y_train = Y_TRAIN_DF_3[\"y\"].values\n",
+ " model = model_class(**config)\n",
+ " fcst = NeuralForecast(models=[model], freq=FREQ)\n",
+ " fcst.fit(df=Y_TRAIN_DF_3, static_df=STATIC_3, val_size=24)\n",
+ " _ = fcst.predict(futr_df=Y_TEST_DF_3)\n",
+ "\n",
+ " # DF_4\n",
+ " if model_class.MULTIVARIATE:\n",
+ " config[\"n_series\"] = N_SERIES_4\n",
+ " if isinstance(config[\"loss\"], losses.relMSE):\n",
+ " config[\"loss\"].y_train = Y_TRAIN_DF_4[\"y\"].values \n",
+ " if isinstance(config[\"valid_loss\"], losses.relMSE):\n",
+ " config[\"valid_loss\"].y_train = Y_TRAIN_DF_4[\"y\"].values \n",
+ " model = model_class(**config)\n",
+ " fcst = NeuralForecast(models=[model], freq=FREQ)\n",
+ " fcst.fit(df=Y_TRAIN_DF_4, static_df=STATIC_4, val_size=24)\n",
+ " _ = fcst.predict(futr_df=Y_TEST_DF_4) \n",
+ "\n",
+ "# Tests a model against every loss function\n",
+ "def check_loss_functions(model_class):\n",
+ " loss_list = [losses.MAE(), losses.MSE(), losses.RMSE(), losses.MAPE(), losses.SMAPE(), losses.MASE(seasonality=7), \n",
+ " losses.QuantileLoss(q=0.5), losses.MQLoss(), losses.IQLoss(), losses.DistributionLoss(\"Normal\"), \n",
+ " losses.DistributionLoss(\"StudentT\"), losses.DistributionLoss(\"Poisson\"), losses.DistributionLoss(\"NegativeBinomial\"), \n",
+ " losses.DistributionLoss(\"Tweedie\", rho=1.5), losses.DistributionLoss(\"ISQF\"), losses.PMM(), losses.PMM(weighted=True), \n",
+ " losses.GMM(), losses.GMM(weighted=True), losses.NBMM(), losses.NBMM(weighted=True), losses.HuberLoss(), \n",
+ " losses.TukeyLoss(), losses.HuberQLoss(q=0.5), losses.HuberMQLoss()]\n",
+ " for loss in loss_list:\n",
+ " test_name = f\"{model_class.__name__}: checking {loss._get_name()}\"\n",
+ " print(f\"{test_name}\")\n",
+ " config = {'max_steps': 2,\n",
+ " 'h': 7,\n",
+ " 'input_size': 28,\n",
+ " 'loss': loss,\n",
+ " 'valid_loss': None,\n",
+ " 'enable_progress_bar': False,\n",
+ " 'enable_model_summary': False,\n",
+ " 'val_check_steps': 2} \n",
+ " try:\n",
+ " _run_model_tests(model_class, config) \n",
+ " except RuntimeError:\n",
+ " raise Exception(f\"{test_name} failed.\")\n",
+ " except Exception:\n",
+ " print(f\"{test_name} skipped on raised Exception.\")\n",
+ " pass\n",
+ "\n",
+ "# Tests a model against the AirPassengers dataset\n",
+ "def check_airpassengers(model_class):\n",
+ " print(f\"{model_class.__name__}: checking forecast AirPassengers dataset\")\n",
+ " Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
+ "\n",
+ " config = {'max_steps': 2,\n",
+ " 'h': 12,\n",
+ " 'input_size': 24,\n",
+ " 'enable_progress_bar': False,\n",
+ " 'enable_model_summary': False,\n",
+ " 'val_check_steps': 2,\n",
+ " }\n",
+ "\n",
+ " if model_class.MULTIVARIATE:\n",
+ " config[\"n_series\"] = Y_train_df[\"unique_id\"].nunique()\n",
+ " # Normal forecast\n",
+ " fcst = NeuralForecast(models=[model_class(**config)], freq='M')\n",
+ " fcst.fit(df=Y_train_df, static_df=AirPassengersStatic)\n",
+ " _ = fcst.predict(futr_df=Y_test_df) \n",
+ "\n",
+ " # Cross-validation\n",
+ " fcst = NeuralForecast(models=[model_class(**config)], freq='M')\n",
+ " _ = fcst.cross_validation(df=AirPassengersPanel, static_df=AirPassengersStatic, n_windows=2, step_size=12)\n",
+ "\n",
+ "# Add unit test functions to this function\n",
+ "def check_model(model_class, checks=[\"losses\", \"airpassengers\"]):\n",
+ " \"\"\"\n",
+ " Check model with various tests. Options for checks are:
\n",
+ " \"losses\": test the model against all loss functions
\n",
+ " \"airpassengers\": test the model against the airpassengers dataset for forecasting and cross-validation
\n",
+ " \n",
+ " \"\"\"\n",
+ " if \"losses\" in checks:\n",
+ " check_loss_functions(model_class) \n",
+ " if \"airpassengers\" in checks:\n",
+ " try:\n",
+ " check_airpassengers(model_class) \n",
+ " except RuntimeError:\n",
+ " raise Exception(f\"{model_class.__name__}: AirPassengers forecast test failed.\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| eval: false\n",
+ "#| hide\n",
+ "# Run tests in this file. This is a slow test\n",
+ "import warnings\n",
+ "import logging\n",
+ "from neuralforecast.models import RNN, GRU, TCN, LSTM, DeepAR, DilatedRNN, BiTCN, MLP, NBEATS, NBEATSx, NHITS, DLinear, NLinear, TiDE, DeepNPTS, TFT, VanillaTransformer, Informer, Autoformer, FEDformer, TimesNet, iTransformer, KAN, RMoK, StemGNN, TSMixer, TSMixerx, MLPMultivariate, SOFTS, TimeMixer\n",
+ "\n",
+ "models = [RNN, GRU, TCN, LSTM, DeepAR, DilatedRNN, BiTCN, MLP, NBEATS, NBEATSx, NHITS, DLinear, NLinear, TiDE, DeepNPTS, TFT, VanillaTransformer, Informer, Autoformer, FEDformer, TimesNet, iTransformer, KAN, RMoK, StemGNN, TSMixer, TSMixerx, MLPMultivariate, SOFTS, TimeMixer]\n",
+ "\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " for model in models:\n",
+ " check_model(model, checks=[\"losses\"])"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "python3",
+ "language": "python",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/nbs/common.modules.ipynb b/nbs/common.modules.ipynb
index f90e936da..403a2a5d6 100644
--- a/nbs/common.modules.ipynb
+++ b/nbs/common.modules.ipynb
@@ -691,6 +691,66 @@
" x = x + self.mean\n",
" return x"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "class RevINMultivariate(nn.Module):\n",
+ " \"\"\" \n",
+ " ReversibleInstanceNorm1d for Multivariate models\n",
+ " \"\"\" \n",
+ " def __init__(self, num_features: int, eps=1e-5, affine=False, subtract_last=False, non_norm=False):\n",
+ " super().__init__()\n",
+ " self.num_features = num_features\n",
+ " self.eps = eps\n",
+ " self.affine = affine\n",
+ " if self.affine:\n",
+ " self._init_params()\n",
+ "\n",
+ " def forward(self, x, mode: str):\n",
+ " if mode == 'norm':\n",
+ " x = self._normalize(x)\n",
+ " elif mode == 'denorm':\n",
+ " x = self._denormalize(x)\n",
+ " else:\n",
+ " raise NotImplementedError\n",
+ " return x\n",
+ "\n",
+ " def _init_params(self):\n",
+ " # initialize RevIN params: (C,)\n",
+ " self.affine_weight = nn.Parameter(torch.ones((1, 1, self.num_features)))\n",
+ " self.affine_bias = nn.Parameter(torch.zeros((1, 1, self.num_features)))\n",
+ "\n",
+ " def _normalize(self, x):\n",
+ " # Batch statistics\n",
+ " self.batch_mean = torch.mean(x, axis=1, keepdim=True).detach()\n",
+ " self.batch_std = torch.sqrt(torch.var(x, axis=1, keepdim=True, unbiased=False) + self.eps).detach()\n",
+ " \n",
+ " # Instance normalization\n",
+ " x = x - self.batch_mean\n",
+ " x = x / self.batch_std\n",
+ " \n",
+ " if self.affine:\n",
+ " x = x * self.affine_weight\n",
+ " x = x + self.affine_bias\n",
+ "\n",
+ " return x\n",
+ "\n",
+ " def _denormalize(self, x):\n",
+ " # Reverse the normalization\n",
+ " if self.affine:\n",
+ " x = x - self.affine_bias\n",
+ " x = x / self.affine_weight \n",
+ " \n",
+ " x = x * self.batch_std\n",
+ " x = x + self.batch_mean \n",
+ "\n",
+ " return x"
+ ]
}
],
"metadata": {
diff --git a/nbs/common.scalers.ipynb b/nbs/common.scalers.ipynb
index 9e6737c3c..f49714a6b 100644
--- a/nbs/common.scalers.ipynb
+++ b/nbs/common.scalers.ipynb
@@ -682,11 +682,11 @@
" def _init_params(self, num_features):\n",
" # Initialize RevIN scaler params to broadcast:\n",
" if self.dim==1: # [B,T,C] [1,1,C]\n",
- " self.revin_bias = nn.Parameter(torch.zeros(1,1,num_features))\n",
- " self.revin_weight = nn.Parameter(torch.ones(1,1,num_features))\n",
+ " self.revin_bias = nn.Parameter(torch.zeros(1, 1, num_features, 1))\n",
+ " self.revin_weight = nn.Parameter(torch.ones(1, 1, num_features, 1))\n",
" elif self.dim==-1: # [B,C,T] [1,C,1]\n",
- " self.revin_bias = nn.Parameter(torch.zeros(1,num_features,1))\n",
- " self.revin_weight = nn.Parameter(torch.ones(1,num_features,1))\n",
+ " self.revin_bias = nn.Parameter(torch.zeros(1, num_features, 1, 1))\n",
+ " self.revin_weight = nn.Parameter(torch.ones(1, num_features, 1, 1))\n",
"\n",
" #@torch.no_grad()\n",
" def transform(self, x, mask):\n",
@@ -863,8 +863,8 @@
"#| hide\n",
"# Validate scalers\n",
"for scaler_type in [None, 'identity', 'standard', 'robust', 'minmax', 'minmax1', 'invariant', 'revin']:\n",
- " x = 1.0*torch.tensor(np_x)\n",
- " mask = torch.tensor(np_mask)\n",
+ " x = 1.0*torch.tensor(np_x).unsqueeze(-1)\n",
+ " mask = torch.tensor(np_mask).unsqueeze(-1)\n",
" scaler = TemporalNorm(scaler_type=scaler_type, dim=1, num_features=np_x.shape[-1])\n",
" x_scaled = scaler.transform(x=x, mask=mask)\n",
" x_recovered = scaler.inverse_transform(x_scaled)\n",
@@ -987,14 +987,6 @@
"nf = NeuralForecast(models=[model], freq='MS')\n",
"Y_hat_df = nf.cross_validation(df=Y_df, val_size=12, n_windows=1)"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b2f50bd8",
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/nbs/core.ipynb b/nbs/core.ipynb
index 3bb61dbdc..c44411f07 100644
--- a/nbs/core.ipynb
+++ b/nbs/core.ipynb
@@ -84,6 +84,7 @@
"\n",
"from neuralforecast.common._base_model import DistributedConfig\n",
"from neuralforecast.compat import SparkDataFrame\n",
+ "from neuralforecast.losses.pytorch import IQLoss\n",
"from neuralforecast.tsdataset import _FilesDataset, TimeSeriesDataset, LocalFilesTimeSeriesDataset\n",
"from neuralforecast.models import (\n",
" GRU, LSTM, RNN, TCN, DeepAR, DilatedRNN,\n",
@@ -96,7 +97,7 @@
" TimeMixer, KAN, RMoK\n",
")\n",
"from neuralforecast.common._base_auto import BaseAuto, MockTrial\n",
- "from neuralforecast.utils import PredictionIntervals, get_prediction_interval_method"
+ "from neuralforecast.utils import PredictionIntervals, get_prediction_interval_method, level_to_quantiles, quantiles_to_level"
]
},
{
@@ -337,6 +338,7 @@
" # Flags and attributes\n",
" self._fitted = False\n",
" self._reset_models()\n",
+ " self._add_level = False\n",
"\n",
" def _scalers_fit_transform(self, dataset: TimeSeriesDataset) -> None:\n",
" self.scalers_ = {} \n",
@@ -737,13 +739,14 @@
" names: List[str] = []\n",
" count_names = {'model': 0}\n",
" for model in self.models:\n",
- " if add_level and model.loss.outputsize_multiplier > 1:\n",
- " continue\n",
- "\n",
" model_name = repr(model)\n",
" count_names[model_name] = count_names.get(model_name, -1) + 1\n",
" if count_names[model_name] > 0:\n",
" model_name += str(count_names[model_name])\n",
+ "\n",
+ " if add_level and (model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss)):\n",
+ " continue\n",
+ "\n",
" names.extend(model_name + n for n in model.loss.output_names)\n",
" return names\n",
"\n",
@@ -863,6 +866,7 @@
" verbose: bool = False,\n",
" engine = None,\n",
" level: Optional[List[Union[int, float]]] = None,\n",
+ " quantiles: Optional[List[float]] = None,\n",
" **data_kwargs\n",
" ):\n",
" \"\"\"Predict with core.NeuralForecast.\n",
@@ -886,6 +890,8 @@
" Distributed engine for inference. Only used if df is a spark dataframe or if fit was called on a spark dataframe.\n",
" level : list of ints or floats, optional (default=None)\n",
" Confidence levels between 0 and 100.\n",
+ " quantiles : list of floats, optional (default=None)\n",
+ " Alternative to level, target quantiles to predict.\n",
" data_kwargs : kwargs\n",
" Extra arguments to be passed to the dataset within each model.\n",
"\n",
@@ -900,6 +906,22 @@
"\n",
" if not self._fitted:\n",
" raise Exception(\"You must fit the model before predicting.\")\n",
+ " \n",
+ " quantiles_ = None\n",
+ " level_ = None\n",
+ " has_level = False \n",
+ " if level is not None:\n",
+ " has_level = True\n",
+ " if quantiles is not None:\n",
+ " raise ValueError(\"You can't set both level and quantiles.\")\n",
+ " level_ = sorted(list(set(level)))\n",
+ " quantiles_ = level_to_quantiles(level_)\n",
+ " \n",
+ " if quantiles is not None:\n",
+ " if level is not None:\n",
+ " raise ValueError(\"You can't set both level and quantiles.\") \n",
+ " quantiles_ = sorted(list(set(quantiles)))\n",
+ " level_ = quantiles_to_level(quantiles_)\n",
"\n",
" needed_futr_exog = self._get_needed_futr_exog()\n",
" if needed_futr_exog:\n",
@@ -949,8 +971,6 @@
" if verbose: print('Using stored dataset.')\n",
" \n",
"\n",
- " cols = self._get_model_names()\n",
- "\n",
" # Placeholder dataframe for predictions with unique_id and ds\n",
" fcsts_df = ufp.make_future_dataframe(\n",
" uids=uids,\n",
@@ -994,24 +1014,14 @@
" )\n",
" self._scalers_transform(futr_dataset)\n",
" dataset = dataset.append(futr_dataset)\n",
- "\n",
- " col_idx = 0\n",
- " fcsts = np.full((self.h * len(uids), len(cols)), fill_value=np.nan, dtype=np.float32)\n",
- " for model in self.models:\n",
- " old_test_size = model.get_test_size()\n",
- " model.set_test_size(self.h) # To predict h steps ahead\n",
- " model_fcsts = model.predict(dataset=dataset, **data_kwargs)\n",
- " # Append predictions in memory placeholder\n",
- " output_length = len(model.loss.output_names)\n",
- " fcsts[:, col_idx : col_idx + output_length] = model_fcsts\n",
- " col_idx += output_length\n",
- " model.set_test_size(old_test_size) # Set back to original value\n",
+ " \n",
+ " fcsts, cols = self._generate_forecasts(dataset=dataset, uids=uids, quantiles_=quantiles_, level_=level_, has_level=has_level, **data_kwargs)\n",
+ " \n",
" if self.scalers_:\n",
" indptr = np.append(0, np.full(len(uids), self.h).cumsum())\n",
" fcsts = self._scalers_target_inverse_transform(fcsts, indptr)\n",
"\n",
" # Declare predictions pd.DataFrame\n",
- " cols = self._get_model_names() # Needed for IQLoss as column names may have changed during the call to .predict()\n",
" if isinstance(fcsts_df, pl_DataFrame):\n",
" fcsts = pl_DataFrame(dict(zip(cols, fcsts.T)))\n",
" else:\n",
@@ -1021,25 +1031,6 @@
" _warn_id_as_idx()\n",
" fcsts_df = fcsts_df.set_index(self.id_col)\n",
"\n",
- " # add prediction intervals\n",
- " if level is not None:\n",
- " if self._cs_df is None or self.prediction_intervals is None:\n",
- " raise Exception('You must fit the model with prediction_intervals to use level.')\n",
- " else:\n",
- " level_ = sorted(level)\n",
- " model_names = self._get_model_names(add_level=True)\n",
- " prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method)\n",
- "\n",
- " fcsts_df = prediction_interval_method(\n",
- " fcsts_df,\n",
- " self._cs_df,\n",
- " model_names=list(model_names),\n",
- " level=level_,\n",
- " cs_n_windows=self.prediction_intervals.n_windows,\n",
- " n_series=len(uids),\n",
- " horizon=self.h,\n",
- " )\n",
- "\n",
" return fcsts_df\n",
"\n",
" def _reset_models(self):\n",
@@ -1085,15 +1076,6 @@
" if self.dataset.min_size < (val_size+test_size):\n",
" warnings.warn('Validation and test sets are larger than the shorter time-series.')\n",
"\n",
- " cols = []\n",
- " count_names = {'model': 0}\n",
- " for model in self.models:\n",
- " model_name = repr(model)\n",
- " count_names[model_name] = count_names.get(model_name, -1) + 1\n",
- " if count_names[model_name] > 0:\n",
- " model_name += str(count_names[model_name])\n",
- " cols += [model_name + n for n in model.loss.output_names]\n",
- "\n",
" fcsts_df = ufp.cv_times(\n",
" times=self.ds,\n",
" uids=self.uids,\n",
@@ -1107,20 +1089,20 @@
" # the cv_times is sorted by window and then id\n",
" fcsts_df = ufp.sort(fcsts_df, [id_col, 'cutoff', time_col])\n",
"\n",
- " col_idx = 0\n",
- " fcsts = np.full((self.dataset.n_groups * self.h * n_windows, len(cols)),\n",
- " np.nan, dtype=np.float32)\n",
- " \n",
+ " fcsts_list: List = []\n",
" for model in self.models:\n",
+ " if self._add_level and (model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss)):\n",
+ " continue\n",
+ "\n",
" model.fit(dataset=self.dataset,\n",
" val_size=val_size, \n",
" test_size=test_size)\n",
" model_fcsts = model.predict(self.dataset, step_size=step_size, **data_kwargs)\n",
"\n",
" # Append predictions in memory placeholder\n",
- " output_length = len(model.loss.output_names)\n",
- " fcsts[:,col_idx:(col_idx + output_length)] = model_fcsts\n",
- " col_idx += output_length\n",
+ " fcsts_list.append(model_fcsts)\n",
+ "\n",
+ " fcsts = np.concatenate(fcsts_list, axis=-1)\n",
" # we may have allocated more space than needed\n",
" # each serie can produce at most (serie.size - 1) // self.h CV windows\n",
" effective_sizes = ufp.counts_by_id(fcsts_df, id_col)['counts'].to_numpy()\n",
@@ -1148,6 +1130,7 @@
" self._fitted = True\n",
"\n",
" # Add predictions to forecasts DataFrame\n",
+ " cols = self._get_model_names(add_level=self._add_level)\n",
" if isinstance(self.uids, pl_Series):\n",
" fcsts = pl_DataFrame(dict(zip(cols, fcsts.T)))\n",
" else:\n",
@@ -1164,7 +1147,7 @@
" if isinstance(fcsts_df, pd.DataFrame) and _id_as_idx():\n",
" _warn_id_as_idx()\n",
" fcsts_df = fcsts_df.set_index(id_col)\n",
- " return fcsts_df\n",
+ " return fcsts_df \n",
"\n",
" def cross_validation(\n",
" self,\n",
@@ -1183,6 +1166,7 @@
" target_col: str = 'y',\n",
" prediction_intervals: Optional[PredictionIntervals] = None,\n",
" level: Optional[List[Union[int, float]]] = None,\n",
+ " quantiles: Optional[List[float]] = None,\n",
" **data_kwargs\n",
" ) -> DataFrame:\n",
" \"\"\"Temporal Cross-Validation with core.NeuralForecast.\n",
@@ -1224,7 +1208,9 @@
" prediction_intervals : PredictionIntervals, optional (default=None)\n",
" Configuration to calibrate prediction intervals (Conformal Prediction). \n",
" level : list of ints or floats, optional (default=None)\n",
- " Confidence levels between 0 and 100. Use with prediction_intervals. \n",
+ " Confidence levels between 0 and 100.\n",
+ " quantiles : list of floats, optional (default=None)\n",
+ " Alternative to level, target quantiles to predict.\n",
" data_kwargs : kwargs\n",
" Extra arguments to be passed to the dataset within each model.\n",
"\n",
@@ -1257,15 +1243,15 @@
" df = df.reset_index(id_col) \n",
"\n",
" # Checks for prediction intervals\n",
- " if prediction_intervals is not None or level is not None:\n",
- " if level is None:\n",
- " warnings.warn('Level not provided, using level=[90].')\n",
- " level = [90]\n",
- " if prediction_intervals is None:\n",
- " raise Exception('You must set prediction_intervals to use level.')\n",
+ " if prediction_intervals is not None:\n",
+ " if level is None and quantiles is None:\n",
+ " raise Exception('When passing prediction_intervals you need to set the level or quantiles argument.') \n",
" if not refit:\n",
- " raise Exception('Passing prediction_intervals and/or level is only supported with refit=True.') \n",
+ " raise Exception('Passing prediction_intervals is only supported with refit=True.') \n",
"\n",
+ " if level is not None and quantiles is not None:\n",
+ " raise ValueError(\"You can't set both level and quantiles argument.\")\n",
+ " \n",
" if not refit:\n",
"\n",
" return self._no_refit_cross_validation(\n",
@@ -1326,6 +1312,7 @@
" sort_df=sort_df,\n",
" verbose=verbose,\n",
" level=level,\n",
+ " quantiles=quantiles,\n",
" **data_kwargs\n",
" )\n",
" preds = ufp.join(preds, cutoffs, on=id_col, how='left')\n",
@@ -1347,7 +1334,7 @@
" out = out.set_index(id_col)\n",
" return out\n",
"\n",
- " def predict_insample(self, step_size: int = 1):\n",
+ " def predict_insample(self, step_size: int = 1, **data_kwargs):\n",
" \"\"\"Predict insample with core.NeuralForecast.\n",
"\n",
" `core.NeuralForecast`'s `predict_insample` uses stored fitted `models`\n",
@@ -1365,23 +1352,7 @@
" \"\"\"\n",
" if not self._fitted:\n",
" raise Exception('The models must be fitted first with `fit` or `cross_validation`.')\n",
- "\n",
- " for model in self.models:\n",
- " if model.SAMPLING_TYPE == 'recurrent':\n",
- " warnings.warn(f'Predict insample might not provide accurate predictions for \\\n",
- " recurrent model {repr(model)} class yet due to scaling.')\n",
- " print(f'WARNING: Predict insample might not provide accurate predictions for \\\n",
- " recurrent model {repr(model)} class yet due to scaling.')\n",
" \n",
- " cols = []\n",
- " count_names = {'model': 0}\n",
- " for model in self.models:\n",
- " model_name = repr(model)\n",
- " count_names[model_name] = count_names.get(model_name, -1) + 1\n",
- " if count_names[model_name] > 0:\n",
- " model_name += str(count_names[model_name])\n",
- " cols += [model_name + n for n in model.loss.output_names]\n",
- "\n",
" # Remove test set from dataset and last dates\n",
" test_size = self.models[0].get_test_size()\n",
"\n",
@@ -1417,9 +1388,7 @@
" time_col=self.time_col,\n",
" )\n",
"\n",
- " col_idx = 0\n",
- " fcsts = np.full((len(fcsts_df), len(cols)), np.nan, dtype=np.float32)\n",
- "\n",
+ " fcsts_list: List = []\n",
" for model in self.models:\n",
" # Test size is the number of periods to forecast (full size of trimmed dataset)\n",
" model.set_test_size(test_size=trimmed_dataset.max_size)\n",
@@ -1427,10 +1396,9 @@
" # Predict\n",
" model_fcsts = model.predict(trimmed_dataset, step_size=step_size)\n",
" # Append predictions in memory placeholder\n",
- " output_length = len(model.loss.output_names)\n",
- " fcsts[:,col_idx:(col_idx + output_length)] = model_fcsts\n",
- " col_idx += output_length \n",
+ " fcsts_list.append(model_fcsts) \n",
" model.set_test_size(test_size=test_size) # Set original test_size\n",
+ " fcsts = np.concatenate(fcsts_list, axis=-1)\n",
"\n",
" # original y\n",
" original_y = {\n",
@@ -1440,6 +1408,7 @@
" }\n",
"\n",
" # Add predictions to forecasts DataFrame\n",
+ " cols = self._get_model_names()\n",
" if isinstance(self.uids, pl_Series):\n",
" fcsts = pl_DataFrame(dict(zip(cols, fcsts.T)))\n",
" Y_df = pl_DataFrame(original_y)\n",
@@ -1703,6 +1672,7 @@
" \"Please reduce the number of windows, horizon or remove those series.\"\n",
" )\n",
" \n",
+ " self._add_level = True\n",
" cv_results = self.cross_validation(\n",
" df=df,\n",
" static_df=static_df,\n",
@@ -1711,7 +1681,8 @@
" time_col=time_col,\n",
" target_col=target_col,\n",
" )\n",
- " \n",
+ " self._add_level = False\n",
+ "\n",
" kept = [time_col, id_col, 'cutoff']\n",
" # conformity score for each model\n",
" for model in self._get_model_names(add_level=True):\n",
@@ -1721,7 +1692,102 @@
" abs_err = abs(cv_results[model] - cv_results[target_col])\n",
" cv_results = ufp.assign_columns(cv_results, model, abs_err)\n",
" dropped = list(set(cv_results.columns) - set(kept))\n",
- " return ufp.drop_columns(cv_results, dropped) "
+ " return ufp.drop_columns(cv_results, dropped) \n",
+ " \n",
+ " def _generate_forecasts(self, dataset: TimeSeriesDataset, uids: Series, quantiles_: Optional[List[float]] = None, level_: Optional[List[Union[int, float]]] = None, has_level: Optional[bool] = False, **data_kwargs) -> np.array:\n",
+ " fcsts_list: List = []\n",
+ " cols = []\n",
+ " count_names = {'model': 0}\n",
+ " for model in self.models:\n",
+ " old_test_size = model.get_test_size()\n",
+ " model.set_test_size(self.h) # To predict h steps ahead\n",
+ " \n",
+ " # Increment model name if the same model is used more than once\n",
+ " model_name = repr(model)\n",
+ " count_names[model_name] = count_names.get(model_name, -1) + 1\n",
+ " if count_names[model_name] > 0:\n",
+ " model_name += str(count_names[model_name])\n",
+ "\n",
+ " # Predict for every quantile or level if requested and the loss function supports it\n",
+ " # case 1: DistributionLoss and MixtureLosses\n",
+ " if quantiles_ is not None and not isinstance(model.loss, IQLoss) and hasattr(model.loss, 'update_quantile') and callable(model.loss.update_quantile):\n",
+ " model_fcsts = model.predict(dataset=dataset, quantiles = quantiles_, **data_kwargs)\n",
+ " fcsts_list.append(model_fcsts) \n",
+ " col_names = []\n",
+ " for i, quantile in enumerate(quantiles_):\n",
+ " col_name = self._get_column_name(model_name, quantile, has_level)\n",
+ " if i == 0:\n",
+ " col_names.extend([f\"{model_name}\", col_name])\n",
+ " else:\n",
+ " col_names.extend([col_name])\n",
+ " if hasattr(model.loss, 'return_params') and model.loss.return_params:\n",
+ " cols.extend(col_names + [model_name + param_name for param_name in model.loss.param_names])\n",
+ " else:\n",
+ " cols.extend(col_names)\n",
+ " # case 2: IQLoss\n",
+ " elif quantiles_ is not None and isinstance(model.loss, IQLoss):\n",
+ " # IQLoss does not give monotonically increasing quantiles, so we apply a hack: compute all quantiles, and take the quantile over the quantiles\n",
+ " quantiles_iqloss = np.linspace(0.01, 0.99, 20)\n",
+ " fcsts_list_iqloss = []\n",
+ " for i, quantile in enumerate(quantiles_iqloss):\n",
+ " model_fcsts = model.predict(dataset=dataset, quantiles = [quantile], **data_kwargs) \n",
+ " fcsts_list_iqloss.append(model_fcsts) \n",
+ " fcsts_iqloss = np.concatenate(fcsts_list_iqloss, axis=-1)\n",
+ "\n",
+ " # Get the actual requested quantiles\n",
+ " model_fcsts = np.quantile(fcsts_iqloss, quantiles_, axis=-1).T\n",
+ " fcsts_list.append(model_fcsts) \n",
+ "\n",
+ " # Get the right column names\n",
+ " col_names = []\n",
+ " for i, quantile in enumerate(quantiles_):\n",
+ " col_name = self._get_column_name(model_name, quantile, has_level)\n",
+ " col_names.extend([col_name]) \n",
+ " cols.extend(col_names)\n",
+ " # case 3: PointLoss via prediction intervals\n",
+ " elif quantiles_ is not None and model.loss.outputsize_multiplier == 1:\n",
+ " if self.prediction_intervals is None:\n",
+ " raise AttributeError(\n",
+ " f\"You have trained {model_name} with loss={type(model.loss).__name__}(). \\n\"\n",
+ " \" You then must set `prediction_intervals` during fit to use level or quantiles during predict.\") \n",
+ " model_fcsts = model.predict(dataset=dataset, quantiles = quantiles_, **data_kwargs)\n",
+ " prediction_interval_method = get_prediction_interval_method(self.prediction_intervals.method)\n",
+ " fcsts_with_intervals, out_cols = prediction_interval_method(\n",
+ " model_fcsts,\n",
+ " self._cs_df,\n",
+ " model=model_name,\n",
+ " level=level_ if has_level else None,\n",
+ " cs_n_windows=self.prediction_intervals.n_windows,\n",
+ " n_series=len(uids),\n",
+ " horizon=self.h,\n",
+ " quantiles=quantiles_ if not has_level else None,\n",
+ " ) \n",
+ " fcsts_list.append(fcsts_with_intervals) \n",
+ " cols.extend([model_name] + out_cols)\n",
+ " # base case: quantiles or levels are not supported or provided as arguments\n",
+ " else:\n",
+ " model_fcsts = model.predict(dataset=dataset, **data_kwargs)\n",
+ " fcsts_list.append(model_fcsts)\n",
+ " cols.extend(model_name + n for n in model.loss.output_names)\n",
+ " model.set_test_size(old_test_size) # Set back to original value\n",
+ " fcsts = np.concatenate(fcsts_list, axis=-1)\n",
+ "\n",
+ " return fcsts, cols\n",
+ " \n",
+ " @staticmethod\n",
+ " def _get_column_name(model_name, quantile, has_level) -> str:\n",
+ " if not has_level:\n",
+ " col_name = f\"{model_name}_ql{quantile}\" \n",
+ " elif quantile < 0.5:\n",
+ " level_lo = int(round(100 - 200 * quantile))\n",
+ " col_name = f\"{model_name}-lo-{level_lo}\"\n",
+ " elif quantile > 0.5:\n",
+ " level_hi = int(round(100 - 200 * (1 - quantile)))\n",
+ " col_name = f\"{model_name}-hi-{level_hi}\"\n",
+ " else:\n",
+ " col_name = f\"{model_name}-median\"\n",
+ "\n",
+ " return col_name\n"
]
},
{
@@ -1849,7 +1915,7 @@
"from neuralforecast.models.tsmixer import TSMixer\n",
"from neuralforecast.models.tsmixerx import TSMixerx\n",
"\n",
- "from neuralforecast.losses.pytorch import MQLoss, MAE, MSE\n",
+ "from neuralforecast.losses.pytorch import MQLoss, MAE, MSE, DistributionLoss, IQLoss\n",
"from neuralforecast.utils import AirPassengersDF, AirPassengersPanel, AirPassengersStatic\n",
"\n",
"from datetime import date"
@@ -3465,6 +3531,71 @@
")\n",
"assert all([col in cv2.columns for col in ['NHITS-lo-30', 'NHITS-hi-30']])"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b82e7c70",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Test quantile and level argument in predict for different models and errors\n",
+ "prediction_intervals = PredictionIntervals(method=\"conformal_error\")\n",
+ "\n",
+ "models = []\n",
+ "for nf_model in [NHITS, LSTM, TSMixer]:\n",
+ " params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1, \"loss\": MAE()}\n",
+ " if nf_model.__name__ == \"TSMixer\":\n",
+ " params.update({\"n_series\": 2})\n",
+ " models.append(nf_model(**params))\n",
+ "\n",
+ " params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1, \"loss\": DistributionLoss(distribution=\"Normal\")}\n",
+ " if nf_model.__name__ == \"TSMixer\":\n",
+ " params.update({\"n_series\": 2})\n",
+ " models.append(nf_model(**params))\n",
+ "\n",
+ " params = {\"h\": 12, \"input_size\": 24, \"max_steps\": 1, \"loss\": IQLoss()}\n",
+ " if nf_model.__name__ == \"TSMixer\":\n",
+ " params.update({\"n_series\": 2})\n",
+ " models.append(nf_model(**params))\n",
+ "\n",
+ "nf = NeuralForecast(models=models, freq='M')\n",
+ "nf.fit(AirPassengersPanel_train, prediction_intervals=prediction_intervals)\n",
+ "# Test default prediction\n",
+ "preds = nf.predict(futr_df=AirPassengersPanel_test)\n",
+ "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS1', 'NHITS1-median', 'NHITS1-lo-90',\n",
+ " 'NHITS1-lo-80', 'NHITS1-hi-80', 'NHITS1-hi-90', 'NHITS2_ql0.5', 'LSTM',\n",
+ " 'LSTM1', 'LSTM1-median', 'LSTM1-lo-90', 'LSTM1-lo-80', 'LSTM1-hi-80',\n",
+ " 'LSTM1-hi-90', 'LSTM2_ql0.5', 'TSMixer', 'TSMixer1', 'TSMixer1-median',\n",
+ " 'TSMixer1-lo-90', 'TSMixer1-lo-80', 'TSMixer1-hi-80', 'TSMixer1-hi-90',\n",
+ " 'TSMixer2_ql0.5']\n",
+ "# Test quantile prediction\n",
+ "preds = nf.predict(futr_df=AirPassengersPanel_test, quantiles=[0.2, 0.3])\n",
+ "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS-ql0.2', 'NHITS-ql0.3', 'NHITS1',\n",
+ " 'NHITS1_ql0.2', 'NHITS1_ql0.3', 'NHITS2_ql0.2', 'NHITS2_ql0.3', 'LSTM',\n",
+ " 'LSTM-ql0.2', 'LSTM-ql0.3', 'LSTM1', 'LSTM1_ql0.2', 'LSTM1_ql0.3',\n",
+ " 'LSTM2_ql0.2', 'LSTM2_ql0.3', 'TSMixer', 'TSMixer-ql0.2',\n",
+ " 'TSMixer-ql0.3', 'TSMixer1', 'TSMixer1_ql0.2', 'TSMixer1_ql0.3',\n",
+ " 'TSMixer2_ql0.2', 'TSMixer2_ql0.3']\n",
+ "# Test level prediction\n",
+ "preds = nf.predict(futr_df=AirPassengersPanel_test, level=[80, 90])\n",
+ "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS-lo-90', 'NHITS-lo-80', 'NHITS-hi-80',\n",
+ " 'NHITS-hi-90', 'NHITS1', 'NHITS1-lo-90', 'NHITS1-lo-80', 'NHITS1-hi-80',\n",
+ " 'NHITS1-hi-90', 'NHITS2-lo-90', 'NHITS2-lo-80', 'NHITS2-hi-80',\n",
+ " 'NHITS2-hi-90', 'LSTM', 'LSTM-lo-90', 'LSTM-lo-80', 'LSTM-hi-80',\n",
+ " 'LSTM-hi-90', 'LSTM1', 'LSTM1-lo-90', 'LSTM1-lo-80', 'LSTM1-hi-80',\n",
+ " 'LSTM1-hi-90', 'LSTM2-lo-90', 'LSTM2-lo-80', 'LSTM2-hi-80',\n",
+ " 'LSTM2-hi-90', 'TSMixer', 'TSMixer-lo-90', 'TSMixer-lo-80',\n",
+ " 'TSMixer-hi-80', 'TSMixer-hi-90', 'TSMixer1', 'TSMixer1-lo-90',\n",
+ " 'TSMixer1-lo-80', 'TSMixer1-hi-80', 'TSMixer1-hi-90', 'TSMixer2-lo-90',\n",
+ " 'TSMixer2-lo-80', 'TSMixer2-hi-80', 'TSMixer2-hi-90']\n",
+ "# Re-Test default prediction - note that they are different from the first test (this is expected)\n",
+ "preds = nf.predict(futr_df=AirPassengersPanel_test)\n",
+ "assert list(preds.columns) == ['unique_id', 'ds', 'NHITS', 'NHITS1', 'NHITS1-median', 'NHITS2_ql0.5',\n",
+ " 'LSTM', 'LSTM1', 'LSTM1-median', 'LSTM2_ql0.5', 'TSMixer', 'TSMixer1',\n",
+ " 'TSMixer1-median', 'TSMixer2_ql0.5']"
+ ]
}
],
"metadata": {
diff --git a/nbs/docs/capabilities/01_overview.ipynb b/nbs/docs/capabilities/01_overview.ipynb
index 11b964a7f..de1f3e374 100644
--- a/nbs/docs/capabilities/01_overview.ipynb
+++ b/nbs/docs/capabilities/01_overview.ipynb
@@ -19,11 +19,11 @@
"|`BiTCN` | `AutoBiTCN` | CNN | Univariate | Direct | F/H/S | \n",
"|`DeepAR` | `AutoDeepAR` | RNN | Univariate | Recursive | F/S | \n",
"|`DeepNPTS` | `AutoDeepNPTS` | MLP | Univariate | Direct | F/H/S | \n",
- "|`DilatedRNN` | `AutoDilatedRNN` | RNN | Univariate | Recursive | F/H/S | \n",
+ "|`DilatedRNN` | `AutoDilatedRNN` | RNN | Univariate | Direct | F/H/S | \n",
"|`FEDformer` | `AutoFEDformer` | Transformer | Univariate | Direct | F | \n",
"|`GRU` | `AutoGRU` | RNN | Univariate | Recursive | F/H/S | \n",
"|`HINT` | `AutoHINT` | Any7 | Both7 | Both7 | F/H/S | \n",
- "|`Informer` | `AutoInformer` | Transformer | Multivariate | Direct | F | \n",
+ "|`Informer` | `AutoInformer` | Transformer | Univariate | Direct | F | \n",
"|`iTransformer` | `AutoiTransformer` | Transformer | Multivariate | Direct | - | \n",
"|`KAN` | `AutoKAN` | KAN | Univariate | Direct | F/H/S | \n",
"|`LSTM` | `AutoLSTM` | RNN | Univariate | Recursive | F/H/S | \n",
@@ -38,7 +38,7 @@
"|`RNN` | `AutoRNN` | RNN | Univariate | Recursive | F/H/S | \n",
"|`SOFTS` | `AutoSOFTS` | MLP | Multivariate | Direct | - | \n",
"|`StemGNN` | `AutoStemGNN` | GNN | Multivariate | Direct | - | \n",
- "|`TCN` | `AutoTCN` | CNN | Univariate | Recursive | F/H/S | \n",
+ "|`TCN` | `AutoTCN` | CNN | Univariate | Direct | F/H/S | \n",
"|`TFT` | `AutoTFT` | Transformer | Univariate | Direct | F/H/S | \n",
"|`TiDE` | `AutoTiDE` | MLP | Univariate | Direct | F/H/S | \n",
"|`TimeMixer` | `AutoTimeMixer` | MLP | Multivariate | Direct | - | \n",
diff --git a/nbs/losses.pytorch.ipynb b/nbs/losses.pytorch.ipynb
index d8d333dd7..70cceb571 100644
--- a/nbs/losses.pytorch.ipynb
+++ b/nbs/losses.pytorch.ipynb
@@ -54,9 +54,8 @@
"outputs": [],
"source": [
"#| export\n",
- "from typing import Optional, Union, Tuple\n",
+ "from typing import Optional, Union, Tuple, List\n",
"\n",
- "import math\n",
"import numpy as np\n",
"import torch\n",
"\n",
@@ -70,6 +69,9 @@
" Poisson,\n",
" NegativeBinomial,\n",
" Beta,\n",
+ " Gamma,\n",
+ " MixtureSameFamily,\n",
+ " Categorical,\n",
" AffineTransform, \n",
" TransformedDistribution,\n",
")\n",
@@ -140,7 +142,7 @@
" `outputsize_multiplier`: Multiplier for the output size.
\n",
" `output_names`: Names of the outputs.
\n",
" \"\"\"\n",
- " def __init__(self, horizon_weight, outputsize_multiplier, output_names):\n",
+ " def __init__(self, horizon_weight=None, outputsize_multiplier=None, output_names=None):\n",
" super(BasePointLoss, self).__init__()\n",
" if horizon_weight is not None:\n",
" horizon_weight = torch.Tensor(horizon_weight.flatten())\n",
@@ -151,10 +153,13 @@
"\n",
" def domain_map(self, y_hat: torch.Tensor):\n",
" \"\"\"\n",
- " Univariate loss operates in dimension [B,T,H]/[B,H]\n",
- " This changes the network's output from [B,H,1]->[B,H]\n",
+ " Input:\n",
+ " Univariate: [B, H, 1]\n",
+ " Multivariate: [B, H, N]\n",
+ "\n",
+ " Output: [B, H, N]\n",
" \"\"\"\n",
- " return y_hat.squeeze(-1)\n",
+ " return y_hat\n",
"\n",
" def _compute_weights(self, y, mask):\n",
" \"\"\"\n",
@@ -163,16 +168,17 @@
" If set, check that it has the same length as the horizon in x.\n",
" \"\"\"\n",
" if mask is None:\n",
- " mask = torch.ones_like(y, device=y.device)\n",
+ " mask = torch.ones_like(y)\n",
"\n",
" if self.horizon_weight is None:\n",
- " self.horizon_weight = torch.ones(mask.shape[-1])\n",
+ " weights = torch.ones_like(mask)\n",
" else:\n",
- " assert mask.shape[-1] == len(self.horizon_weight), \\\n",
+ " assert mask.shape[1] == len(self.horizon_weight), \\\n",
" 'horizon_weight must have same length as Y'\n",
- "\n",
- " weights = self.horizon_weight.clone()\n",
- " weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)\n",
+ " weights = self.horizon_weight.clone()\n",
+ " weights = weights[None, :, None].to(mask.device)\n",
+ " weights = torch.ones_like(mask, device=mask.device) * weights\n",
+ " \n",
" return weights * mask"
]
},
@@ -227,7 +233,8 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " y_insample: Union[torch.Tensor, None] = None) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -311,7 +318,9 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -398,7 +407,8 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " y_insample: Union[torch.Tensor, None] = None) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -498,7 +508,9 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -590,7 +602,8 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " y_insample: Union[torch.Tensor, None] = None) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -685,12 +698,13 @@
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
" y_insample: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor (batch_size, output_size), Actual values.
\n",
" `y_hat`: tensor (batch_size, output_size)), Predicted values.
\n",
- " `y_insample`: tensor (batch_size, input_size), Actual insample Seasonal Naive predictions.
\n",
+ " `y_insample`: tensor (batch_size, input_size), Actual insample values.
\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.
\n",
"\n",
" **Returns:**
\n",
@@ -699,7 +713,7 @@
" delta_y = torch.abs(y - y_hat)\n",
" scale = torch.mean(torch.abs(y_insample[:, self.seasonality:] - \\\n",
" y_insample[:, :-self.seasonality]), axis=1)\n",
- " losses = _divide_no_nan(delta_y, scale[:, None])\n",
+ " losses = _divide_no_nan(delta_y, scale[:, None, None])\n",
" weights = self._compute_weights(y=y, mask=mask)\n",
" return _weighted_mean(losses=losses, weights=weights)"
]
@@ -754,11 +768,11 @@
" \"\"\"Relative Mean Squared Error\n",
" Computes Relative Mean Squared Error (relMSE), as proposed by Hyndman & Koehler (2006)\n",
" as an alternative to percentage errors, to avoid measure unstability.\n",
- " $$ \\mathrm{relMSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}, \\\\mathbf{\\hat{y}}^{naive1}) =\n",
- " \\\\frac{\\mathrm{MSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}})}{\\mathrm{MSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}^{naive1})} $$\n",
+ " $$ \\mathrm{relMSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}, \\\\mathbf{\\hat{y}}^{benchmark}) =\n",
+ " \\\\frac{\\mathrm{MSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}})}{\\mathrm{MSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}^{benchmark})} $$\n",
"\n",
" **Parameters:**
\n",
- " `y_train`: numpy array, Training values.
\n",
+ " `y_train`: numpy array, deprecated.
\n",
" `horizon_weight`: Tensor of size h, weight for each timestamp of the forecasting window.
\n",
"\n",
" **References:**
\n",
@@ -769,32 +783,31 @@
" \"Probabilistic Hierarchical Forecasting with Deep Poisson Mixtures. \n",
" Submitted to the International Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)\n",
" \"\"\"\n",
- " def __init__(self, y_train, horizon_weight=None):\n",
+ " def __init__(self, y_train=None, horizon_weight=None):\n",
" super(relMSE, self).__init__(horizon_weight=horizon_weight,\n",
" outputsize_multiplier=1,\n",
" output_names=[''])\n",
- " self.y_train = y_train\n",
+ " if y_train is not None:\n",
+ " raise DeprecationWarning(\"y_train will be deprecated in a future release.\")\n",
" self.mse = MSE(horizon_weight=horizon_weight)\n",
"\n",
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_benchmark: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor (batch_size, output_size), Actual values.
\n",
" `y_hat`: tensor (batch_size, output_size)), Predicted values.
\n",
- " `y_insample`: tensor (batch_size, input_size), Actual insample Seasonal Naive predictions.
\n",
+ " `y_benchmark`: tensor (batch_size, output_size), Benchmark predicted values.
\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.
\n",
"\n",
" **Returns:**
\n",
" `relMSE`: tensor (single value).\n",
" \"\"\"\n",
- " horizon = y.shape[-1]\n",
- " last_col = self.y_train[:, -1].unsqueeze(1)\n",
- " y_naive = last_col.repeat(1, horizon)\n",
- "\n",
- " norm = self.mse(y=y, y_hat=y_naive, mask=mask) # Already weighted\n",
+ " norm = self.mse(y=y, y_hat=y_benchmark, mask=mask) # Already weighted\n",
" norm = norm + 1e-5 # Numerical stability\n",
" loss = self.mse(y=y, y_hat=y_hat, mask=mask) # Already weighted\n",
" loss = _divide_no_nan(loss, norm)\n",
@@ -880,7 +893,9 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -1022,35 +1037,47 @@
"\n",
" def domain_map(self, y_hat: torch.Tensor):\n",
" \"\"\"\n",
- " Identity domain map [B,T,H,Q]/[B,H,Q]\n",
+ " Input:\n",
+ " Univariate: [B, H, 1 * Q]\n",
+ " Multivariate: [B, H, N * Q]\n",
+ "\n",
+ " Output: [B, H, N, Q]\n",
" \"\"\"\n",
- " return y_hat\n",
- " \n",
+ " output = y_hat.reshape(y_hat.shape[0],\n",
+ " y_hat.shape[1],\n",
+ " -1,\n",
+ " self.outputsize_multiplier)\n",
+ "\n",
+ " return output\n",
+ "\n",
" def _compute_weights(self, y, mask):\n",
" \"\"\"\n",
" Compute final weights for each datapoint (based on all weights and all masks)\n",
" Set horizon_weight to a ones[H] tensor if not set.\n",
" If set, check that it has the same length as the horizon in x.\n",
+ "\n",
+ " y: [B, h, N, 1]\n",
+ " mask: [B, h, N, 1]\n",
" \"\"\"\n",
- " if mask is None:\n",
- " mask = torch.ones_like(y, device=y.device)\n",
- " else:\n",
- " mask = mask.unsqueeze(1) # Add Q dimension.\n",
"\n",
" if self.horizon_weight is None:\n",
- " self.horizon_weight = torch.ones(mask.shape[-1])\n",
+ " weights = torch.ones_like(mask)\n",
" else:\n",
- " assert mask.shape[-1] == len(self.horizon_weight), \\\n",
- " 'horizon_weight must have same length as Y'\n",
- " \n",
- " weights = self.horizon_weight.clone()\n",
- " weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)\n",
+ " assert mask.shape[1] == len(self.horizon_weight), \\\n",
+ " 'horizon_weight must have same length as Y' \n",
+ " weights = self.horizon_weight.clone()\n",
+ " weights = weights[None, :, None, None]\n",
+ " weights = weights.to(mask.device)\n",
+ " weights = torch.ones_like(mask, device=mask.device) * weights\n",
+ " \n",
" return weights * mask\n",
"\n",
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -1060,20 +1087,24 @@
" **Returns:**
\n",
" `mqloss`: tensor (single value).\n",
" \"\"\"\n",
- " \n",
- " error = y_hat - y.unsqueeze(-1)\n",
- " sq = torch.maximum(-error, torch.zeros_like(error))\n",
- " s1_q = torch.maximum(error, torch.zeros_like(error))\n",
- " losses = (1/len(self.quantiles))*(self.quantiles * sq + (1 - self.quantiles) * s1_q)\n",
+ " # [B, h, N] -> [B, h, N, 1]\n",
+ " if y_hat.ndim == 3:\n",
+ " y_hat = y_hat.unsqueeze(-1)\n",
+ "\n",
+ " y = y.unsqueeze(-1)\n",
+ " if mask is not None:\n",
+ " mask = mask.unsqueeze(-1)\n",
+ " else:\n",
+ " mask = torch.ones_like(y, device=y.device)\n",
"\n",
- " if y_hat.ndim == 3: # BaseWindows\n",
- " losses = losses.swapaxes(-2,-1) # [B,H,Q] -> [B,Q,H] (needed for horizon weighting, H at the end)\n",
- " elif y_hat.ndim == 4: # BaseRecurrent\n",
- " losses = losses.swapaxes(-2,-1)\n",
- " losses = losses.swapaxes(-2,-3) # [B,seq_len,H,Q] -> [B,Q,seq_len,H] (needed for horizon weighting, H at the end)\n",
+ " error = y_hat - y\n",
"\n",
+ " sq = torch.maximum(-error, torch.zeros_like(error))\n",
+ " s1_q = torch.maximum(error, torch.zeros_like(error))\n",
+ " \n",
+ " quantiles = self.quantiles[None, None, None, :]\n",
+ " losses = (1 / len(quantiles)) * (quantiles * sq + (1 - quantiles) * s1_q)\n",
" weights = self._compute_weights(y=losses, mask=mask) # Use losses for extra dim\n",
- " # NOTE: Weights do not have Q dimension.\n",
"\n",
" return _weighted_mean(losses=losses, weights=weights)"
]
@@ -1228,9 +1259,9 @@
" self.sampling_distr = Beta(concentration0 = concentration0,\n",
" concentration1 = concentration1)\n",
"\n",
- " def update_quantile(self, q: float = 0.5):\n",
- " self.q = q\n",
- " self.output_names = [f\"_ql{q}\"]\n",
+ " def update_quantile(self, q: List[float] = [0.5]):\n",
+ " self.q = q[0]\n",
+ " self.output_names = [f\"_ql{q[0]}\"]\n",
" self.has_predicted = True\n",
"\n",
" def domain_map(self, y_hat):\n",
@@ -1239,9 +1270,8 @@
"\n",
" Input shapes to this function:\n",
" \n",
- " base_windows: y_hat = [B, h, 1] \n",
- " base_multivariate: y_hat = [B, h, n_series]\n",
- " base_recurrent: y_hat = [B, seq_len, h, n_series]\n",
+ " Univariate: y_hat = [B, h, 1] \n",
+ " Multivariate: y_hat = [B, h, N]\n",
" \"\"\"\n",
" if self.eval() and self.has_predicted:\n",
" quantiles = torch.full(size=y_hat.shape, \n",
@@ -1259,7 +1289,7 @@
" emb_outputs = self.output_layer(emb_inputs)\n",
" \n",
" # Domain map\n",
- " y_hat = emb_outputs.squeeze(-1).squeeze(-1)\n",
+ " y_hat = emb_outputs.squeeze(-1)\n",
"\n",
" return y_hat\n"
]
@@ -1299,7 +1329,7 @@
"\n",
"# Check that quantiles are correctly updated - prediction\n",
"check = IQLoss()\n",
- "check.update_quantile(0.7)\n",
+ "check.update_quantile([0.7])\n",
"test_eq(check.q, 0.7)"
]
},
@@ -1357,19 +1387,6 @@
"outputs": [],
"source": [
"#| exporti\n",
- "def bernoulli_domain_map(input: torch.Tensor):\n",
- " \"\"\" Bernoulli Domain Map\n",
- " Maps input into distribution constraints, by construction input's \n",
- " last dimension is of matching `distr_args` length.\n",
- "\n",
- " **Parameters:**
\n",
- " `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
\n",
- "\n",
- " **Returns:**
\n",
- " `(probs,)`: tuple with tensors of Poisson distribution arguments.
\n",
- " \"\"\"\n",
- " return (input.squeeze(-1),)\n",
- "\n",
"def bernoulli_scale_decouple(output, loc=None, scale=None):\n",
" \"\"\" Bernoulli Scale Decouple\n",
"\n",
@@ -1383,21 +1400,6 @@
" probs = F.sigmoid(probs)#.clone()\n",
" return (probs,)\n",
"\n",
- "def student_domain_map(input: torch.Tensor):\n",
- " \"\"\" Student T Domain Map\n",
- " Maps input into distribution constraints, by construction input's \n",
- " last dimension is of matching `distr_args` length.\n",
- "\n",
- " **Parameters:**
\n",
- " `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
\n",
- " `eps`: float, helps the initialization of scale for easier optimization.
\n",
- "\n",
- " **Returns:**
\n",
- " `(df, loc, scale)`: tuple with tensors of StudentT distribution arguments.
\n",
- " \"\"\"\n",
- " df, loc, scale = torch.tensor_split(input, 3, dim=-1)\n",
- " return df.squeeze(-1), loc.squeeze(-1), scale.squeeze(-1)\n",
- "\n",
"def student_scale_decouple(output, loc=None, scale=None, eps: float=0.1):\n",
" \"\"\" Normal Scale Decouple\n",
"\n",
@@ -1413,21 +1415,6 @@
" df = 3.0 + F.softplus(df)\n",
" return (df, mean, tscale)\n",
"\n",
- "def normal_domain_map(input: torch.Tensor):\n",
- " \"\"\" Normal Domain Map\n",
- " Maps input into distribution constraints, by construction input's \n",
- " last dimension is of matching `distr_args` length.\n",
- "\n",
- " **Parameters:**
\n",
- " `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
\n",
- " `eps`: float, helps the initialization of scale for easier optimization.
\n",
- "\n",
- " **Returns:**
\n",
- " `(mean, std)`: tuple with tensors of Normal distribution arguments.
\n",
- " \"\"\"\n",
- " mean, std = torch.tensor_split(input, 2, dim=-1)\n",
- " return mean.squeeze(-1), std.squeeze(-1)\n",
- "\n",
"def normal_scale_decouple(output, loc=None, scale=None, eps: float=0.2):\n",
" \"\"\" Normal Scale Decouple\n",
"\n",
@@ -1442,19 +1429,6 @@
" std = (std + eps) * scale\n",
" return (mean, std)\n",
"\n",
- "def poisson_domain_map(input: torch.Tensor):\n",
- " \"\"\" Poisson Domain Map\n",
- " Maps input into distribution constraints, by construction input's \n",
- " last dimension is of matching `distr_args` length.\n",
- "\n",
- " **Parameters:**
\n",
- " `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
\n",
- "\n",
- " **Returns:**
\n",
- " `(rate,)`: tuple with tensors of Poisson distribution arguments.
\n",
- " \"\"\"\n",
- " return (input.squeeze(-1),)\n",
- "\n",
"def poisson_scale_decouple(output, loc=None, scale=None):\n",
" \"\"\" Poisson Scale Decouple\n",
"\n",
@@ -1467,21 +1441,7 @@
" if (loc is not None) and (scale is not None):\n",
" rate = (rate * scale) + loc\n",
" rate = F.softplus(rate) + eps\n",
- " return (rate,)\n",
- "\n",
- "def nbinomial_domain_map(input: torch.Tensor):\n",
- " \"\"\" Negative Binomial Domain Map\n",
- " Maps input into distribution constraints, by construction input's \n",
- " last dimension is of matching `distr_args` length.\n",
- "\n",
- " **Parameters:**
\n",
- " `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
\n",
- "\n",
- " **Returns:**
\n",
- " `(total_count, alpha)`: tuple with tensors of N.Binomial distribution arguments.
\n",
- " \"\"\"\n",
- " mu, alpha = torch.tensor_split(input, 2, dim=-1)\n",
- " return mu.squeeze(-1), alpha.squeeze(-1)\n",
+ " return (rate, )\n",
"\n",
"def nbinomial_scale_decouple(output, loc=None, scale=None):\n",
" \"\"\" Negative Binomial Scale Decouple\n",
@@ -1550,10 +1510,12 @@
" - [Jorgensen, B. (1987). Exponential Dispersion Models. Journal of the Royal Statistical Society. \n",
" Series B (Methodological), 49(2), 127–162. http://www.jstor.org/stable/2345415](http://www.jstor.org/stable/2345415)
\n",
" \"\"\"\n",
+ " arg_constraints = {'log_mu': constraints.real}\n",
+ " support = constraints.nonnegative\n",
+ "\n",
" def __init__(self, log_mu, rho, validate_args=None):\n",
" # TODO: add sigma2 dispersion\n",
" # TODO add constraints\n",
- " # arg_constraints = {'log_mu': constraints.real, 'rho': constraints.positive}\n",
" # support = constraints.real\n",
" self.log_mu = log_mu\n",
" self.rho = rho\n",
@@ -1587,7 +1549,7 @@
" beta = beta.expand(shape)\n",
"\n",
" N = torch.poisson(rate) + 1e-5\n",
- " gamma = torch.distributions.gamma.Gamma(N * alpha, beta)\n",
+ " gamma = Gamma(N*alpha, beta)\n",
" samples = gamma.sample()\n",
" samples[N==0] = 0\n",
"\n",
@@ -1602,12 +1564,12 @@
"\n",
" return a - b\n",
"\n",
- "def tweedie_domain_map(input: torch.Tensor):\n",
+ "def tweedie_domain_map(input: torch.Tensor, rho: float = 1.5):\n",
" \"\"\"\n",
" Maps output of neural network to domain of distribution loss\n",
"\n",
" \"\"\"\n",
- " return (input.squeeze(-1),)\n",
+ " return (input, rho)\n",
"\n",
"def tweedie_scale_decouple(output, loc=None, scale=None):\n",
" \"\"\"Tweedie Scale Decouple\n",
@@ -1616,14 +1578,14 @@
" count and logits based on anchoring `loc`, `scale`.\n",
" Also adds Tweedie domain protection to the distribution parameters.\n",
" \"\"\"\n",
- " log_mu = output[0]\n",
+ " log_mu, rho = output\n",
" log_mu = F.softplus(log_mu)\n",
" log_mu = torch.clamp(log_mu, 1e-9, 37)\n",
" if (loc is not None) and (scale is not None):\n",
" log_mu += torch.log(loc)\n",
"\n",
" log_mu = torch.clamp(log_mu, 1e-9, 37)\n",
- " return (log_mu,)"
+ " return (log_mu, rho)"
]
},
{
@@ -1687,6 +1649,15 @@
" scale *= t.scale\n",
" p = self.base_dist.crps(z)\n",
" return p * scale\n",
+ " \n",
+ " @property\n",
+ " def mean(self):\n",
+ " \"\"\"\n",
+ " Function used to compute the empirical mean\n",
+ " \"\"\"\n",
+ " samples = self.sample([1000])\n",
+ " return samples.mean(dim=0)\n",
+ " \n",
"\n",
"class BaseISQF(Distribution):\n",
" \"\"\"\n",
@@ -2357,7 +2328,7 @@
" last dimension is of matching `distr_args` length.\n",
"\n",
" **Parameters:**
\n",
- " `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
\n",
+ " `input`: tensor, of dimensions [B, H, N * n_outputs].
\n",
" `tol`: float, tolerance.
\n",
" `quantiles`: tensor, quantiles used for ISQF (i.e. x-positions for the knots).
\n",
" `num_pieces`: int, num_pieces used for each quantile spline.
\n",
@@ -2371,7 +2342,14 @@
" #\n",
" # Because in this case the spline knots could be squeezed together\n",
" # and cause overflow in spline CRPS computation\n",
- " num_qk = len(quantiles) \n",
+ " num_qk = len(quantiles)\n",
+ " n_outputs = 2 * (num_qk - 1) * num_pieces + 2 + num_qk\n",
+ " \n",
+ " # Reshape: [B, h, N * n_outputs] -> [B, h, N, n_outputs]\n",
+ " input = input.reshape(input.shape[0],\n",
+ " input.shape[1],\n",
+ " -1,\n",
+ " n_outputs)\n",
" start_index = 0\n",
" spline_knots = input[..., start_index: start_index + (num_qk - 1) * num_pieces]\n",
" start_index += (num_qk - 1) * num_pieces\n",
@@ -2381,27 +2359,19 @@
" start_index += 1\n",
" beta_r = input[..., start_index: start_index + 1]\n",
" start_index += 1\n",
- " quantile_knots = input[..., start_index: start_index + num_qk]\n",
- "\n",
- " qk_y = torch.cat(\n",
- " [\n",
- " quantile_knots[..., 0:1],\n",
- " torch.abs(quantile_knots[..., 1:]) + tol,\n",
- " ],\n",
- " dim=-1,\n",
- " )\n",
- " qk_y = torch.cumsum(qk_y, dim=-1)\n",
+ " quantile_knots = F.softplus(input[..., start_index: start_index + num_qk]) + tol\n",
+ "\n",
+ " qk_y = torch.cumsum(quantile_knots, dim=-1)\n",
"\n",
" # Prevent overflow when we compute 1/beta\n",
- " beta_l = torch.abs(beta_l.squeeze(-1)) + tol\n",
- " beta_r = torch.abs(beta_r.squeeze(-1)) + tol\n",
+ " beta_l = F.softplus(beta_l.squeeze(-1)) + tol\n",
+ " beta_r = F.softplus(beta_r.squeeze(-1)) + tol\n",
"\n",
" # Reshape spline arguments\n",
" batch_shape = spline_knots.shape[:-1]\n",
"\n",
" # repeat qk_x from (num_qk,) to (*batch_shape, num_qk)\n",
- " qk_x_repeat = torch.sort(quantiles)\\\n",
- " .values\\\n",
+ " qk_x_repeat = quantiles\\\n",
" .repeat(*batch_shape, 1)\\\n",
" .to(input.device)\n",
"\n",
@@ -2502,15 +2472,6 @@
" NegativeBinomial=NegativeBinomial,\n",
" Tweedie=Tweedie,\n",
" ISQF=ISQF)\n",
- " domain_maps = dict(Bernoulli=bernoulli_domain_map,\n",
- " Normal=normal_domain_map,\n",
- " Poisson=poisson_domain_map,\n",
- " StudentT=student_domain_map,\n",
- " NegativeBinomial=nbinomial_domain_map,\n",
- " Tweedie=tweedie_domain_map,\n",
- " ISQF=partial(isqf_domain_map, \n",
- " quantiles=qs, \n",
- " num_pieces=num_pieces))\n",
" scale_decouples = dict(\n",
" Bernoulli=bernoulli_scale_decouple,\n",
" Normal=normal_scale_decouple,\n",
@@ -2531,9 +2492,24 @@
" [f\"-quantile_knot_{i + 1}\" for i in range(num_qk)],\n",
" )\n",
" assert (distribution in available_distributions.keys()), f'{distribution} not available'\n",
+ " if distribution == 'ISQF':\n",
+ " quantiles = torch.sort(qs).values\n",
+ " self.domain_map = partial(isqf_domain_map, \n",
+ " quantiles=quantiles, \n",
+ " num_pieces=num_pieces)\n",
+ " if return_params:\n",
+ " raise Exception(\"ISQF does not support 'return_params=True'\") \n",
+ " elif distribution == 'Tweedie':\n",
+ " rho = distribution_kwargs.pop(\"rho\")\n",
+ " self.domain_map = partial(tweedie_domain_map,\n",
+ " rho=rho)\n",
+ " if return_params:\n",
+ " raise Exception(\"Tweedie does not support 'return_params=True'\") \n",
+ " else:\n",
+ " self.domain_map = self._domain_map\n",
+ "\n",
" self.distribution = distribution\n",
" self._base_distribution = available_distributions[distribution]\n",
- " self.domain_map = domain_maps[distribution]\n",
" self.scale_decouple = scale_decouples[distribution]\n",
" self.distribution_kwargs = distribution_kwargs\n",
" self.num_samples = num_samples \n",
@@ -2549,6 +2525,16 @@
"\n",
" self.outputsize_multiplier = len(self.param_names)\n",
" self.is_distribution_output = True\n",
+ " self.has_predicted = False\n",
+ "\n",
+ " def _domain_map(self, input: torch.Tensor):\n",
+ " \"\"\"\n",
+ " Maps output of neural network to domain of distribution loss\n",
+ "\n",
+ " \"\"\"\n",
+ " output = torch.tensor_split(input, self.outputsize_multiplier, dim=2)\n",
+ "\n",
+ " return output\n",
"\n",
" def get_distribution(self, distr_args, **distribution_kwargs) -> Distribution:\n",
" \"\"\"\n",
@@ -2561,10 +2547,10 @@
" **Returns**
\n",
" `Distribution`: AffineTransformed distribution.
\n",
" \"\"\"\n",
- " # TransformedDistribution(distr, [AffineTransform(loc=loc, scale=scale)])\n",
" distr = self._base_distribution(*distr_args, **distribution_kwargs)\n",
+ " self.distr_mean = distr.mean\n",
" \n",
- " if self.distribution =='Poisson':\n",
+ " if self.distribution in ('Poisson', 'NegativeBinomial'):\n",
" distr.support = constraints.nonnegative\n",
" return distr\n",
"\n",
@@ -2577,7 +2563,7 @@
"\n",
" **Parameters**
\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
- " `num_samples`: int=500, overwrite number of samples for the empirical quantiles.
\n",
+ " `num_samples`: int, overwrite number of samples for the empirical quantiles.
\n",
"\n",
" **Returns**
\n",
" `samples`: tensor, shape [B,H,`num_samples`].
\n",
@@ -2586,30 +2572,31 @@
" if num_samples is None:\n",
" num_samples = self.num_samples\n",
"\n",
- " # print(distr_args[0].size())\n",
- " B, H = distr_args[0].shape[:2]\n",
- " Q = len(self.quantiles)\n",
- "\n",
" # Instantiate Scaled Decoupled Distribution\n",
" distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)\n",
" samples = distr.sample(sample_shape=(num_samples,))\n",
- " samples = samples.permute(1,2,0) # [samples,B,H] -> [B,H,samples]\n",
- " samples = samples.view(B*H, num_samples)\n",
- " sample_mean = torch.mean(samples, dim=-1)\n",
+ " samples = samples.permute(1, 2, 3, 0) # [samples, B, H, N] -> [B, H, N, samples]\n",
+ "\n",
+ " sample_mean = torch.mean(samples, dim=-1, keepdim=True) \n",
"\n",
" # Compute quantiles\n",
" quantiles_device = self.quantiles.to(distr_args[0].device)\n",
" quants = torch.quantile(input=samples, \n",
- " q=quantiles_device, dim=1)\n",
- " quants = quants.permute((1,0)) # [Q, B*H] -> [B*H, Q]\n",
- "\n",
- " # Final reshapes\n",
- " samples = samples.view(B, H, num_samples)\n",
- " sample_mean = sample_mean.view(B, H, 1)\n",
- " quants = quants.view(B, H, Q)\n",
+ " q=quantiles_device, \n",
+ " dim=-1)\n",
+ " quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]\n",
"\n",
" return samples, sample_mean, quants\n",
"\n",
+ " def update_quantile(self, q: Optional[List[float]] = None):\n",
+ " if q is not None:\n",
+ " self.quantiles = nn.Parameter(torch.tensor(q, dtype=torch.float32), requires_grad=False)\n",
+ " self.output_names = [\"\"] + [f\"_ql{q_i}\" for q_i in q] + self.return_params * self.param_names\n",
+ " self.has_predicted = True\n",
+ " elif q is None and self.has_predicted:\n",
+ " self.quantiles = nn.Parameter(torch.tensor([0.5], dtype=torch.float32), requires_grad=False)\n",
+ " self.output_names = [\"\", \"-median\"] + self.return_params * self.param_names\n",
+ "\n",
" def __call__(self,\n",
" y: torch.Tensor,\n",
" distr_args: torch.Tensor,\n",
@@ -2626,10 +2613,6 @@
" **Parameters**
\n",
" `y`: tensor, Actual values.
\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
- " `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
- " of the resulting distribution.
\n",
- " `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
- " of the resulting distribution.
\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.
\n",
"\n",
" **Returns**
\n",
@@ -2739,7 +2722,8 @@
" \"\"\"\n",
" def __init__(self, n_components=10, level=[80, 90], quantiles=None,\n",
" num_samples=1000, return_params=False,\n",
- " batch_correlation=False, horizon_correlation=False):\n",
+ " batch_correlation=False, horizon_correlation=False, \n",
+ " weighted=False):\n",
" super(PMM, self).__init__()\n",
" # Transform level to MQLoss parameters\n",
" qs, self.output_names = level_to_outputs(level)\n",
@@ -2753,21 +2737,37 @@
" self.num_samples = num_samples\n",
" self.batch_correlation = batch_correlation\n",
" self.horizon_correlation = horizon_correlation\n",
+ " self.weighted = weighted \n",
"\n",
" # If True, predict_step will return Distribution's parameters\n",
" self.return_params = return_params\n",
- " if self.return_params:\n",
- " self.param_names = [f\"-lambda-{i}\" for i in range(1, n_components + 1)]\n",
+ "\n",
+ " lambda_names = [f\"-lambda-{i}\" for i in range(1, n_components + 1)]\n",
+ " if weighted:\n",
+ " weight_names = [f\"-weight-{i}\" for i in range(1, n_components + 1)]\n",
+ " self.param_names = [i for j in zip(lambda_names, weight_names) for i in j]\n",
+ " else:\n",
+ " self.param_names = lambda_names\n",
+ "\n",
+ " if self.return_params: \n",
" self.output_names = self.output_names + self.param_names\n",
"\n",
" # Add first output entry for the sample_mean\n",
" self.output_names.insert(0, \"\")\n",
"\n",
- " self.outputsize_multiplier = n_components\n",
+ " self.n_outputs = 1 + weighted\n",
+ " self.n_components = n_components\n",
+ " self.outputsize_multiplier = self.n_outputs * n_components\n",
" self.is_distribution_output = True\n",
+ " self.has_predicted = False\n",
"\n",
" def domain_map(self, output: torch.Tensor):\n",
- " return (output,)#, weights\n",
+ " output = output.reshape(output.shape[0],\n",
+ " output.shape[1],\n",
+ " -1,\n",
+ " self.outputsize_multiplier)\n",
+ " \n",
+ " return torch.tensor_split(output, self.n_outputs, dim=-1)\n",
" \n",
" def scale_decouple(self, \n",
" output,\n",
@@ -2779,26 +2779,62 @@
" variance and residual location based on anchoring `loc`, `scale`.\n",
" Also adds domain protection to the distribution parameters.\n",
" \"\"\"\n",
- " lambdas = output[0]\n",
+ " if self.weighted:\n",
+ " lambdas, weights = output\n",
+ " weights = F.softmax(weights, dim=-1)\n",
+ " else:\n",
+ " lambdas = output[0]\n",
+ "\n",
" if (loc is not None) and (scale is not None):\n",
- " loc = loc.view(lambdas.size(dim=0), 1, -1)\n",
- " scale = scale.view(lambdas.size(dim=0), 1, -1)\n",
+ " if loc.ndim == 3:\n",
+ " loc = loc.unsqueeze(-1)\n",
+ " scale = scale.unsqueeze(-1)\n",
" lambdas = (lambdas * scale) + loc\n",
- " lambdas = F.softplus(lambdas)\n",
- " return (lambdas,)\n",
"\n",
- " def sample(self, distr_args, num_samples=None):\n",
+ " lambdas = F.softplus(lambdas) + 1e-3\n",
+ " \n",
+ " if self.weighted:\n",
+ " return (lambdas, weights)\n",
+ " else:\n",
+ " return (lambdas, )\n",
+ " \n",
+ " def get_distribution(self, distr_args) -> Distribution:\n",
+ " \"\"\"\n",
+ " Construct the associated Pytorch Distribution, given the collection of\n",
+ " constructor arguments and, optionally, location and scale tensors.\n",
+ "\n",
+ " **Parameters**
\n",
+ " `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
+ "\n",
+ " **Returns**
\n",
+ " `Distribution`: AffineTransformed distribution.
\n",
+ " \"\"\"\n",
+ " if self.weighted:\n",
+ " lambdas, weights = distr_args\n",
+ " else:\n",
+ " lambdas = distr_args[0]\n",
+ " weights = torch.full_like(lambdas, fill_value=1 / self.n_components)\n",
+ "\n",
+ " mix = Categorical(weights)\n",
+ " components = Poisson(rate=lambdas)\n",
+ " components.support = constraints.nonnegative\n",
+ " distr = MixtureSameFamily(mixture_distribution=mix,\n",
+ " component_distribution=components) \n",
+ "\n",
+ " self.distr_mean = distr.mean\n",
+ " \n",
+ " return distr\n",
+ "\n",
+ " def sample(self,\n",
+ " distr_args: torch.Tensor,\n",
+ " num_samples: Optional[int] = None):\n",
" \"\"\"\n",
" Construct the empirical quantiles from the estimated Distribution,\n",
" sampling from it `num_samples` independently.\n",
"\n",
" **Parameters**
\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
- " `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
- " of the resulting distribution.
\n",
- " `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
- " of the resulting distribution.
\n",
- " `num_samples`: int=500, overwrites number of samples for the empirical quantiles.
\n",
+ " `num_samples`: int, overwrite number of samples for the empirical quantiles.
\n",
"\n",
" **Returns**
\n",
" `samples`: tensor, shape [B,H,`num_samples`].
\n",
@@ -2807,93 +2843,65 @@
" if num_samples is None:\n",
" num_samples = self.num_samples\n",
"\n",
- " lambdas = distr_args[0]\n",
- " B, H, K = lambdas.size()\n",
- " Q = len(self.quantiles)\n",
- "\n",
- " # Sample K ~ Mult(weights)\n",
- " # shared across B, H\n",
- " # weights = torch.repeat_interleave(input=weights, repeats=H, dim=2)\n",
- " weights = (1/K) * torch.ones_like(lambdas, device=lambdas.device)\n",
- "\n",
- " # Avoid loop, vectorize\n",
- " weights = weights.reshape(-1, K)\n",
- " lambdas = lambdas.flatten() \n",
- "\n",
- " # Vectorization trick to recover row_idx\n",
- " sample_idxs = torch.multinomial(input=weights, \n",
- " num_samples=num_samples,\n",
- " replacement=True)\n",
- " aux_col_idx = torch.unsqueeze(torch.arange(B * H, device=lambdas.device), -1) * K\n",
- "\n",
- " # To device\n",
- " sample_idxs = sample_idxs.to(lambdas.device)\n",
- "\n",
- " sample_idxs = sample_idxs + aux_col_idx\n",
- " sample_idxs = sample_idxs.flatten()\n",
- "\n",
- " sample_lambdas = lambdas[sample_idxs]\n",
+ " # Instantiate Scaled Decoupled Distribution\n",
+ " distr = self.get_distribution(distr_args=distr_args)\n",
+ " samples = distr.sample(sample_shape=(num_samples,))\n",
+ " samples = samples.permute(1, 2, 3, 0) # [samples, B, H, N] -> [B, H, N, samples]\n",
"\n",
- " # Sample y ~ Poisson(lambda) independently\n",
- " samples = torch.poisson(sample_lambdas).to(lambdas.device)\n",
- " samples = samples.view(B*H, num_samples)\n",
- " sample_mean = torch.mean(samples, dim=-1)\n",
+ " sample_mean = torch.mean(samples, dim=-1, keepdim=True) \n",
"\n",
" # Compute quantiles\n",
- " quantiles_device = self.quantiles.to(lambdas.device)\n",
- " quants = torch.quantile(input=samples, q=quantiles_device, dim=1)\n",
- " quants = quants.permute((1,0)) # Q, B*H\n",
- "\n",
- " # Final reshapes\n",
- " samples = samples.view(B, H, num_samples)\n",
- " sample_mean = sample_mean.view(B, H, 1)\n",
- " quants = quants.view(B, H, Q)\n",
+ " quantiles_device = self.quantiles.to(distr_args[0].device)\n",
+ " quants = torch.quantile(input=samples, \n",
+ " q=quantiles_device, \n",
+ " dim=-1)\n",
+ " quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]\n",
"\n",
" return samples, sample_mean, quants\n",
" \n",
- " def neglog_likelihood(self,\n",
- " y: torch.Tensor,\n",
- " distr_args: Tuple[torch.Tensor],\n",
- " mask: Union[torch.Tensor, None] = None,):\n",
- " if mask is None: \n",
- " mask = (y > 0) * 1\n",
- " else:\n",
- " mask = mask * ((y > 0) * 1)\n",
+ " def update_quantile(self, q: Optional[List[float]] = None):\n",
+ " if q is not None:\n",
+ " self.quantiles = nn.Parameter(torch.tensor(q, dtype=torch.float32), requires_grad=False)\n",
+ " self.output_names = [\"\"] + [f\"_ql{q_i}\" for q_i in q] + self.return_params * self.param_names\n",
+ " self.has_predicted = True\n",
+ " elif q is None and self.has_predicted:\n",
+ " self.quantiles = nn.Parameter(torch.tensor([0.5], dtype=torch.float32), requires_grad=False) \n",
+ " self.output_names = [\"\", \"-median\"] + self.return_params * self.param_names\n",
"\n",
- " eps = 1e-10\n",
- " lambdas = distr_args[0]\n",
- " B, H, K = lambdas.size()\n",
+ " def __call__(self,\n",
+ " y: torch.Tensor,\n",
+ " distr_args: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None):\n",
+ " \"\"\"\n",
+ " Computes the negative log-likelihood objective function. \n",
+ " To estimate the following predictive distribution:\n",
"\n",
- " weights = (1/K) * torch.ones_like(lambdas, device=lambdas.device)\n",
+ " $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
"\n",
- " y = y[:,:,None]\n",
- " mask = mask[:,:,None]\n",
+ " where $\\\\theta$ represents the distributions parameters. It aditionally \n",
+ " summarizes the objective signal using a weighted average using the `mask` tensor. \n",
"\n",
- " y = y * mask # Protect y negative entries\n",
- " \n",
- " # Single Poisson likelihood\n",
- " log_pi = y.xlogy(lambdas + eps) - lambdas - (y + 1).lgamma()\n",
+ " **Parameters**
\n",
+ " `y`: tensor, Actual values.
\n",
+ " `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
+ " `mask`: tensor, Specifies date stamps per serie to consider in loss.
\n",
"\n",
+ " **Returns**
\n",
+ " `loss`: scalar, weighted loss function against which backpropagation will be performed.
\n",
+ " \"\"\"\n",
+ " # Instantiate Scaled Decoupled Distribution\n",
+ " distr = self.get_distribution(distr_args=distr_args)\n",
+ " x = distr._pad(y)\n",
+ " log_prob_x = distr.component_distribution.log_prob(x)\n",
+ " log_mix_prob = torch.log_softmax(distr.mixture_distribution.logits, dim=-1)\n",
" if self.batch_correlation:\n",
- " log_pi = torch.sum(log_pi, dim=0, keepdim=True)\n",
- "\n",
+ " log_prob_x = torch.sum(log_prob_x, dim=0, keepdim=True)\n",
" if self.horizon_correlation:\n",
- " log_pi = torch.sum(log_pi, dim=1, keepdim=True)\n",
- "\n",
- " # Numerically Stable Mixture loglikelihood\n",
- " loglik = torch.logsumexp((torch.log(weights) + log_pi), dim=2, keepdim=True)\n",
- " loglik = loglik * mask\n",
- "\n",
- " mean = torch.sum(weights * lambdas, axis=-1, keepdims=True)\n",
- " reglrz = torch.mean(torch.square(y - mean) * mask)\n",
- " loss = -torch.mean(loglik) + 0.001 * reglrz\n",
- " return loss\n",
- "\n",
- " def __call__(self, y: torch.Tensor,\n",
- " distr_args: Tuple[torch.Tensor],\n",
- " mask: Union[torch.Tensor, None] = None):\n",
- "\n",
- " return self.neglog_likelihood(y=y, distr_args=distr_args, mask=mask)\n"
+ " log_prob_x = torch.sum(log_prob_x, dim=1, keepdim=True)\n",
+ " \n",
+ " loss_values = -torch.logsumexp(log_prob_x + log_mix_prob, dim=-1) \n",
+ " \n",
+ " return weighted_average(loss_values, weights=mask)\n"
]
},
{
@@ -2967,30 +2975,31 @@
"outputs": [],
"source": [
"#| hide\n",
- "# Create single mixture and broadcast to N,H,K\n",
- "weights = torch.ones((1,3))[None, :, :]\n",
- "lambdas = torch.Tensor([[5,10,15], [10,20,30]])[None, :, :]\n",
+ "# Create single mixture and broadcast to N,H,1,K\n",
+ "weights = torch.ones((1,3))[None, :, :].unsqueeze(2)\n",
+ "lambdas = torch.Tensor([[5,10,15], [10,20,30]])[None, :, :].unsqueeze(2)\n",
"\n",
"# Create repetitions for the batch dimension N.\n",
"N=2\n",
"weights = torch.repeat_interleave(input=weights, repeats=N, dim=0)\n",
"lambdas = torch.repeat_interleave(input=lambdas, repeats=N, dim=0)\n",
"\n",
- "print('weights.shape (N,H,K) \\t', weights.shape)\n",
- "print('lambdas.shape (N,H,K) \\t', lambdas.shape)\n",
+ "print('weights.shape (N,H,1,K) \\t', weights.shape)\n",
+ "print('lambdas.shape (N,H,1, K) \\t', lambdas.shape)\n",
"\n",
- "distr = PMM(quantiles=[0.1, 0.40, 0.5, 0.60, 0.9])\n",
- "distr_args = (lambdas,)\n",
+ "distr = PMM(quantiles=[0.1, 0.40, 0.5, 0.60, 0.9], weighted=True)\n",
+ "weights = torch.ones_like(lambdas)\n",
+ "distr_args = (lambdas, weights)\n",
"samples, sample_mean, quants = distr.sample(distr_args)\n",
"\n",
- "print('samples.shape (N,H,num_samples) ', samples.shape)\n",
- "print('sample_mean.shape (N,H) ', sample_mean.shape)\n",
- "print('quants.shape (N,H,Q) \\t\\t', quants.shape)\n",
+ "print('samples.shape (N,H,1,num_samples) ', samples.shape)\n",
+ "print('sample_mean.shape (N,H,1,1) ', sample_mean.shape)\n",
+ "print('quants.shape (N,H,1,Q) \\t\\t', quants.shape)\n",
"\n",
"# Plot synthethic data\n",
"x_plot = range(quants.shape[1]) # H length\n",
- "y_plot_hat = quants[0,:,:] # Filter N,G,T -> H,Q\n",
- "samples_hat = samples[0,:,:] # Filter N,G,T -> H,num_samples\n",
+ "y_plot_hat = quants[0,:,0,:] # Filter N,G,T -> H,Q\n",
+ "samples_hat = samples[0,:,0,:] # Filter N,G,T -> H,num_samples\n",
"\n",
"# Kernel density plot for single forecast horizon \\tau = t+1\n",
"fig, ax = plt.subplots(figsize=(3.7, 2.9))\n",
@@ -3065,7 +3074,8 @@
" \"\"\"\n",
" def __init__(self, n_components=1, level=[80, 90], quantiles=None, \n",
" num_samples=1000, return_params=False,\n",
- " batch_correlation=False, horizon_correlation=False):\n",
+ " batch_correlation=False, horizon_correlation=False,\n",
+ " weighted=False):\n",
" super(GMM, self).__init__()\n",
" # Transform level to MQLoss parameters\n",
" qs, self.output_names = level_to_outputs(level)\n",
@@ -3078,25 +3088,41 @@
" self.quantiles = torch.nn.Parameter(qs, requires_grad=False)\n",
" self.num_samples = num_samples\n",
" self.batch_correlation = batch_correlation\n",
- " self.horizon_correlation = horizon_correlation \n",
+ " self.horizon_correlation = horizon_correlation \n",
+ " self.weighted = weighted \n",
"\n",
" # If True, predict_step will return Distribution's parameters\n",
" self.return_params = return_params\n",
+ "\n",
+ " mu_names = [f\"-mu-{i}\" for i in range(1, n_components + 1)]\n",
+ " std_names = [f\"-std-{i}\" for i in range(1, n_components + 1)]\n",
+ " if weighted:\n",
+ " weight_names = [f\"-weight-{i}\" for i in range(1, n_components + 1)]\n",
+ " self.param_names = [\n",
+ " i for j in zip(mu_names, std_names, weight_names) for i in j\n",
+ " ]\n",
+ " else:\n",
+ " self.param_names = [i for j in zip(mu_names, std_names) for i in j]\n",
+ "\n",
" if self.return_params:\n",
- " mu_names = [f\"-mu-{i}\" for i in range(1, n_components + 1)]\n",
- " std_names = [f\"-std-{i}\" for i in range(1, n_components + 1)]\n",
- " mu_std_names = [i for j in zip(mu_names, std_names) for i in j]\n",
- " self.output_names = self.output_names + mu_std_names\n",
+ " self.output_names = self.output_names + self.param_names\n",
"\n",
" # Add first output entry for the sample_mean\n",
" self.output_names.insert(0, \"\")\n",
"\n",
- " self.outputsize_multiplier = 2 * n_components\n",
+ " self.n_outputs = 2 + weighted\n",
+ " self.n_components = n_components\n",
+ " self.outputsize_multiplier = self.n_outputs * n_components\n",
" self.is_distribution_output = True\n",
+ " self.has_predicted = False\n",
"\n",
" def domain_map(self, output: torch.Tensor):\n",
- " means, stds = torch.tensor_split(output, 2, dim=-1)\n",
- " return (means, stds)\n",
+ " output = output.reshape(output.shape[0],\n",
+ " output.shape[1],\n",
+ " -1,\n",
+ " self.outputsize_multiplier)\n",
+ " \n",
+ " return torch.tensor_split(output, self.n_outputs, dim=-1)\n",
"\n",
" def scale_decouple(self, \n",
" output,\n",
@@ -3109,27 +3135,61 @@
" variance and residual location based on anchoring `loc`, `scale`.\n",
" Also adds domain protection to the distribution parameters.\n",
" \"\"\"\n",
- " means, stds = output\n",
+ " if self.weighted:\n",
+ " means, stds, weights = output\n",
+ " weights = F.softmax(weights, dim=-1)\n",
+ " else:\n",
+ " means, stds = output\n",
+ " \n",
" stds = F.softplus(stds)\n",
" if (loc is not None) and (scale is not None):\n",
- " loc = loc.view(means.size(dim=0), 1, -1)\n",
- " scale = scale.view(means.size(dim=0), 1, -1) \n",
+ " if loc.ndim == 3:\n",
+ " loc = loc.unsqueeze(-1)\n",
+ " scale = scale.unsqueeze(-1)\n",
" means = (means * scale) + loc\n",
" stds = (stds + eps) * scale\n",
- " return (means, stds)\n",
+ " \n",
+ " if self.weighted:\n",
+ " return (means, stds, weights)\n",
+ " else:\n",
+ " return (means, stds)\n",
+ "\n",
+ " def get_distribution(self, distr_args) -> Distribution:\n",
+ " \"\"\"\n",
+ " Construct the associated Pytorch Distribution, given the collection of\n",
+ " constructor arguments and, optionally, location and scale tensors.\n",
"\n",
- " def sample(self, distr_args, num_samples=None):\n",
+ " **Parameters**
\n",
+ " `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
+ "\n",
+ " **Returns**
\n",
+ " `Distribution`: AffineTransformed distribution.
\n",
+ " \"\"\"\n",
+ " if self.weighted:\n",
+ " means, stds, weights = distr_args\n",
+ " else:\n",
+ " means, stds = distr_args\n",
+ " weights = torch.full_like(means, fill_value=1 / self.n_components)\n",
+ " \n",
+ " mix = Categorical(weights)\n",
+ " components = Normal(loc=means, scale=stds)\n",
+ " distr = MixtureSameFamily(mixture_distribution=mix,\n",
+ " component_distribution=components) \n",
+ "\n",
+ " self.distr_mean = distr.mean\n",
+ " \n",
+ " return distr\n",
+ "\n",
+ " def sample(self,\n",
+ " distr_args: torch.Tensor,\n",
+ " num_samples: Optional[int] = None):\n",
" \"\"\"\n",
" Construct the empirical quantiles from the estimated Distribution,\n",
" sampling from it `num_samples` independently.\n",
"\n",
" **Parameters**
\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
- " `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
- " of the resulting distribution.
\n",
- " `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
- " of the resulting distribution.
\n",
- " `num_samples`: int=500, number of samples for the empirical quantiles.
\n",
+ " `num_samples`: int, overwrite number of samples for the empirical quantiles.
\n",
"\n",
" **Returns**
\n",
" `samples`: tensor, shape [B,H,`num_samples`].
\n",
@@ -3137,94 +3197,65 @@
" \"\"\"\n",
" if num_samples is None:\n",
" num_samples = self.num_samples\n",
- " \n",
- " means, stds = distr_args\n",
- " B, H, K = means.size()\n",
- " Q = len(self.quantiles)\n",
- " assert means.shape == stds.shape\n",
- "\n",
- " # Sample K ~ Mult(weights)\n",
- " # shared across B, H\n",
- " # weights = torch.repeat_interleave(input=weights, repeats=H, dim=2)\n",
- " \n",
- " weights = (1/K) * torch.ones_like(means, device=means.device)\n",
- " \n",
- " # Avoid loop, vectorize\n",
- " weights = weights.reshape(-1, K)\n",
- " means = means.flatten()\n",
- " stds = stds.flatten()\n",
- "\n",
- " # Vectorization trick to recover row_idx\n",
- " sample_idxs = torch.multinomial(input=weights, \n",
- " num_samples=num_samples,\n",
- " replacement=True)\n",
- " aux_col_idx = torch.unsqueeze(torch.arange(B * H, device=means.device),-1) * K\n",
- "\n",
- " # To device\n",
- " sample_idxs = sample_idxs.to(means.device)\n",
"\n",
- " sample_idxs = sample_idxs + aux_col_idx\n",
- " sample_idxs = sample_idxs.flatten()\n",
- "\n",
- " sample_means = means[sample_idxs]\n",
- " sample_stds = stds[sample_idxs]\n",
+ " # Instantiate Scaled Decoupled Distribution\n",
+ " distr = self.get_distribution(distr_args=distr_args)\n",
+ " samples = distr.sample(sample_shape=(num_samples,))\n",
+ " samples = samples.permute(1, 2, 3, 0) # [samples, B, H, N] -> [B, H, N, samples]\n",
"\n",
- " # Sample y ~ Normal(mu, std) independently\n",
- " samples = torch.normal(sample_means, sample_stds).to(means.device)\n",
- " samples = samples.view(B*H, num_samples)\n",
- " sample_mean = torch.mean(samples, dim=-1)\n",
+ " sample_mean = torch.mean(samples, dim=-1, keepdim=True) \n",
"\n",
" # Compute quantiles\n",
- " quantiles_device = self.quantiles.to(means.device)\n",
- " quants = torch.quantile(input=samples, q=quantiles_device, dim=1)\n",
- " quants = quants.permute((1,0)) # Q, B*H\n",
- "\n",
- " # Final reshapes\n",
- " samples = samples.view(B, H, num_samples)\n",
- " sample_mean = sample_mean.view(B, H, 1)\n",
- " quants = quants.view(B, H, Q)\n",
+ " quantiles_device = self.quantiles.to(distr_args[0].device)\n",
+ " quants = torch.quantile(input=samples, \n",
+ " q=quantiles_device, \n",
+ " dim=-1)\n",
+ " quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]\n",
"\n",
" return samples, sample_mean, quants\n",
+ " \n",
+ " def update_quantile(self, q: Optional[List[float]] = None):\n",
+ " if q is not None:\n",
+ " self.quantiles = nn.Parameter(torch.tensor(q, dtype=torch.float32), requires_grad=False)\n",
+ " self.output_names = [\"\"] + [f\"_ql{q_i}\" for q_i in q] + self.return_params * self.param_names\n",
+ " self.has_predicted = True\n",
+ " elif q is None and self.has_predicted:\n",
+ " self.quantiles = nn.Parameter(torch.tensor([0.5], dtype=torch.float32), requires_grad=False) \n",
+ " self.output_names = [\"\", \"-median\"] + self.return_params * self.param_names\n",
"\n",
- " def neglog_likelihood(self,\n",
- " y: torch.Tensor,\n",
- " distr_args: Tuple[torch.Tensor, torch.Tensor],\n",
- " mask: Union[torch.Tensor, None] = None):\n",
- "\n",
- " if mask is None: \n",
- " mask = torch.ones_like(y)\n",
- " \n",
- " means, stds = distr_args\n",
- " B, H, K = means.size()\n",
- " \n",
- " weights = (1/K) * torch.ones_like(means, device=means.device)\n",
- " \n",
- " y = y[:,:, None]\n",
- " mask = mask[:,:,None]\n",
- " \n",
- " var = stds ** 2\n",
- " log_stds = torch.log(stds)\n",
- " log_pi = - ((y - means) ** 2 / (2 * var)) - log_stds \\\n",
- " - math.log(math.sqrt(2 * math.pi))\n",
- "\n",
- " if self.batch_correlation:\n",
- " log_pi = torch.sum(log_pi, dim=0, keepdim=True)\n",
+ " def __call__(self,\n",
+ " y: torch.Tensor,\n",
+ " distr_args: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None):\n",
+ " \"\"\"\n",
+ " Computes the negative log-likelihood objective function. \n",
+ " To estimate the following predictive distribution:\n",
"\n",
- " if self.horizon_correlation: \n",
- " log_pi = torch.sum(log_pi, dim=1, keepdim=True)\n",
+ " $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
"\n",
- " # Numerically Stable Mixture loglikelihood\n",
- " loglik = torch.logsumexp((torch.log(weights) + log_pi), dim=2, keepdim=True)\n",
- " loglik = loglik * mask\n",
+ " where $\\\\theta$ represents the distributions parameters. It aditionally \n",
+ " summarizes the objective signal using a weighted average using the `mask` tensor. \n",
"\n",
- " loss = -torch.mean(loglik)\n",
- " return loss\n",
- " \n",
- " def __call__(self, y: torch.Tensor,\n",
- " distr_args: Tuple[torch.Tensor, torch.Tensor],\n",
- " mask: Union[torch.Tensor, None] = None,):\n",
+ " **Parameters**
\n",
+ " `y`: tensor, Actual values.
\n",
+ " `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
+ " `mask`: tensor, Specifies date stamps per serie to consider in loss.
\n",
"\n",
- " return self.neglog_likelihood(y=y, distr_args=distr_args, mask=mask)"
+ " **Returns**
\n",
+ " `loss`: scalar, weighted loss function against which backpropagation will be performed.
\n",
+ " \"\"\"\n",
+ " # Instantiate Scaled Decoupled Distribution\n",
+ " distr = self.get_distribution(distr_args=distr_args)\n",
+ " x = distr._pad(y)\n",
+ " log_prob_x = distr.component_distribution.log_prob(x)\n",
+ " log_mix_prob = torch.log_softmax(distr.mixture_distribution.logits, dim=-1)\n",
+ " if self.batch_correlation:\n",
+ " log_prob_x = torch.sum(log_prob_x, dim=0, keepdim=True)\n",
+ " if self.horizon_correlation:\n",
+ " log_prob_x = torch.sum(log_prob_x, dim=1, keepdim=True)\n",
+ " loss_values = -torch.logsumexp(log_prob_x + log_mix_prob, dim=-1) \n",
+ " \n",
+ " return weighted_average(loss_values, weights=mask)"
]
},
{
@@ -3298,8 +3329,8 @@
"outputs": [],
"source": [
"#| hide\n",
- "# Create single mixture and broadcast to N,H,K\n",
- "means = torch.Tensor([[5,10,15], [10,20,30]])[None, :, :]\n",
+ "# Create single mixture and broadcast to N,H,1,K\n",
+ "means = torch.Tensor([[5,10,15], [10,20,30]])[None, :, :].unsqueeze(2)\n",
"\n",
"# # Create repetitions for the batch dimension N.\n",
"N=2\n",
@@ -3307,22 +3338,22 @@
"weights = torch.ones_like(means)\n",
"stds = torch.ones_like(means)\n",
"\n",
- "print('weights.shape (N,H,K) \\t', weights.shape)\n",
- "print('means.shape (N,H,K) \\t', means.shape)\n",
- "print('stds.shape (N,H,K) \\t', stds.shape)\n",
+ "print('weights.shape (N,H,1,K) \\t', weights.shape)\n",
+ "print('means.shape (N,H,1,K) \\t', means.shape)\n",
+ "print('stds.shape (N,H,1,K) \\t', stds.shape)\n",
"\n",
- "distr = GMM(quantiles=[0.1, 0.40, 0.5, 0.60, 0.9])\n",
- "distr_args = (means, stds)\n",
+ "distr = GMM(quantiles=[0.1, 0.40, 0.5, 0.60, 0.9], weighted=True)\n",
+ "distr_args = (means, stds, weights)\n",
"samples, sample_mean, quants = distr.sample(distr_args)\n",
"\n",
- "print('samples.shape (N,H,num_samples) ', samples.shape)\n",
- "print('sample_mean.shape (N,H) ', sample_mean.shape)\n",
- "print('quants.shape (N,H,Q) \\t\\t', quants.shape)\n",
+ "print('samples.shape (N,H,1,num_samples) ', samples.shape)\n",
+ "print('sample_mean.shape (N,H,1,1) ', sample_mean.shape)\n",
+ "print('quants.shape (N,H,1, Q) \\t\\t', quants.shape)\n",
"\n",
"# Plot synthethic data\n",
"x_plot = range(quants.shape[1]) # H length\n",
- "y_plot_hat = quants[0,:,:] # Filter N,G,T -> H,Q\n",
- "samples_hat = samples[0,:,:] # Filter N,G,T -> H,num_samples\n",
+ "y_plot_hat = quants[0,:,0,:] # Filter N,G,T -> H,Q\n",
+ "samples_hat = samples[0,:,0,:] # Filter N,G,T -> H,num_samples\n",
"\n",
"# Kernel density plot for single forecast horizon \\tau = t+1\n",
"fig, ax = plt.subplots(figsize=(3.7, 2.9))\n",
@@ -3396,7 +3427,7 @@
" Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)\n",
" \"\"\"\n",
" def __init__(self, n_components=1, level=[80, 90], quantiles=None, \n",
- " num_samples=1000, return_params=False):\n",
+ " num_samples=1000, return_params=False, weighted=False):\n",
" super(NBMM, self).__init__()\n",
" # Transform level to MQLoss parameters\n",
" qs, self.output_names = level_to_outputs(level)\n",
@@ -3408,24 +3439,40 @@
" qs = torch.Tensor(quantiles)\n",
" self.quantiles = torch.nn.Parameter(qs, requires_grad=False)\n",
" self.num_samples = num_samples\n",
+ " self.weighted = weighted \n",
"\n",
" # If True, predict_step will return Distribution's parameters\n",
" self.return_params = return_params\n",
+ "\n",
+ " total_count_names = [f\"-total_count-{i}\" for i in range(1, n_components + 1)]\n",
+ " probs_names = [f\"-probs-{i}\" for i in range(1, n_components + 1)]\n",
+ " if weighted:\n",
+ " weight_names = [f\"-weight-{i}\" for i in range(1, n_components + 1)]\n",
+ " self.param_names = [\n",
+ " i for j in zip(total_count_names, probs_names, weight_names) for i in j\n",
+ " ]\n",
+ " else:\n",
+ " self.param_names = [i for j in zip(total_count_names, probs_names) for i in j]\n",
+ "\n",
" if self.return_params:\n",
- " total_count_names = [f\"-total_count-{i}\" for i in range(1, n_components + 1)]\n",
- " probs_names = [f\"-probs-{i}\" for i in range(1, n_components + 1)]\n",
- " param_names = [i for j in zip(total_count_names, probs_names) for i in j]\n",
- " self.output_names = self.output_names + param_names\n",
+ " self.output_names = self.output_names + self.param_names\n",
"\n",
" # Add first output entry for the sample_mean\n",
" self.output_names.insert(0, \"\") \n",
"\n",
- " self.outputsize_multiplier = 2 * n_components\n",
+ " self.n_outputs = 2 + weighted\n",
+ " self.n_components = n_components\n",
+ " self.outputsize_multiplier = self.n_outputs * n_components\n",
" self.is_distribution_output = True\n",
+ " self.has_predicted = False\n",
"\n",
" def domain_map(self, output: torch.Tensor):\n",
- " mu, alpha = torch.tensor_split(output, 2, dim=-1)\n",
- " return (mu, alpha)\n",
+ " output = output.reshape(output.shape[0],\n",
+ " output.shape[1],\n",
+ " -1,\n",
+ " self.outputsize_multiplier)\n",
+ " \n",
+ " return torch.tensor_split(output, self.n_outputs, dim=-1)\n",
"\n",
" def scale_decouple(self, \n",
" output,\n",
@@ -3439,11 +3486,18 @@
" Also adds domain protection to the distribution parameters.\n",
" \"\"\"\n",
" # Efficient NBinomial parametrization\n",
- " mu, alpha = output\n",
+ " if self.weighted:\n",
+ " mu, alpha, weights = output\n",
+ " weights = F.softmax(weights, dim=-1)\n",
+ " else:\n",
+ " mu, alpha = output\n",
+ "\n",
" mu = F.softplus(mu) + 1e-8\n",
" alpha = F.softplus(alpha) + 1e-8 # alpha = 1/total_counts\n",
" if (loc is not None) and (scale is not None):\n",
- " loc = loc.view(mu.size(dim=0), 1, -1)\n",
+ " if loc.ndim == 3:\n",
+ " loc = loc.unsqueeze(-1)\n",
+ " scale = scale.unsqueeze(-1) \n",
" mu *= loc\n",
" alpha /= (loc + 1.)\n",
"\n",
@@ -3452,20 +3506,48 @@
" # => probs = mu / [total_count * (1 + mu * (1/total_count))]\n",
" total_count = 1.0 / alpha\n",
" probs = (mu * alpha / (1.0 + mu * alpha)) + 1e-8 \n",
- " return (total_count, probs)\n",
+ " if self.weighted:\n",
+ " return (total_count, probs, weights)\n",
+ " else:\n",
+ " return (total_count, probs)\n",
+ "\n",
+ " def get_distribution(self, distr_args) -> Distribution:\n",
+ " \"\"\"\n",
+ " Construct the associated Pytorch Distribution, given the collection of\n",
+ " constructor arguments and, optionally, location and scale tensors.\n",
+ "\n",
+ " **Parameters**
\n",
+ " `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
+ "\n",
+ " **Returns**
\n",
+ " `Distribution`: AffineTransformed distribution.
\n",
+ " \"\"\"\n",
+ " if self.weighted:\n",
+ " total_count, probs, weights = distr_args\n",
+ " else:\n",
+ " total_count, probs = distr_args\n",
+ " weights = torch.full_like(total_count, fill_value=1 / self.n_components)\n",
+ "\n",
+ " mix = Categorical(weights)\n",
+ " components = NegativeBinomial(total_count, probs)\n",
+ " components.support = constraints.nonnegative\n",
+ " distr = MixtureSameFamily(mixture_distribution=mix,\n",
+ " component_distribution=components) \n",
"\n",
- " def sample(self, distr_args, num_samples=None):\n",
+ " self.distr_mean = distr.mean\n",
+ " \n",
+ " return distr\n",
+ "\n",
+ " def sample(self,\n",
+ " distr_args: torch.Tensor,\n",
+ " num_samples: Optional[int] = None):\n",
" \"\"\"\n",
" Construct the empirical quantiles from the estimated Distribution,\n",
" sampling from it `num_samples` independently.\n",
"\n",
" **Parameters**
\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
- " `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
- " of the resulting distribution.
\n",
- " `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
- " of the resulting distribution.
\n",
- " `num_samples`: int=500, number of samples for the empirical quantiles.
\n",
+ " `num_samples`: int, overwrite number of samples for the empirical quantiles.
\n",
"\n",
" **Returns**
\n",
" `samples`: tensor, shape [B,H,`num_samples`].
\n",
@@ -3473,97 +3555,59 @@
" \"\"\"\n",
" if num_samples is None:\n",
" num_samples = self.num_samples\n",
- " \n",
- " total_count, probs = distr_args\n",
- " B, H, K = total_count.size()\n",
- " Q = len(self.quantiles)\n",
- " assert total_count.shape == probs.shape\n",
- "\n",
- " # Sample K ~ Mult(weights)\n",
- " # shared across B, H\n",
- " # weights = torch.repeat_interleave(input=weights, repeats=H, dim=2)\n",
- " \n",
- " weights = (1/K) * torch.ones_like(probs, device=probs.device)\n",
- " \n",
- " # Avoid loop, vectorize\n",
- " weights = weights.reshape(-1, K)\n",
- " total_count = total_count.flatten()\n",
- " probs = probs.flatten()\n",
- "\n",
- " # Vectorization trick to recover row_idx\n",
- " sample_idxs = torch.multinomial(input=weights, \n",
- " num_samples=num_samples,\n",
- " replacement=True)\n",
- " aux_col_idx = torch.unsqueeze(torch.arange(B * H, device=probs.device),-1) * K\n",
- "\n",
- " # To device\n",
- " sample_idxs = sample_idxs.to(probs.device)\n",
- "\n",
- " sample_idxs = sample_idxs + aux_col_idx\n",
- " sample_idxs = sample_idxs.flatten()\n",
"\n",
- " sample_total_count = total_count[sample_idxs]\n",
- " sample_probs = probs[sample_idxs]\n",
+ " # Instantiate Scaled Decoupled Distribution\n",
+ " distr = self.get_distribution(distr_args=distr_args)\n",
+ " samples = distr.sample(sample_shape=(num_samples,))\n",
+ " samples = samples.permute(1, 2, 3, 0) # [samples, B, H, N] -> [B, H, N, samples]\n",
"\n",
- " # Sample y ~ NBinomial(total_count, probs) independently\n",
- " dist = NegativeBinomial(total_count=sample_total_count, \n",
- " probs=sample_probs)\n",
- " samples = dist.sample(sample_shape=(1,)).to(probs.device)[0]\n",
- " samples = samples.view(B*H, num_samples)\n",
- " sample_mean = torch.mean(samples, dim=-1)\n",
+ " sample_mean = torch.mean(samples, dim=-1, keepdim=True) \n",
"\n",
" # Compute quantiles\n",
- " quantiles_device = self.quantiles.to(probs.device)\n",
- " quants = torch.quantile(input=samples, q=quantiles_device, dim=1)\n",
- " quants = quants.permute((1,0)) # Q, B*H\n",
- "\n",
- " # Final reshapes\n",
- " samples = samples.view(B, H, num_samples)\n",
- " sample_mean = sample_mean.view(B, H, 1)\n",
- " quants = quants.view(B, H, Q)\n",
+ " quantiles_device = self.quantiles.to(distr_args[0].device)\n",
+ " quants = torch.quantile(input=samples, \n",
+ " q=quantiles_device, \n",
+ " dim=-1)\n",
+ " quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]\n",
"\n",
" return samples, sample_mean, quants\n",
"\n",
- " def neglog_likelihood(self,\n",
- " y: torch.Tensor,\n",
- " distr_args: Tuple[torch.Tensor, torch.Tensor],\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " def update_quantile(self, q: Optional[List[float]] = None):\n",
+ " if q is not None:\n",
+ " self.quantiles = nn.Parameter(torch.tensor(q, dtype=torch.float32), requires_grad=False)\n",
+ " self.output_names = [\"\"] + [f\"_ql{q_i}\" for q_i in q] + self.return_params * self.param_names\n",
+ " self.has_predicted = True\n",
+ " elif q is None and self.has_predicted:\n",
+ " self.quantiles = nn.Parameter(torch.tensor([0.5], dtype=torch.float32), requires_grad=False)\n",
+ " self.output_names = [\"\", \"-median\"] + self.return_params * self.param_names\n",
"\n",
- " if mask is None: \n",
- " mask = torch.ones_like(y)\n",
- " \n",
- " total_count, probs = distr_args\n",
- " B, H, K = total_count.size()\n",
- " \n",
- " weights = (1/K) * torch.ones_like(probs, device=probs.device)\n",
- " \n",
- " y = y[:,:, None]\n",
- " mask = mask[:,:,None]\n",
- "\n",
- " log_unnormalized_prob = (total_count * torch.log(1.-probs) + y * torch.log(probs))\n",
- " log_normalization = (-torch.lgamma(total_count + y) + torch.lgamma(1. + y) +\n",
- " torch.lgamma(total_count))\n",
- " log_normalization[total_count + y == 0.] = 0.\n",
- " log = log_unnormalized_prob - log_normalization\n",
- "\n",
- " #log = torch.sum(log, dim=0, keepdim=True) # Joint within batch/group\n",
- " #log = torch.sum(log, dim=1, keepdim=True) # Joint within horizon\n",
- "\n",
- " # Numerical stability mixture and loglik\n",
- " log_max = torch.amax(log, dim=2, keepdim=True) # [1,1,K] (collapsed joints)\n",
- " lik = weights * torch.exp(log-log_max) # Take max\n",
- " loglik = torch.log(torch.sum(lik, dim=2, keepdim=True)) + log_max # Return max\n",
- " \n",
- " loglik = loglik * mask #replace with mask\n",
+ " def __call__(self,\n",
+ " y: torch.Tensor,\n",
+ " distr_args: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None):\n",
+ " \"\"\"\n",
+ " Computes the negative log-likelihood objective function. \n",
+ " To estimate the following predictive distribution:\n",
"\n",
- " loss = -torch.mean(loglik)\n",
- " return loss\n",
- " \n",
- " def __call__(self, y: torch.Tensor,\n",
- " distr_args: Tuple[torch.Tensor, torch.Tensor],\n",
- " mask: Union[torch.Tensor, None] = None,):\n",
+ " $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
+ "\n",
+ " where $\\\\theta$ represents the distributions parameters. It aditionally \n",
+ " summarizes the objective signal using a weighted average using the `mask` tensor. \n",
"\n",
- " return self.neglog_likelihood(y=y, distr_args=distr_args, mask=mask)"
+ " **Parameters**
\n",
+ " `y`: tensor, Actual values.
\n",
+ " `distr_args`: Constructor arguments for the underlying Distribution type.
\n",
+ " `mask`: tensor, Specifies date stamps per serie to consider in loss.
\n",
+ "\n",
+ " **Returns**
\n",
+ " `loss`: scalar, weighted loss function against which backpropagation will be performed.
\n",
+ " \"\"\"\n",
+ " # Instantiate Scaled Decoupled Distribution\n",
+ " distr = self.get_distribution(distr_args=distr_args)\n",
+ " loss_values = -distr.log_prob(y)\n",
+ " loss_weights = mask\n",
+ " \n",
+ " return weighted_average(loss_values, weights=loss_weights)"
]
},
{
@@ -3604,8 +3648,8 @@
"outputs": [],
"source": [
"#| hide\n",
- "# Create single mixture and broadcast to N,H,K\n",
- "counts = torch.Tensor([[10,20,30], [20,40,60]])[None, :, :]\n",
+ "# Create single mixture and broadcast to N,H,1,K\n",
+ "counts = torch.Tensor([[5,10,15], [10,20,30]])[None, :, :].unsqueeze(2)\n",
"\n",
"# # Create repetitions for the batch dimension N.\n",
"N=2\n",
@@ -3613,22 +3657,22 @@
"weights = torch.ones_like(counts)\n",
"probs = torch.ones_like(counts) * 0.5\n",
"\n",
- "print('weights.shape (N,H,K) \\t', weights.shape)\n",
- "print('counts.shape (N,H,K) \\t', counts.shape)\n",
- "print('probs.shape (N,H,K) \\t', probs.shape)\n",
+ "print('weights.shape (N,H,1,K) \\t', weights.shape)\n",
+ "print('counts.shape (N,H,1,K) \\t', counts.shape)\n",
+ "print('probs.shape (N,H,1,K) \\t', probs.shape)\n",
"\n",
- "model = NBMM(quantiles=[0.1, 0.40, 0.5, 0.60, 0.9])\n",
- "distr_args = (counts, probs)\n",
+ "model = NBMM(quantiles=[0.1, 0.40, 0.5, 0.60, 0.9], weighted=True)\n",
+ "distr_args = (counts, probs, weights)\n",
"samples, sample_mean, quants = model.sample(distr_args, num_samples=2000)\n",
"\n",
- "print('samples.shape (N,H,num_samples) ', samples.shape)\n",
- "print('sample_mean.shape (N,H) ', sample_mean.shape)\n",
- "print('quants.shape (N,H,Q) \\t\\t', quants.shape)\n",
+ "print('samples.shape (N,H,1,num_samples) ', samples.shape)\n",
+ "print('sample_mean.shape (N,H,1,1) ', sample_mean.shape)\n",
+ "print('quants.shape (N,H,1,Q) \\t\\t', quants.shape)\n",
"\n",
"# Plot synthethic data\n",
"x_plot = range(quants.shape[1]) # H length\n",
- "y_plot_hat = quants[0,:,:] # Filter N,G,T -> H,Q\n",
- "samples_hat = samples[0,:,:] # Filter N,G,T -> H,num_samples\n",
+ "y_plot_hat = quants[0,:,0,:] # Filter N,G,T -> H,Q\n",
+ "samples_hat = samples[0,:,0,:] # Filter N,G,T -> H,num_samples\n",
"\n",
"# Kernel density plot for single forecast horizon \\tau = t+1\n",
"fig, ax = plt.subplots(figsize=(3.7, 2.9))\n",
@@ -3723,7 +3767,9 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -3784,7 +3830,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class TukeyLoss(torch.nn.Module):\n",
+ "class TukeyLoss(BasePointLoss):\n",
" \"\"\" Tukey Loss\n",
"\n",
" The Tukey loss function, also known as Tukey's biweight function, is a \n",
@@ -3823,10 +3869,14 @@
"\n",
" def domain_map(self, y_hat: torch.Tensor):\n",
" \"\"\"\n",
- " Univariate loss operates in dimension [B,T,H]/[B,H]\n",
- " This changes the network's output from [B,H,1]->[B,H]\n",
+ " Input:\n",
+ " Univariate: [B, H, 1]\n",
+ " Multivariate: [B, H, N]\n",
+ "\n",
+ " Output: [B, H, N]\n",
" \"\"\"\n",
- " return y_hat.squeeze(-1)\n",
+ "\n",
+ " return y_hat\n",
"\n",
" def masked_mean(self, x, mask, dim):\n",
" x_nan = x.masked_fill(mask < 1, float(\"nan\"))\n",
@@ -3834,8 +3884,12 @@
" x_mean = torch.nan_to_num(x_mean, nan=0.0)\n",
" return x_mean\n",
"\n",
- " def __call__(self, y: torch.Tensor, y_hat: torch.Tensor, \n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " def __call__(self,\n",
+ " y: torch.Tensor,\n",
+ " y_hat: torch.Tensor,\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -3942,7 +3996,9 @@
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -3952,6 +4008,7 @@
" **Returns:**
\n",
" `huber_qloss`: tensor (single value).\n",
" \"\"\"\n",
+ " \n",
" error = y_hat - y\n",
" zero_error = torch.zeros_like(error)\n",
" sq = torch.maximum(-error, zero_error)\n",
@@ -4051,9 +4108,18 @@
"\n",
" def domain_map(self, y_hat: torch.Tensor):\n",
" \"\"\"\n",
- " Identity domain map [B,T,H,Q]/[B,H,Q]\n",
+ " Input:\n",
+ " Univariate: [B, H, 1 * Q]\n",
+ " Multivariate: [B, H, N * Q]\n",
+ "\n",
+ " Output: [B, H, N, Q]\n",
" \"\"\"\n",
- " return y_hat\n",
+ " output = y_hat.reshape(y_hat.shape[0],\n",
+ " y_hat.shape[1],\n",
+ " -1,\n",
+ " self.outputsize_multiplier)\n",
+ "\n",
+ " return output\n",
" \n",
" def _compute_weights(self, y, mask):\n",
" \"\"\"\n",
@@ -4061,25 +4127,24 @@
" Set horizon_weight to a ones[H] tensor if not set.\n",
" If set, check that it has the same length as the horizon in x.\n",
" \"\"\"\n",
- " if mask is None:\n",
- " mask = torch.ones_like(y, device=y.device)\n",
- " else:\n",
- " mask = mask.unsqueeze(1) # Add Q dimension.\n",
"\n",
" if self.horizon_weight is None:\n",
- " self.horizon_weight = torch.ones(mask.shape[-1])\n",
+ " weights = torch.ones_like(mask)\n",
" else:\n",
- " assert mask.shape[-1] == len(self.horizon_weight), \\\n",
- " 'horizon_weight must have same length as Y'\n",
- " \n",
- " weights = self.horizon_weight.clone()\n",
- " weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)\n",
+ " assert mask.shape[1] == len(self.horizon_weight), \\\n",
+ " 'horizon_weight must have same length as Y' \n",
+ " weights = self.horizon_weight.clone()\n",
+ " weights = weights[None, :, None, None].to(mask.device)\n",
+ " weights = torch.ones_like(mask, device=mask.device) * weights\n",
+ " \n",
" return weights * mask\n",
"\n",
" def __call__(self,\n",
" y: torch.Tensor,\n",
" y_hat: torch.Tensor,\n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -4089,25 +4154,27 @@
" **Returns:**
\n",
" `hmqloss`: tensor (single value).\n",
" \"\"\"\n",
- "\n",
- " error = y_hat - y.unsqueeze(-1)\n",
+ " y = y.unsqueeze(-1)\n",
+ " \n",
+ " if mask is not None:\n",
+ " mask = mask.unsqueeze(-1)\n",
+ " else:\n",
+ " mask = torch.ones_like(y, device=y.device)\n",
+ " \n",
+ " error = y_hat - y\n",
+ " \n",
" zero_error = torch.zeros_like(error) \n",
" sq = torch.maximum(-error, torch.zeros_like(error))\n",
" s1_q = torch.maximum(error, torch.zeros_like(error))\n",
- " losses = F.huber_loss(self.quantiles * sq, zero_error, \n",
+ " \n",
+ " quantiles = self.quantiles[None, None, None, :]\n",
+ " losses = F.huber_loss(quantiles * sq, zero_error, \n",
" reduction='none', delta=self.delta) + \\\n",
- " F.huber_loss((1 - self.quantiles) * s1_q, zero_error, \n",
+ " F.huber_loss((1 - quantiles) * s1_q, zero_error, \n",
" reduction='none', delta=self.delta)\n",
- " losses = (1/len(self.quantiles)) * losses\n",
+ " losses = (1 / len(quantiles)) * losses\n",
"\n",
- " if y_hat.ndim == 3: # BaseWindows\n",
- " losses = losses.swapaxes(-2,-1) # [B,H,Q] -> [B,Q,H] (needed for horizon weighting, H at the end)\n",
- " elif y_hat.ndim == 4: # BaseRecurrent\n",
- " losses = losses.swapaxes(-2,-1)\n",
- " losses = losses.swapaxes(-2,-3) # [B,seq_len,H,Q] -> [B,Q,seq_len,H] (needed for horizon weighting, H at the end)\n",
- "\n",
- " weights = self._compute_weights(y=losses, mask=mask) # Use losses for extra dim\n",
- " # NOTE: Weights do not have Q dimension.\n",
+ " weights = self._compute_weights(y=losses, mask=mask) \n",
"\n",
" return _weighted_mean(losses=losses, weights=weights)"
]
@@ -4167,7 +4234,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class Accuracy(torch.nn.Module):\n",
+ "class Accuracy(BasePointLoss):\n",
" \"\"\" Accuracy\n",
"\n",
" Computes the accuracy between categorical `y` and `y_hat`.\n",
@@ -4180,16 +4247,25 @@
" def __init__(self,):\n",
" super(Accuracy, self).__init__()\n",
" self.is_distribution_output = False\n",
+ " self.outputsize_multiplier = 1\n",
"\n",
" def domain_map(self, y_hat: torch.Tensor):\n",
" \"\"\"\n",
- " Univariate loss operates in dimension [B,T,H]/[B,H]\n",
- " This changes the network's output from [B,H,1]->[B,H]\n",
+ " Input:\n",
+ " Univariate: [B, H, 1]\n",
+ " Multivariate: [B, H, N]\n",
+ "\n",
+ " Output: [B, H, N]\n",
" \"\"\"\n",
- " return y_hat.squeeze(-1)\n",
"\n",
- " def __call__(self, y: torch.Tensor, y_hat: torch.Tensor, \n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " return y_hat\n",
+ " \n",
+ " def __call__(self,\n",
+ " y: torch.Tensor,\n",
+ " y_hat: torch.Tensor,\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -4199,10 +4275,11 @@
" **Returns:**
\n",
" `accuracy`: tensor (single value).\n",
" \"\"\"\n",
+ "\n",
" if mask is None:\n",
" mask = torch.ones_like(y_hat)\n",
"\n",
- " measure = (y.unsqueeze(-1) == y_hat) * mask.unsqueeze(-1)\n",
+ " measure = (y == y_hat) * mask\n",
" accuracy = torch.mean(measure)\n",
" return accuracy"
]
@@ -4244,7 +4321,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class sCRPS(torch.nn.Module):\n",
+ "class sCRPS(BasePointLoss):\n",
" \"\"\"Scaled Continues Ranked Probability Score\n",
"\n",
" Calculates a scaled variation of the CRPS, as proposed by Rangapuram (2021),\n",
@@ -4279,8 +4356,12 @@
" self.mql = MQLoss(level=level, quantiles=quantiles)\n",
" self.is_distribution_output = False\n",
" \n",
- " def __call__(self, y: torch.Tensor, y_hat: torch.Tensor, \n",
- " mask: Union[torch.Tensor, None] = None):\n",
+ " def __call__(self,\n",
+ " y: torch.Tensor,\n",
+ " y_hat: torch.Tensor,\n",
+ " y_insample: torch.Tensor,\n",
+ " mask: Union[torch.Tensor, None] = None,\n",
+ " ) -> torch.Tensor:\n",
" \"\"\"\n",
" **Parameters:**
\n",
" `y`: tensor, Actual values.
\n",
@@ -4290,7 +4371,7 @@
" **Returns:**
\n",
" `scrps`: tensor (single value).\n",
" \"\"\"\n",
- " mql = self.mql(y=y, y_hat=y_hat, mask=mask)\n",
+ " mql = self.mql(y=y, y_hat=y_hat, mask=mask, y_insample=y_insample)\n",
" norm = torch.sum(torch.abs(y))\n",
" unmean = torch.sum(mask)\n",
" scrps = 2 * mql * unmean / (norm + 1e-5)\n",
@@ -4326,11 +4407,11 @@
"source": [
"#| hide\n",
"# Each 1 is an error, there are 6 datapoints.\n",
- "y = torch.Tensor([[0,0,0],[0,0,0]])\n",
- "y_hat = torch.Tensor([[0,0,1],[1,0,1]])\n",
+ "y = torch.Tensor([[0,0,0],[0,0,0]]).unsqueeze(-1)\n",
+ "y_hat = torch.Tensor([[0,0,1],[1,0,1]]).unsqueeze(-1)\n",
"\n",
"# Complete mask and horizon_weight\n",
- "mask = torch.Tensor([[1,1,1],[1,1,1]])\n",
+ "mask = torch.Tensor([[1,1,1],[1,1,1]]).unsqueeze(-1)\n",
"horizon_weight = torch.Tensor([1,1,1])\n",
"\n",
"mae = MAE(horizon_weight=horizon_weight)\n",
@@ -4338,21 +4419,21 @@
"assert loss==(3/6), 'Should be 3/6'\n",
"\n",
"# Incomplete mask and complete horizon_weight\n",
- "mask = torch.Tensor([[1,1,1],[0,1,1]]) # Only 1 error and points is masked.\n",
+ "mask = torch.Tensor([[1,1,1],[0,1,1]]).unsqueeze(-1) # Only 1 error and points is masked.\n",
"horizon_weight = torch.Tensor([1,1,1])\n",
"mae = MAE(horizon_weight=horizon_weight)\n",
"loss = mae(y=y, y_hat=y_hat, mask=mask)\n",
"assert loss==(2/5), 'Should be 2/5'\n",
"\n",
"# Complete mask and incomplete horizon_weight\n",
- "mask = torch.Tensor([[1,1,1],[1,1,1]])\n",
+ "mask = torch.Tensor([[1,1,1],[1,1,1]]).unsqueeze(-1)\n",
"horizon_weight = torch.Tensor([1,1,0]) # 2 errors and points are masked.\n",
"mae = MAE(horizon_weight=horizon_weight)\n",
"loss = mae(y=y, y_hat=y_hat, mask=mask)\n",
"assert loss==(1/4), 'Should be 1/4'\n",
"\n",
"# Incomplete mask and incomplete horizon_weight\n",
- "mask = torch.Tensor([[0,1,1],[1,1,1]])\n",
+ "mask = torch.Tensor([[0,1,1],[1,1,1]]).unsqueeze(-1)\n",
"horizon_weight = torch.Tensor([1,1,0]) # 2 errors are masked, and 3 points.\n",
"mae = MAE(horizon_weight=horizon_weight)\n",
"loss = mae(y=y, y_hat=y_hat, mask=mask)\n",
diff --git a/nbs/models.autoformer.ipynb b/nbs/models.autoformer.ipynb
index 9c6567f2e..999c4ca62 100644
--- a/nbs/models.autoformer.ipynb
+++ b/nbs/models.autoformer.ipynb
@@ -68,7 +68,7 @@
"import torch.nn.functional as F\n",
"\n",
"from neuralforecast.common._modules import DataEmbedding, SeriesDecomp\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -80,8 +80,12 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
+ "\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -410,7 +414,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class Autoformer(BaseWindows):\n",
+ "class Autoformer(BaseModel):\n",
" \"\"\" Autoformer\n",
"\n",
" The Autoformer model tackles the challenge of finding reliable dependencies on intricate temporal patterns of long-horizon forecasting.\n",
@@ -469,10 +473,11 @@
"\t- [Wu, Haixu, Jiehui Xu, Jianmin Wang, and Mingsheng Long. \"Autoformer: Decomposition transformers with auto-correlation for long-term series forecasting\"](https://proceedings.neurips.cc/paper/2021/hash/bcc0d400288793e8bdcd7c19a8ac0c2b-Abstract.html)
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -616,13 +621,9 @@
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
" insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
" futr_exog = windows_batch['futr_exog']\n",
"\n",
" # Parse inputs\n",
- " insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]\n",
" if self.futr_exog_size > 0:\n",
" x_mark_enc = futr_exog[:,:self.input_size,:]\n",
" x_mark_dec = futr_exog[:,-(self.label_len+self.h):,:]\n",
@@ -650,7 +651,8 @@
" # final\n",
" dec_out = trend_part + seasonal_part\n",
"\n",
- " forecast = self.loss.domain_map(dec_out[:, -self.h:])\n",
+ " forecast = dec_out[:, -self.h:]\n",
+ " \n",
" return forecast"
]
},
@@ -681,6 +683,21 @@
"show_doc(Autoformer.predict, name='Autoformer.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(Autoformer, [\"airpassengers\"])"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
diff --git a/nbs/models.bitcn.ipynb b/nbs/models.bitcn.ipynb
index cd78bb194..b7363dba4 100644
--- a/nbs/models.bitcn.ipynb
+++ b/nbs/models.bitcn.ipynb
@@ -55,8 +55,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -74,7 +77,7 @@
"import numpy as np\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_windows import BaseWindows"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -146,7 +149,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class BiTCN(BaseWindows):\n",
+ "class BiTCN(BaseModel):\n",
" \"\"\" BiTCN\n",
"\n",
" Bidirectional Temporal Convolutional Network (BiTCN) is a forecasting architecture based on two temporal convolutional networks (TCNs). The first network ('forward') encodes future covariates of the time series, whereas the second network ('backward') encodes past observations and covariates. This is a univariate model.\n",
@@ -170,7 +173,7 @@
" `batch_size`: int=32, number of different series in each batch.
\n",
" `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
" `windows_batch_size`: int=1024, number of windows to sample in each training batch, default uses all.
\n",
- " `inference_windows_batch_size`: int=-1, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `inference_windows_batch_size`: int=1024, number of windows to sample in each inference batch, -1 uses all.
\n",
" `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
@@ -190,10 +193,11 @@
"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int,\n",
@@ -315,7 +319,7 @@
"\n",
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
- " x = windows_batch['insample_y'].unsqueeze(-1) # [B, L, 1]\n",
+ " x = windows_batch['insample_y'].contiguous() # [B, L, 1]\n",
" hist_exog = windows_batch['hist_exog'] # [B, L, X]\n",
" futr_exog = windows_batch['futr_exog'] # [B, L + h, F]\n",
" stat_exog = windows_batch['stat_exog'] # [B, S]\n",
@@ -358,11 +362,8 @@
"\n",
" # Output layer to create forecasts\n",
" x = x.permute(0, 2, 1) # [B, 3 * hidden_size, h] -> [B, h, 3 * hidden_size]\n",
- " x = self.output_lin(x) # [B, h, 3 * hidden_size] -> [B, h, n_outputs] \n",
+ " forecast = self.output_lin(x) # [B, h, 3 * hidden_size] -> [B, h, n_outputs] \n",
"\n",
- " # Map to output domain\n",
- " forecast = self.loss.domain_map(x)\n",
- " \n",
" return forecast"
]
},
@@ -393,6 +394,21 @@
"show_doc(BiTCN.predict, name='BiTCN.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(BiTCN, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -411,8 +427,8 @@
"import matplotlib.pyplot as plt\n",
"\n",
"from neuralforecast import NeuralForecast\n",
- "from neuralforecast.models import BiTCN\n",
"from neuralforecast.losses.pytorch import GMM\n",
+ "from neuralforecast.models import BiTCN\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds\n",
@@ -196,10 +195,11 @@
"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False\n",
+ " RECURRENT = True\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -215,7 +215,7 @@
" stat_exog_list = None,\n",
" exclude_insample_y = False,\n",
" loss = DistributionLoss(distribution='StudentT', level=[80, 90], return_params=False),\n",
- " valid_loss = MQLoss(level=[80, 90]),\n",
+ " valid_loss = MAE(),\n",
" max_steps: int = 1000,\n",
" learning_rate: float = 1e-3,\n",
" num_lr_decays: int = 3,\n",
@@ -241,15 +241,6 @@
" if exclude_insample_y:\n",
" raise Exception('DeepAR has no possibility for excluding y.')\n",
" \n",
- " if not loss.is_distribution_output:\n",
- " raise Exception('DeepAR only supports distributional outputs.')\n",
- " \n",
- " if str(type(valid_loss)) not in [\"\"]:\n",
- " raise Exception('DeepAR only supports MQLoss as validation loss.')\n",
- "\n",
- " if loss.return_params:\n",
- " raise Exception('DeepAR does not return distribution parameters due to Monte Carlo sampling.')\n",
- " \n",
" # Inherit BaseWindows class\n",
" super(DeepAR, self).__init__(h=h,\n",
" input_size=input_size,\n",
@@ -281,8 +272,7 @@
" dataloader_kwargs=dataloader_kwargs,\n",
" **trainer_kwargs)\n",
"\n",
- " self.horizon_backup = self.h # Used because h=0 during training\n",
- " self.trajectory_samples = trajectory_samples\n",
+ " self.n_samples = trajectory_samples\n",
"\n",
" # LSTM\n",
" self.encoder_n_layers = lstm_n_layers\n",
@@ -293,6 +283,8 @@
" input_encoder = 1 + self.futr_exog_size + self.stat_exog_size\n",
"\n",
" # Instantiate model\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = False\n",
" self.hist_encoder = nn.LSTM(input_size=input_encoder,\n",
" hidden_size=self.encoder_hidden_size,\n",
" num_layers=self.encoder_n_layers,\n",
@@ -305,268 +297,38 @@
" hidden_size=decoder_hidden_size,\n",
" hidden_layers=decoder_hidden_layers)\n",
"\n",
- " # Override BaseWindows method\n",
- " def training_step(self, batch, batch_idx):\n",
- "\n",
- " # During training h=0 \n",
- " self.h = 0\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Create and normalize windows [Ws, L, C]\n",
- " windows = self._create_windows(batch, step='train')\n",
- " original_insample_y = windows['temporal'][:, :, y_idx].clone() # windows: [B, L, Feature] -> [B, L]\n",
- " original_insample_y = original_insample_y[:,1:] # Remove first (shift in DeepAr, cell at t outputs t+1)\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, _, _, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- "\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L]\n",
- " insample_mask=insample_mask, # [Ws, L]\n",
- " futr_exog=futr_exog, # [Ws, L+H]\n",
- " hist_exog=None, # None\n",
- " stat_exog=stat_exog,\n",
- " y_idx=y_idx) # [Ws, 1]\n",
- "\n",
- " # Model Predictions\n",
- " output = self.train_forward(windows_batch)\n",
- "\n",
- " if self.loss.is_distribution_output:\n",
- " _, y_loc, y_scale = self._inv_normalization(y_hat=original_insample_y,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " outsample_y = original_insample_y\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " mask = insample_mask[:,1:].clone() # Remove first (shift in DeepAr, cell at t outputs t+1)\n",
- " loss = self.loss(y=outsample_y, distr_args=distr_args, mask=mask)\n",
- " else:\n",
- " raise Exception('DeepAR only supports distributional outputs.')\n",
- "\n",
- " if torch.isnan(loss):\n",
- " print('Model Parameters', self.hparams)\n",
- " print('insample_y', torch.isnan(insample_y).sum())\n",
- " print('outsample_y', torch.isnan(outsample_y).sum())\n",
- " print('output', torch.isnan(output).sum())\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'train_loss',\n",
- " loss.item(),\n",
- " batch_size=outsample_y.size(0),\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.train_trajectories.append((self.global_step, loss.item()))\n",
- "\n",
- " self.h = self.horizon_backup # Restore horizon\n",
- " return loss\n",
- "\n",
- " def validation_step(self, batch, batch_idx):\n",
- "\n",
- " self.h == self.horizon_backup\n",
- "\n",
- " if self.val_size == 0:\n",
- " return np.nan\n",
- "\n",
- " # TODO: Hack to compute number of windows\n",
- " windows = self._create_windows(batch, step='val')\n",
- " n_windows = len(windows['temporal'])\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Number of windows in batch\n",
- " windows_batch_size = self.inference_windows_batch_size\n",
- " if windows_batch_size < 0:\n",
- " windows_batch_size = n_windows\n",
- " n_batches = int(np.ceil(n_windows/windows_batch_size))\n",
- "\n",
- " valid_losses = []\n",
- " batch_sizes = []\n",
- " for i in range(n_batches):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " w_idxs = np.arange(i*windows_batch_size, \n",
- " min((i+1)*windows_batch_size, n_windows))\n",
- " windows = self._create_windows(batch, step='val', w_idxs=w_idxs)\n",
- " original_outsample_y = torch.clone(windows['temporal'][:,-self.h:,0])\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, outsample_mask, \\\n",
- " _, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- " windows_batch = dict(insample_y=insample_y,\n",
- " insample_mask=insample_mask,\n",
- " futr_exog=futr_exog,\n",
- " hist_exog=None,\n",
- " stat_exog=stat_exog,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx) \n",
- " \n",
- " # Model Predictions\n",
- " output_batch = self(windows_batch)\n",
- " # Monte Carlo already returns y_hat with mean and quantiles\n",
- " output_batch = output_batch[:,:, 1:] # Remove mean\n",
- " valid_loss_batch = self.valid_loss(y=original_outsample_y, y_hat=output_batch, mask=outsample_mask)\n",
- " valid_losses.append(valid_loss_batch)\n",
- " batch_sizes.append(len(output_batch))\n",
- "\n",
- " valid_loss = torch.stack(valid_losses)\n",
- " batch_sizes = torch.tensor(batch_sizes, device=valid_loss.device)\n",
- " batch_size = torch.sum(batch_sizes)\n",
- " valid_loss = torch.sum(valid_loss * batch_sizes) / batch_size\n",
- "\n",
- " if torch.isnan(valid_loss):\n",
- " raise Exception('Loss is NaN, training stopped.')\n",
- "\n",
- " self.log(\n",
- " 'valid_loss',\n",
- " valid_loss.item(),\n",
- " batch_size=batch_size,\n",
- " prog_bar=True,\n",
- " on_epoch=True,\n",
- " )\n",
- " self.validation_step_outputs.append(valid_loss)\n",
- " return valid_loss\n",
- "\n",
- " def predict_step(self, batch, batch_idx):\n",
- "\n",
- " self.h == self.horizon_backup\n",
- "\n",
- " # TODO: Hack to compute number of windows\n",
- " windows = self._create_windows(batch, step='predict')\n",
- " n_windows = len(windows['temporal'])\n",
- " y_idx = batch['y_idx']\n",
- "\n",
- " # Number of windows in batch\n",
- " windows_batch_size = self.inference_windows_batch_size\n",
- " if windows_batch_size < 0:\n",
- " windows_batch_size = n_windows\n",
- " n_batches = int(np.ceil(n_windows/windows_batch_size))\n",
- "\n",
- " y_hats = []\n",
- " for i in range(n_batches):\n",
- " # Create and normalize windows [Ws, L+H, C]\n",
- " w_idxs = np.arange(i*windows_batch_size, \n",
- " min((i+1)*windows_batch_size, n_windows))\n",
- " windows = self._create_windows(batch, step='predict', w_idxs=w_idxs)\n",
- " windows = self._normalization(windows=windows, y_idx=y_idx)\n",
- "\n",
- " # Parse windows\n",
- " insample_y, insample_mask, _, _, _, futr_exog, stat_exog = self._parse_windows(batch, windows)\n",
- " windows_batch = dict(insample_y=insample_y, # [Ws, L]\n",
- " insample_mask=insample_mask, # [Ws, L]\n",
- " futr_exog=futr_exog, # [Ws, L+H]\n",
- " stat_exog=stat_exog,\n",
- " temporal_cols=batch['temporal_cols'],\n",
- " y_idx=y_idx)\n",
- " \n",
- " # Model Predictions\n",
- " y_hat = self(windows_batch)\n",
- " # Monte Carlo already returns y_hat with mean and quantiles\n",
- " y_hats.append(y_hat)\n",
- " y_hat = torch.cat(y_hats, dim=0)\n",
- " return y_hat\n",
- "\n",
- " def train_forward(self, windows_batch):\n",
+ " def forward(self, windows_batch):\n",
"\n",
" # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'][:,:, None] # <- [B,T,1]\n",
+ " encoder_input = windows_batch['insample_y'] # <- [B, T, 1]\n",
" futr_exog = windows_batch['futr_exog']\n",
" stat_exog = windows_batch['stat_exog']\n",
"\n",
- " #[B, input_size-1, X]\n",
- " encoder_input = encoder_input[:,:-1,:] # Remove last (shift in DeepAr, cell at t outputs t+1)\n",
" _, input_size = encoder_input.shape[:2]\n",
" if self.futr_exog_size > 0:\n",
- " # Shift futr_exog (t predicts t+1, last output is outside insample_y)\n",
- " encoder_input = torch.cat((encoder_input, futr_exog[:,1:,:]), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, futr_exog), dim=2)\n",
+ "\n",
" if self.stat_exog_size > 0:\n",
- " stat_exog = stat_exog.unsqueeze(1).repeat(1, input_size, 1) # [B, S] -> [B, input_size-1, S]\n",
+ " stat_exog = stat_exog.unsqueeze(1).repeat(1, input_size, 1) # [B, S] -> [B, input_size-1, S]\n",
" encoder_input = torch.cat((encoder_input, stat_exog), dim=2)\n",
"\n",
" # RNN forward\n",
- " hidden_state, _ = self.hist_encoder(encoder_input) # [B, input_size-1, rnn_hidden_state]\n",
+ " if self.maintain_state:\n",
+ " rnn_state = self.rnn_state\n",
+ " else:\n",
+ " rnn_state = None\n",
"\n",
- " # Decoder forward\n",
- " output = self.decoder(hidden_state) # [B, input_size-1, output_size]\n",
- " output = self.loss.domain_map(output)\n",
- " return output\n",
- " \n",
- " def forward(self, windows_batch):\n",
+ " hidden_state, rnn_state = self.hist_encoder(encoder_input, \n",
+ " rnn_state) # [B, input_size-1, rnn_hidden_state]\n",
"\n",
- " # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'][:,:, None] # <- [B,L,1]\n",
- " futr_exog = windows_batch['futr_exog'] # <- [B,L+H, n_f]\n",
- " stat_exog = windows_batch['stat_exog']\n",
- " y_idx = windows_batch['y_idx']\n",
+ " if self.maintain_state:\n",
+ " self.rnn_state = rnn_state\n",
"\n",
- " #[B, seq_len, X]\n",
- " batch_size, input_size = encoder_input.shape[:2]\n",
- " if self.futr_exog_size > 0:\n",
- " futr_exog_input_window = futr_exog[:,1:input_size+1,:] # Align y_t with futr_exog_t+1\n",
- " encoder_input = torch.cat((encoder_input, futr_exog_input_window), dim=2)\n",
- " if self.stat_exog_size > 0:\n",
- " stat_exog_input_window = stat_exog.unsqueeze(1).repeat(1, input_size, 1) # [B, S] -> [B, input_size, S]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog_input_window), dim=2)\n",
- "\n",
- " # Use input_size history to predict first h of the forecasting window\n",
- " _, h_c_tuple = self.hist_encoder(encoder_input)\n",
- " h_n = h_c_tuple[0] # [n_layers, B, lstm_hidden_state]\n",
- " c_n = h_c_tuple[1] # [n_layers, B, lstm_hidden_state]\n",
- "\n",
- " # Vectorizes trajectory samples in batch dimension [1]\n",
- " h_n = torch.repeat_interleave(h_n, self.trajectory_samples, 1) # [n_layers, B*trajectory_samples, rnn_hidden_state]\n",
- " c_n = torch.repeat_interleave(c_n, self.trajectory_samples, 1) # [n_layers, B*trajectory_samples, rnn_hidden_state]\n",
- "\n",
- " # Scales for inverse normalization\n",
- " y_scale = self.scaler.x_scale[:, 0, [y_idx]].squeeze(-1).to(encoder_input.device)\n",
- " y_loc = self.scaler.x_shift[:, 0, [y_idx]].squeeze(-1).to(encoder_input.device)\n",
- " y_scale = torch.repeat_interleave(y_scale, self.trajectory_samples, 0)\n",
- " y_loc = torch.repeat_interleave(y_loc, self.trajectory_samples, 0)\n",
- "\n",
- " # Recursive strategy prediction\n",
- " quantiles = self.loss.quantiles.to(encoder_input.device)\n",
- " y_hat = torch.zeros(batch_size, self.h, len(quantiles)+1, device=encoder_input.device)\n",
- " for tau in range(self.h):\n",
- " # Decoder forward\n",
- " last_layer_h = h_n[-1] # [B*trajectory_samples, lstm_hidden_state]\n",
- " output = self.decoder(last_layer_h) \n",
- " output = self.loss.domain_map(output)\n",
- "\n",
- " # Inverse normalization\n",
- " distr_args = self.loss.scale_decouple(output=output, loc=y_loc, scale=y_scale)\n",
- " # Add horizon (1) dimension\n",
- " distr_args = list(distr_args)\n",
- " for i in range(len(distr_args)):\n",
- " distr_args[i] = distr_args[i].unsqueeze(-1)\n",
- " distr_args = tuple(distr_args)\n",
- " samples_tau, _, _ = self.loss.sample(distr_args=distr_args, num_samples=1)\n",
- " samples_tau = samples_tau.reshape(batch_size, self.trajectory_samples)\n",
- " sample_mean = torch.mean(samples_tau, dim=-1).to(encoder_input.device)\n",
- " quants = torch.quantile(input=samples_tau, \n",
- " q=quantiles, dim=-1).to(encoder_input.device)\n",
- " y_hat[:,tau,0] = sample_mean\n",
- " y_hat[:,tau,1:] = quants.permute((1,0)) # [Q, B] -> [B, Q]\n",
- " \n",
- " # Stop if already in the last step (no need to predict next step)\n",
- " if tau+1 == self.h:\n",
- " continue\n",
- " # Normalize to use as input\n",
- " encoder_input = self.scaler.scaler(samples_tau.flatten(), y_loc, y_scale) # [B*n_samples]\n",
- " encoder_input = encoder_input[:, None, None] # [B*n_samples, 1, 1]\n",
- "\n",
- " # Update input\n",
- " if self.futr_exog_size > 0:\n",
- " futr_exog_tau = futr_exog[:,[input_size+tau+1],:] # [B, 1, n_f]\n",
- " futr_exog_tau = torch.repeat_interleave(futr_exog_tau, self.trajectory_samples, 0) # [B*n_samples, 1, n_f]\n",
- " encoder_input = torch.cat((encoder_input, futr_exog_tau), dim=2) # [B*n_samples, 1, 1+n_f]\n",
- " if self.stat_exog_size > 0:\n",
- " stat_exog_tau = torch.repeat_interleave(stat_exog, self.trajectory_samples, 0) # [B*n_samples, n_s]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog_tau[:,None,:]), dim=2) # [B*n_samples, 1, 1+n_f+n_s]\n",
- " \n",
- " _, h_c_tuple = self.hist_encoder(encoder_input, (h_n, c_n))\n",
- " h_n = h_c_tuple[0] # [n_layers, B, rnn_hidden_state]\n",
- " c_n = h_c_tuple[1] # [n_layers, B, rnn_hidden_state]\n",
- "\n",
- " return y_hat"
+ " # Decoder forward\n",
+ " output = self.decoder(hidden_state) # [B, input_size-1, output_size]\n",
+ "\n",
+ " # Return only horizon part\n",
+ " return output[:, -self.h:]"
]
},
{
@@ -596,6 +358,21 @@
"show_doc(DeepAR.predict, name='DeepAR.predict', title_level=3)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(DeepAR, [\"airpassengers\"])"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -616,18 +393,18 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import DeepAR\n",
- "from neuralforecast.losses.pytorch import DistributionLoss\n",
+ "from neuralforecast.losses.pytorch import DistributionLoss, MQLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
"nf = NeuralForecast(\n",
" models=[DeepAR(h=12,\n",
- " input_size=48,\n",
- " lstm_n_layers=3,\n",
+ " input_size=24,\n",
+ " lstm_n_layers=1,\n",
" trajectory_samples=100,\n",
- " loss=DistributionLoss(distribution='Normal', level=[80, 90], return_params=False),\n",
+ " loss=DistributionLoss(distribution='StudentT', level=[80, 90], return_params=True),\n",
+ " valid_loss=MQLoss(level=[80, 90]),\n",
" learning_rate=0.005,\n",
" stat_exog_list=['airline1'],\n",
" futr_exog_list=['trend'],\n",
@@ -635,7 +412,8 @@
" val_check_steps=10,\n",
" early_stop_patience_steps=-1,\n",
" scaler_type='standard',\n",
- " enable_progress_bar=True),\n",
+ " enable_progress_bar=True,\n",
+ " ),\n",
" ],\n",
" freq='M'\n",
")\n",
diff --git a/nbs/models.deepnpts.ipynb b/nbs/models.deepnpts.ipynb
index 4f5e7ee9f..465dde397 100644
--- a/nbs/models.deepnpts.ipynb
+++ b/nbs/models.deepnpts.ipynb
@@ -51,7 +51,7 @@
"from typing import Optional\n",
"\n",
"\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.losses.pytorch import MAE\n"
]
},
@@ -66,7 +66,8 @@
"import warnings\n",
"\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -77,6 +78,7 @@
"source": [
"#| hide\n",
"logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
"warnings.filterwarnings(\"ignore\")"
]
},
@@ -87,7 +89,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class DeepNPTS(BaseWindows):\n",
+ "class DeepNPTS(BaseModel):\n",
" \"\"\" DeepNPTS\n",
"\n",
" Deep Non-Parametric Time Series Forecaster (`DeepNPTS`) is a baseline model for time-series forecasting. This model generates predictions by (weighted) sampling from the empirical distribution according to a learnable strategy. The strategy is learned by exploiting the information across multiple related time series.\n",
@@ -133,10 +135,11 @@
"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
" \n",
" def __init__(self,\n",
" h,\n",
@@ -176,10 +179,10 @@
" if exclude_insample_y:\n",
" raise Exception('DeepNPTS has no possibility for excluding y.')\n",
"\n",
- " if not isinstance(loss, losses.BasePointLoss):\n",
+ " if loss.outputsize_multiplier > 1:\n",
" raise Exception('DeepNPTS only supports point loss functions (MAE, MSE, etc) as loss function.') \n",
" \n",
- " if not isinstance(valid_loss, losses.BasePointLoss):\n",
+ " if valid_loss is not None and not isinstance(valid_loss, losses.BasePointLoss):\n",
" raise Exception('DeepNPTS only supports point loss functions (MAE, MSE, etc) as valid loss function.') \n",
" \n",
" # Inherit BaseWindows class\n",
@@ -234,13 +237,13 @@
"\n",
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
- " x = windows_batch['insample_y'].unsqueeze(-1) # [B, L, 1]\n",
+ " x = windows_batch['insample_y'] # [B, L, 1]\n",
" hist_exog = windows_batch['hist_exog'] # [B, L, X]\n",
" futr_exog = windows_batch['futr_exog'] # [B, L + h, F]\n",
" stat_exog = windows_batch['stat_exog'] # [B, S]\n",
"\n",
" batch_size, seq_len = x.shape[:2] # B = batch_size, L = seq_len\n",
- " insample_y = windows_batch['insample_y'].unsqueeze(-1) \n",
+ " insample_y = windows_batch['insample_y'] \n",
" \n",
" # Concatenate x_t with future exogenous of input\n",
" if self.futr_exog_size > 0: \n",
@@ -268,9 +271,7 @@
" # Apply softmax for weighted input predictions\n",
" weights = weights.reshape(batch_size, seq_len, -1) # [B, L * h] -> [B, L, h]\n",
" x = F.softmax(weights, dim=1) * insample_y # [B, L, h] * [B, L, 1] = [B, L, h]\n",
- " output = torch.sum(x, dim=1).unsqueeze(-1) # [B, L, h] -> [B, h, 1]\n",
- "\n",
- " forecast = self.loss.domain_map(output) # [B, h, 1] -> [B, h, 1]\n",
+ " forecast = torch.sum(x, dim=1).unsqueeze(-1) # [B, L, h] -> [B, h, 1]\n",
"\n",
" return forecast"
]
@@ -302,6 +303,15 @@
"show_doc(DeepNPTS.predict, name='DeepNPTS.predict', title_level=3)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "check_model(DeepNPTS, [\"airpassengers\"])"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
diff --git a/nbs/models.dilated_rnn.ipynb b/nbs/models.dilated_rnn.ipynb
index 4b3bd374f..b18c5449f 100644
--- a/nbs/models.dilated_rnn.ipynb
+++ b/nbs/models.dilated_rnn.ipynb
@@ -13,7 +13,16 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n"
+ ]
+ }
+ ],
"source": [
"#| hide\n",
"%load_ext autoreload\n",
@@ -58,8 +67,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from nbdev.showdoc import show_doc\n",
- "from neuralforecast.utils import generate_series"
+ "from neuralforecast.utils import generate_series\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -75,7 +87,7 @@
"import torch.nn as nn\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_recurrent import BaseRecurrent\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import MLP"
]
},
@@ -324,8 +336,8 @@
"\n",
" blocks = [dilated_outputs[:, i * batchsize: (i + 1) * batchsize, :] for i in range(rate)]\n",
"\n",
- " interleaved = torch.stack((blocks)).transpose(1, 0).contiguous()\n",
- " interleaved = interleaved.view(dilated_outputs.size(0) * rate,\n",
+ " interleaved = torch.stack((blocks)).transpose(1, 0)\n",
+ " interleaved = interleaved.reshape(dilated_outputs.size(0) * rate,\n",
" batchsize,\n",
" dilated_outputs.size(2))\n",
" return interleaved\n",
@@ -359,7 +371,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class DilatedRNN(BaseRecurrent):\n",
+ "class DilatedRNN(BaseModel):\n",
" \"\"\" DilatedRNN\n",
"\n",
" **Parameters:**
\n",
@@ -398,24 +410,26 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'recurrent'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
- " EXOGENOUS_STAT = True \n",
+ " EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int,\n",
- " input_size: int = -1,\n",
+ " input_size: int,\n",
" inference_input_size: int = -1,\n",
" cell_type: str = 'LSTM',\n",
" dilations: List[List[int]] = [[1, 2], [4, 8]],\n",
- " encoder_hidden_size: int = 200,\n",
+ " encoder_hidden_size: int = 128,\n",
" context_size: int = 10,\n",
- " decoder_hidden_size: int = 200,\n",
+ " decoder_hidden_size: int = 128,\n",
" decoder_layers: int = 2,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" loss = MAE(),\n",
" valid_loss = None,\n",
" max_steps: int = 1000,\n",
@@ -425,6 +439,9 @@
" val_check_steps: int = 100,\n",
" batch_size = 32,\n",
" valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 128,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'robust',\n",
" random_seed: int = 1,\n",
@@ -439,7 +456,10 @@
" super(DilatedRNN, self).__init__(\n",
" h=h,\n",
" input_size=input_size,\n",
- " inference_input_size=inference_input_size,\n",
+ " futr_exog_list=futr_exog_list,\n",
+ " hist_exog_list=hist_exog_list,\n",
+ " stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -449,13 +469,14 @@
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
" valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
+ " step_size=step_size,\n",
" scaler_type=scaler_type,\n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
+ " random_seed=random_seed,\n",
" num_workers_loader=num_workers_loader,\n",
" drop_last_loader=drop_last_loader,\n",
- " random_seed=random_seed,\n",
" optimizer=optimizer,\n",
" optimizer_kwargs=optimizer_kwargs,\n",
" lr_scheduler=lr_scheduler,\n",
@@ -477,14 +498,12 @@
" self.decoder_layers = decoder_layers\n",
"\n",
" # RNN input size (1 for target variable y)\n",
- " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size\n",
+ " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size\n",
"\n",
" # Instantiate model\n",
" layers = []\n",
" for grp_num in range(len(self.dilations)):\n",
- " if grp_num == 0:\n",
- " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size\n",
- " else:\n",
+ " if grp_num > 0:\n",
" input_encoder = self.encoder_hidden_size\n",
" layer = DRNN(input_encoder,\n",
" self.encoder_hidden_size,\n",
@@ -496,11 +515,11 @@
" self.rnn_stack = nn.Sequential(*layers)\n",
"\n",
" # Context adapter\n",
- " self.context_adapter = nn.Linear(in_features=self.encoder_hidden_size + self.futr_exog_size * h,\n",
- " out_features=self.context_size * h)\n",
+ " self.context_adapter = nn.Linear(in_features=self.input_size,\n",
+ " out_features=h)\n",
"\n",
" # Decoder MLP\n",
- " self.mlp_decoder = MLP(in_features=self.context_size + self.futr_exog_size,\n",
+ " self.mlp_decoder = MLP(in_features=self.encoder_hidden_size + self.futr_exog_size,\n",
" out_features=self.loss.outputsize_multiplier,\n",
" hidden_size=self.decoder_hidden_size,\n",
" num_layers=self.decoder_layers,\n",
@@ -510,22 +529,23 @@
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
- " futr_exog = windows_batch['futr_exog']\n",
- " hist_exog = windows_batch['hist_exog']\n",
- " stat_exog = windows_batch['stat_exog']\n",
- "\n",
- " # Concatenate y, historic and static inputs\n",
- " # [B, C, seq_len, 1] -> [B, seq_len, C]\n",
- " # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]\n",
+ " encoder_input = windows_batch['insample_y'] # [B, L, 1]\n",
+ " futr_exog = windows_batch['futr_exog'] # [B, L + h, F]\n",
+ " hist_exog = windows_batch['hist_exog'] # [B, L, X]\n",
+ " stat_exog = windows_batch['stat_exog'] # [B, S]\n",
+ "\n",
+ " # Concatenate y, historic and static inputs \n",
" batch_size, seq_len = encoder_input.shape[:2]\n",
" if self.hist_exog_size > 0:\n",
- " hist_exog = hist_exog.permute(0,2,1,3).squeeze(-1) # [B, X, seq_len, 1] -> [B, seq_len, X]\n",
- " encoder_input = torch.cat((encoder_input, hist_exog), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, hist_exog), dim=2) # [B, L, 1] + [B, L, X] -> [B, L, 1 + X]\n",
"\n",
" if self.stat_exog_size > 0:\n",
- " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog), dim=2)\n",
+ " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, L, S]\n",
+ " encoder_input = torch.cat((encoder_input, stat_exog), dim=2) # [B, L, 1 + X] + [B, L, S] -> [B, L, 1 + X + S]\n",
+ "\n",
+ " if self.futr_exog_size > 0:\n",
+ " encoder_input = torch.cat((encoder_input, \n",
+ " futr_exog[:, :seq_len]), dim=2) # [B, L, 1 + X + S] + [B, L, F] -> [B, L, 1 + X + S + F]\n",
"\n",
" # DilatedRNN forward\n",
" for layer_num in range(len(self.rnn_stack)):\n",
@@ -535,25 +555,313 @@
" output += residual\n",
" encoder_input = output\n",
"\n",
- " if self.futr_exog_size > 0:\n",
- " futr_exog = futr_exog.permute(0,2,3,1)[:,:,1:,:] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]\n",
- " encoder_input = torch.cat(( encoder_input, futr_exog.reshape(batch_size, seq_len, -1)), dim=2)\n",
- "\n",
" # Context adapter\n",
- " context = self.context_adapter(encoder_input)\n",
- " context = context.reshape(batch_size, seq_len, self.h, self.context_size)\n",
+ " output = output.permute(0, 2, 1) # [B, L, C] -> [B, C, L]\n",
+ " context = self.context_adapter(output) # [B, C, L] -> [B, C, h]\n",
"\n",
" # Residual connection with futr_exog\n",
" if self.futr_exog_size > 0:\n",
- " context = torch.cat((context, futr_exog), dim=-1)\n",
+ " futr_exog_futr = futr_exog[:, seq_len:].permute(0, 2, 1) # [B, h, F] -> [B, F, h]\n",
+ " context = torch.cat((context, futr_exog_futr), \n",
+ " dim=1) # [B, C, h] + [B, F, h] = [B, C + F, h]\n",
"\n",
" # Final forecast\n",
- " output = self.mlp_decoder(context)\n",
- " output = self.loss.domain_map(output)\n",
+ " context = context.permute(0, 2, 1) # [B, C + F, h] -> [B, h, C + F]\n",
+ " output = self.mlp_decoder(context) # [B, h, C + F] -> [B, h, n_output]\n",
" \n",
" return output"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "---\n",
+ "\n",
+ "[source](https://github.com/Nixtla/neuralforecast/blob/main/neuralforecast/models/dilated_rnn.py#L289){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "\n",
+ "### DilatedRNN\n",
+ "\n",
+ "> DilatedRNN (h:int, input_size:int, inference_input_size:int=-1,\n",
+ "> cell_type:str='LSTM', dilations:List[List[int]]=[[1, 2], [4,\n",
+ "> 8]], encoder_hidden_size:int=200, context_size:int=10,\n",
+ "> decoder_hidden_size:int=200, decoder_layers:int=2,\n",
+ "> futr_exog_list=None, hist_exog_list=None,\n",
+ "> stat_exog_list=None, exclude_insample_y=False, loss=MAE(),\n",
+ "> valid_loss=None, max_steps:int=1000,\n",
+ "> learning_rate:float=0.001, num_lr_decays:int=3,\n",
+ "> early_stop_patience_steps:int=-1, val_check_steps:int=100,\n",
+ "> batch_size=32, valid_batch_size:Optional[int]=None,\n",
+ "> windows_batch_size=1024, inference_windows_batch_size=1024,\n",
+ "> start_padding_enabled=False, step_size:int=1,\n",
+ "> scaler_type:str='robust', random_seed:int=1,\n",
+ "> num_workers_loader:int=0, drop_last_loader:bool=False,\n",
+ "> optimizer=None, optimizer_kwargs=None, lr_scheduler=None,\n",
+ "> lr_scheduler_kwargs=None, **trainer_kwargs)\n",
+ "\n",
+ "*DilatedRNN\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`h`: int, forecast horizon.
\n",
+ "`input_size`: int, maximum sequence length for truncated train backpropagation. Default -1 uses all history.
\n",
+ "`inference_input_size`: int, maximum sequence length for truncated inference. Default -1 uses all history.
\n",
+ "`cell_type`: str, type of RNN cell to use. Options: 'GRU', 'RNN', 'LSTM', 'ResLSTM', 'AttentiveLSTM'.
\n",
+ "`dilations`: int list, dilations betweem layers.
\n",
+ "`encoder_hidden_size`: int=200, units for the RNN's hidden state size.
\n",
+ "`context_size`: int=10, size of context vector for each timestamp on the forecasting window.
\n",
+ "`decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
\n",
+ "`decoder_layers`: int=2, number of layers for the MLP decoder.
\n",
+ "`futr_exog_list`: str list, future exogenous columns.
\n",
+ "`hist_exog_list`: str list, historic exogenous columns.
\n",
+ "`stat_exog_list`: str list, static exogenous columns.
\n",
+ "`loss`: PyTorch module, instantiated train loss class from [losses collection](https://nixtla.github.io/neuralforecast/losses.pytorch.html).
\n",
+ "`valid_loss`: PyTorch module=`loss`, instantiated valid loss class from [losses collection](https://nixtla.github.io/neuralforecast/losses.pytorch.html).
\n",
+ "`max_steps`: int, maximum number of training steps.
\n",
+ "`learning_rate`: float, Learning rate between (0, 1).
\n",
+ "`num_lr_decays`: int, Number of learning rate decays, evenly distributed across max_steps.
\n",
+ "`early_stop_patience_steps`: int, Number of validation iterations before early stopping.
\n",
+ "`val_check_steps`: int, Number of training steps between every validation loss check.
\n",
+ "`batch_size`: int=32, number of different series in each batch.
\n",
+ "`valid_batch_size`: int=None, number of different series in each validation and test batch.
\n",
+ "`step_size`: int=1, step size between each window of temporal data.
\n",
+ "`scaler_type`: str='robust', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
+ "`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
+ "`num_workers_loader`: int=os.cpu_count(), workers to be used by `TimeSeriesDataLoader`.
\n",
+ "`drop_last_loader`: bool=False, if True `TimeSeriesDataLoader` drops last non-full batch.
\n",
+ "`alias`: str, optional, Custom name of the model.
\n",
+ "`optimizer`: Subclass of 'torch.optim.Optimizer', optional, user specified optimizer instead of the default choice (Adam).
\n",
+ "`optimizer_kwargs`: dict, optional, list of parameters used by the user specified `optimizer`.
\n",
+ "`lr_scheduler`: Subclass of 'torch.optim.lr_scheduler.LRScheduler', optional, user specified lr_scheduler instead of the default choice (StepLR).
\n",
+ "`lr_scheduler_kwargs`: dict, optional, list of parameters used by the user specified `lr_scheduler`.
\n",
+ "`**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
*"
+ ],
+ "text/plain": [
+ "---\n",
+ "\n",
+ "[source](https://github.com/Nixtla/neuralforecast/blob/main/neuralforecast/models/dilated_rnn.py#L289){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "\n",
+ "### DilatedRNN\n",
+ "\n",
+ "> DilatedRNN (h:int, input_size:int, inference_input_size:int=-1,\n",
+ "> cell_type:str='LSTM', dilations:List[List[int]]=[[1, 2], [4,\n",
+ "> 8]], encoder_hidden_size:int=200, context_size:int=10,\n",
+ "> decoder_hidden_size:int=200, decoder_layers:int=2,\n",
+ "> futr_exog_list=None, hist_exog_list=None,\n",
+ "> stat_exog_list=None, exclude_insample_y=False, loss=MAE(),\n",
+ "> valid_loss=None, max_steps:int=1000,\n",
+ "> learning_rate:float=0.001, num_lr_decays:int=3,\n",
+ "> early_stop_patience_steps:int=-1, val_check_steps:int=100,\n",
+ "> batch_size=32, valid_batch_size:Optional[int]=None,\n",
+ "> windows_batch_size=1024, inference_windows_batch_size=1024,\n",
+ "> start_padding_enabled=False, step_size:int=1,\n",
+ "> scaler_type:str='robust', random_seed:int=1,\n",
+ "> num_workers_loader:int=0, drop_last_loader:bool=False,\n",
+ "> optimizer=None, optimizer_kwargs=None, lr_scheduler=None,\n",
+ "> lr_scheduler_kwargs=None, **trainer_kwargs)\n",
+ "\n",
+ "*DilatedRNN\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`h`: int, forecast horizon.
\n",
+ "`input_size`: int, maximum sequence length for truncated train backpropagation. Default -1 uses all history.
\n",
+ "`inference_input_size`: int, maximum sequence length for truncated inference. Default -1 uses all history.
\n",
+ "`cell_type`: str, type of RNN cell to use. Options: 'GRU', 'RNN', 'LSTM', 'ResLSTM', 'AttentiveLSTM'.
\n",
+ "`dilations`: int list, dilations betweem layers.
\n",
+ "`encoder_hidden_size`: int=200, units for the RNN's hidden state size.
\n",
+ "`context_size`: int=10, size of context vector for each timestamp on the forecasting window.
\n",
+ "`decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
\n",
+ "`decoder_layers`: int=2, number of layers for the MLP decoder.
\n",
+ "`futr_exog_list`: str list, future exogenous columns.
\n",
+ "`hist_exog_list`: str list, historic exogenous columns.
\n",
+ "`stat_exog_list`: str list, static exogenous columns.
\n",
+ "`loss`: PyTorch module, instantiated train loss class from [losses collection](https://nixtla.github.io/neuralforecast/losses.pytorch.html).
\n",
+ "`valid_loss`: PyTorch module=`loss`, instantiated valid loss class from [losses collection](https://nixtla.github.io/neuralforecast/losses.pytorch.html).
\n",
+ "`max_steps`: int, maximum number of training steps.
\n",
+ "`learning_rate`: float, Learning rate between (0, 1).
\n",
+ "`num_lr_decays`: int, Number of learning rate decays, evenly distributed across max_steps.
\n",
+ "`early_stop_patience_steps`: int, Number of validation iterations before early stopping.
\n",
+ "`val_check_steps`: int, Number of training steps between every validation loss check.
\n",
+ "`batch_size`: int=32, number of different series in each batch.
\n",
+ "`valid_batch_size`: int=None, number of different series in each validation and test batch.
\n",
+ "`step_size`: int=1, step size between each window of temporal data.
\n",
+ "`scaler_type`: str='robust', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
+ "`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
+ "`num_workers_loader`: int=os.cpu_count(), workers to be used by `TimeSeriesDataLoader`.
\n",
+ "`drop_last_loader`: bool=False, if True `TimeSeriesDataLoader` drops last non-full batch.
\n",
+ "`alias`: str, optional, Custom name of the model.
\n",
+ "`optimizer`: Subclass of 'torch.optim.Optimizer', optional, user specified optimizer instead of the default choice (Adam).
\n",
+ "`optimizer_kwargs`: dict, optional, list of parameters used by the user specified `optimizer`.
\n",
+ "`lr_scheduler`: Subclass of 'torch.optim.lr_scheduler.LRScheduler', optional, user specified lr_scheduler instead of the default choice (StepLR).
\n",
+ "`lr_scheduler_kwargs`: dict, optional, list of parameters used by the user specified `lr_scheduler`.
\n",
+ "`**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
*"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "show_doc(DilatedRNN)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "---\n",
+ "\n",
+ "### DilatedRNN.fit\n",
+ "\n",
+ "> DilatedRNN.fit (dataset, val_size=0, test_size=0, random_seed=None,\n",
+ "> distributed_config=None)\n",
+ "\n",
+ "*Fit.\n",
+ "\n",
+ "The `fit` method, optimizes the neural network's weights using the\n",
+ "initialization parameters (`learning_rate`, `windows_batch_size`, ...)\n",
+ "and the `loss` function as defined during the initialization.\n",
+ "Within `fit` we use a PyTorch Lightning `Trainer` that\n",
+ "inherits the initialization's `self.trainer_kwargs`, to customize\n",
+ "its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).\n",
+ "\n",
+ "The method is designed to be compatible with SKLearn-like classes\n",
+ "and in particular to be compatible with the StatsForecast library.\n",
+ "\n",
+ "By default the `model` is not saving training checkpoints to protect\n",
+ "disk memory, to get them change `enable_checkpointing=True` in `__init__`.\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ "`val_size`: int, validation size for temporal cross-validation.
\n",
+ "`random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
+ "`test_size`: int, test size for temporal cross-validation.
*"
+ ],
+ "text/plain": [
+ "---\n",
+ "\n",
+ "### DilatedRNN.fit\n",
+ "\n",
+ "> DilatedRNN.fit (dataset, val_size=0, test_size=0, random_seed=None,\n",
+ "> distributed_config=None)\n",
+ "\n",
+ "*Fit.\n",
+ "\n",
+ "The `fit` method, optimizes the neural network's weights using the\n",
+ "initialization parameters (`learning_rate`, `windows_batch_size`, ...)\n",
+ "and the `loss` function as defined during the initialization.\n",
+ "Within `fit` we use a PyTorch Lightning `Trainer` that\n",
+ "inherits the initialization's `self.trainer_kwargs`, to customize\n",
+ "its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).\n",
+ "\n",
+ "The method is designed to be compatible with SKLearn-like classes\n",
+ "and in particular to be compatible with the StatsForecast library.\n",
+ "\n",
+ "By default the `model` is not saving training checkpoints to protect\n",
+ "disk memory, to get them change `enable_checkpointing=True` in `__init__`.\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ "`val_size`: int, validation size for temporal cross-validation.
\n",
+ "`random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
+ "`test_size`: int, test size for temporal cross-validation.
*"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "show_doc(DilatedRNN.fit, name='DilatedRNN.fit')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "---\n",
+ "\n",
+ "### DilatedRNN.predict\n",
+ "\n",
+ "> DilatedRNN.predict (dataset, test_size=None, step_size=1,\n",
+ "> random_seed=None, **data_module_kwargs)\n",
+ "\n",
+ "*Predict.\n",
+ "\n",
+ "Neural network prediction with PL's `Trainer` execution of `predict_step`.\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ "`test_size`: int=None, test size for temporal cross-validation.
\n",
+ "`step_size`: int=1, Step size between each window.
\n",
+ "`random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
+ "`**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).*"
+ ],
+ "text/plain": [
+ "---\n",
+ "\n",
+ "### DilatedRNN.predict\n",
+ "\n",
+ "> DilatedRNN.predict (dataset, test_size=None, step_size=1,\n",
+ "> random_seed=None, **data_module_kwargs)\n",
+ "\n",
+ "*Predict.\n",
+ "\n",
+ "Neural network prediction with PL's `Trainer` execution of `predict_step`.\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
\n",
+ "`test_size`: int=None, test size for temporal cross-validation.
\n",
+ "`step_size`: int=1, Step size between each window.
\n",
+ "`random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
\n",
+ "`**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).*"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "show_doc(DilatedRNN.predict, name='DilatedRNN.predict')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "DilatedRNN: checking forecast AirPassengers dataset\n"
+ ]
+ }
+ ],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(DilatedRNN, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -565,7 +873,124 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "c:\\Users\\ospra\\OneDrive\\Nixtla\\Repositories\\neuralforecast\\neuralforecast\\common\\_base_model.py:134: UserWarning: Input size too small. Automatically setting input size to 3 * horizon = 36\n",
+ " warnings.warn(\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "c575af1dd4b545f1a017aa6edc64a115",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Sanity Checking: | | 0/? [00:00"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"#| eval: false\n",
"import pandas as pd\n",
diff --git a/nbs/models.dlinear.ipynb b/nbs/models.dlinear.ipynb
index ea1a38a43..4191d5e96 100644
--- a/nbs/models.dlinear.ipynb
+++ b/nbs/models.dlinear.ipynb
@@ -58,7 +58,7 @@
"import torch\n",
"import torch.nn as nn\n",
"\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -70,8 +70,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -135,7 +138,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class DLinear(BaseWindows):\n",
+ "class DLinear(BaseModel):\n",
" \"\"\" DLinear\n",
"\n",
" *Parameters:*
\n",
@@ -173,10 +176,11 @@
"\t- Zeng, Ailing, et al. \"Are transformers effective for time series forecasting?.\" Proceedings of the AAAI conference on artificial intelligence. Vol. 37. No. 9. 2023.\"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -256,11 +260,7 @@
"\n",
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
- " insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
- " #futr_exog = windows_batch['futr_exog']\n",
+ " insample_y = windows_batch['insample_y'].squeeze(-1)\n",
"\n",
" # Parse inputs\n",
" batch_size = len(insample_y)\n",
@@ -272,7 +272,6 @@
" # Final\n",
" forecast = trend_part + seasonal_part\n",
" forecast = forecast.reshape(batch_size, self.h, self.loss.outputsize_multiplier)\n",
- " forecast = self.loss.domain_map(forecast)\n",
" return forecast"
]
},
@@ -303,6 +302,21 @@
"show_doc(DLinear.predict, name='DLinear.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(DLinear, [\"airpassengers\"])"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -322,7 +336,7 @@
"import matplotlib.pyplot as plt\n",
"\n",
"from neuralforecast import NeuralForecast\n",
- "from neuralforecast.models import DLinear\n",
+ "from neuralforecast import DLinear\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic, augment_calendar_df\n",
"\n",
"AirPassengersPanel, calendar_cols = augment_calendar_df(df=AirPassengersPanel, freq='M')\n",
diff --git a/nbs/models.fedformer.ipynb b/nbs/models.fedformer.ipynb
index 2268c058d..5ef61687b 100644
--- a/nbs/models.fedformer.ipynb
+++ b/nbs/models.fedformer.ipynb
@@ -51,6 +51,20 @@
""
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "import logging\n",
+ "import warnings\n",
+ "from fastcore.test import test_eq\n",
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -67,7 +81,7 @@
"\n",
"from neuralforecast.common._modules import DataEmbedding\n",
"from neuralforecast.common._modules import SeriesDecomp\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -402,7 +416,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class FEDformer(BaseWindows):\n",
+ "class FEDformer(BaseModel):\n",
" \"\"\" FEDformer\n",
"\n",
" The FEDformer model tackles the challenge of finding reliable dependencies on intricate temporal patterns of long-horizon forecasting.\n",
@@ -460,10 +474,11 @@
"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -626,13 +641,9 @@
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
" insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
" futr_exog = windows_batch['futr_exog']\n",
"\n",
" # Parse inputs\n",
- " insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]\n",
" if self.futr_exog_size > 0:\n",
" x_mark_enc = futr_exog[:,:self.input_size,:]\n",
" x_mark_dec = futr_exog[:,-(self.label_len+self.h):,:]\n",
@@ -659,11 +670,60 @@
" trend=trend_init)\n",
" # final\n",
" dec_out = trend_part + seasonal_part\n",
- "\n",
- " forecast = self.loss.domain_map(dec_out[:, -self.h:])\n",
+ " forecast = dec_out[:, -self.h:]\n",
+ " \n",
" return forecast"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "show_doc(FEDformer)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "show_doc(FEDformer.fit, name='FEDformer.fit')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "show_doc(FEDformer.predict, name='FEDformer.predict')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(FEDformer, [\"airpassengers\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Usage Example"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -682,6 +742,7 @@
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
+ "\n",
"model = FEDformer(h=12,\n",
" input_size=24,\n",
" modes=64,\n",
diff --git a/nbs/models.gru.ipynb b/nbs/models.gru.ipynb
index 7f0608a5f..0793c37be 100644
--- a/nbs/models.gru.ipynb
+++ b/nbs/models.gru.ipynb
@@ -69,7 +69,10 @@
"outputs": [],
"source": [
"#| hide\n",
- "from nbdev.showdoc import show_doc"
+ "import logging\n",
+ "from fastcore.test import test_eq\n",
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -84,9 +87,10 @@
"\n",
"import torch\n",
"import torch.nn as nn\n",
+ "import warnings\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_recurrent import BaseRecurrent\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import MLP"
]
},
@@ -97,7 +101,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class GRU(BaseRecurrent):\n",
+ "class GRU(BaseModel):\n",
" \"\"\" GRU\n",
"\n",
" Multi Layer Recurrent Network with Gated Units (GRU), and\n",
@@ -105,7 +109,7 @@
" using ADAM stochastic gradient descent. The network accepts static, historic \n",
" and future exogenous data, flattens the inputs.\n",
"\n",
- " **Parameters:**
\n",
+ " **Parameters:**
\n",
" `h`: int, forecast horizon.
\n",
" `input_size`: int, maximum sequence length for truncated train backpropagation. Default -1 uses all history.
\n",
" `inference_input_size`: int, maximum sequence length for truncated inference. Default -1 uses all history.
\n",
@@ -114,7 +118,7 @@
" `encoder_activation`: Optional[str]=None, Deprecated. Activation function in GRU is frozen in PyTorch.
\n",
" `encoder_bias`: bool=True, whether or not to use biases b_ih, b_hh within GRU units.
\n",
" `encoder_dropout`: float=0., dropout regularization applied to GRU outputs.
\n",
- " `context_size`: int=10, size of context vector for each timestamp on the forecasting window.
\n",
+ " `context_size`: deprecated.
\n",
" `decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
\n",
" `decoder_layers`: int=2, number of layers for the MLP decoder.
\n",
" `futr_exog_list`: str list, future exogenous columns.
\n",
@@ -142,10 +146,11 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'recurrent'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = True # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int,\n",
@@ -156,12 +161,14 @@
" encoder_activation: Optional[str] = None,\n",
" encoder_bias: bool = True,\n",
" encoder_dropout: float = 0.,\n",
- " context_size: int = 10,\n",
- " decoder_hidden_size: int = 200,\n",
+ " context_size: Optional[int] = None,\n",
+ " decoder_hidden_size: int = 128,\n",
" decoder_layers: int = 2,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
+ " recurrent = False,\n",
" loss = MAE(),\n",
" valid_loss = None,\n",
" max_steps: int = 1000,\n",
@@ -171,6 +178,10 @@
" val_check_steps: int = 100,\n",
" batch_size=32,\n",
" valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 128,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
+ " step_size: int = 1,\n",
" scaler_type: str='robust',\n",
" random_seed=1,\n",
" num_workers_loader=0,\n",
@@ -181,10 +192,16 @@
" lr_scheduler_kwargs = None,\n",
" dataloader_kwargs = None,\n",
" **trainer_kwargs):\n",
+ " \n",
+ " self.RECURRENT = recurrent\n",
+ "\n",
" super(GRU, self).__init__(\n",
" h=h,\n",
" input_size=input_size,\n",
- " inference_input_size=inference_input_size,\n",
+ " futr_exog_list=futr_exog_list,\n",
+ " hist_exog_list=hist_exog_list,\n",
+ " stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -194,13 +211,14 @@
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
" valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
+ " step_size=step_size,\n",
" scaler_type=scaler_type,\n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
+ " random_seed=random_seed,\n",
" num_workers_loader=num_workers_loader,\n",
" drop_last_loader=drop_last_loader,\n",
- " random_seed=random_seed,\n",
" optimizer=optimizer,\n",
" optimizer_kwargs=optimizer_kwargs,\n",
" lr_scheduler=lr_scheduler,\n",
@@ -224,75 +242,82 @@
" self.encoder_dropout = encoder_dropout\n",
" \n",
" # Context adapter\n",
- " self.context_size = context_size\n",
+ " if context_size is not None:\n",
+ " warnings.warn(\"context_size is deprecated and will be removed in future versions.\")\n",
"\n",
" # MLP decoder\n",
" self.decoder_hidden_size = decoder_hidden_size\n",
" self.decoder_layers = decoder_layers\n",
"\n",
" # RNN input size (1 for target variable y)\n",
- " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size\n",
+ " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size\n",
"\n",
" # Instantiate model\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = False\n",
" self.hist_encoder = nn.GRU(input_size=input_encoder,\n",
- " hidden_size=self.encoder_hidden_size,\n",
- " num_layers=self.encoder_n_layers,\n",
- " bias=self.encoder_bias,\n",
- " dropout=self.encoder_dropout,\n",
- " batch_first=True)\n",
- "\n",
- " # Context adapter\n",
- " self.context_adapter = nn.Linear(in_features=self.encoder_hidden_size + self.futr_exog_size * h,\n",
- " out_features=self.context_size * h)\n",
+ " hidden_size=self.encoder_hidden_size,\n",
+ " num_layers=self.encoder_n_layers,\n",
+ " bias=self.encoder_bias,\n",
+ " dropout=self.encoder_dropout,\n",
+ " batch_first=True)\n",
"\n",
" # Decoder MLP\n",
- " self.mlp_decoder = MLP(in_features=self.context_size + self.futr_exog_size,\n",
- " out_features=self.loss.outputsize_multiplier,\n",
- " hidden_size=self.decoder_hidden_size,\n",
- " num_layers=self.decoder_layers,\n",
- " activation='ReLU',\n",
- " dropout=0.0)\n",
+ " if self.RECURRENT:\n",
+ " self.proj = nn.Linear(self.encoder_hidden_size, self.loss.outputsize_multiplier)\n",
+ " else:\n",
+ " self.mlp_decoder = MLP(in_features=self.encoder_hidden_size + self.futr_exog_size,\n",
+ " out_features=self.loss.outputsize_multiplier,\n",
+ " hidden_size=self.decoder_hidden_size,\n",
+ " num_layers=self.decoder_layers,\n",
+ " activation='ReLU',\n",
+ " dropout=0.0)\n",
"\n",
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
- " futr_exog = windows_batch['futr_exog']\n",
- " hist_exog = windows_batch['hist_exog']\n",
- " stat_exog = windows_batch['stat_exog']\n",
+ " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
+ " futr_exog = windows_batch['futr_exog'] # [B, seq_len, F]\n",
+ " hist_exog = windows_batch['hist_exog'] # [B, seq_len, X]\n",
+ " stat_exog = windows_batch['stat_exog'] # [B, S]\n",
"\n",
- " # Concatenate y, historic and static inputs\n",
- " # [B, C, seq_len, 1] -> [B, seq_len, C]\n",
- " # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]\n",
+ " # Concatenate y, historic and static inputs \n",
" batch_size, seq_len = encoder_input.shape[:2]\n",
" if self.hist_exog_size > 0:\n",
- " hist_exog = hist_exog.permute(0,2,1,3).squeeze(-1) # [B, X, seq_len, 1] -> [B, seq_len, X]\n",
- " encoder_input = torch.cat((encoder_input, hist_exog), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, hist_exog), dim=2) # [B, seq_len, 1] + [B, seq_len, X] -> [B, seq_len, 1 + X]\n",
"\n",
" if self.stat_exog_size > 0:\n",
- " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog), dim=2)\n",
- "\n",
- " # RNN forward\n",
- " hidden_state, _ = self.hist_encoder(encoder_input) # [B, seq_len, rnn_hidden_state]\n",
+ " # print(encoder_input.shape)\n",
+ " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
+ " encoder_input = torch.cat((encoder_input, stat_exog), dim=2) # [B, seq_len, 1 + X] + [B, seq_len, S] -> [B, seq_len, 1 + X + S]\n",
"\n",
" if self.futr_exog_size > 0:\n",
- " futr_exog = futr_exog.permute(0,2,3,1)[:,:,1:,:] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]\n",
- " hidden_state = torch.cat(( hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, \n",
+ " futr_exog[:, :seq_len]), dim=2) # [B, seq_len, 1 + X + S] + [B, seq_len, F] -> [B, seq_len, 1 + X + S + F]\n",
"\n",
- " # Context adapter\n",
- " context = self.context_adapter(hidden_state)\n",
- " context = context.reshape(batch_size, seq_len, self.h, self.context_size)\n",
+ " if self.RECURRENT:\n",
+ " if self.maintain_state:\n",
+ " rnn_state = self.rnn_state\n",
+ " else:\n",
+ " rnn_state = None\n",
+ " \n",
+ " output, rnn_state = self.hist_encoder(encoder_input, \n",
+ " rnn_state) # [B, seq_len, rnn_hidden_state]\n",
+ " output = self.proj(output) # [B, seq_len, rnn_hidden_state] -> [B, seq_len, n_output]\n",
+ " if self.maintain_state:\n",
+ " self.rnn_state = rnn_state\n",
+ " else:\n",
+ " hidden_state, _ = self.hist_encoder(encoder_input, None) # [B, seq_len, rnn_hidden_state]\n",
+ " hidden_state = hidden_state[:, -self.h:] # [B, seq_len, rnn_hidden_state] -> [B, h, rnn_hidden_state]\n",
+ " \n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_futr = futr_exog[:, -self.h:] # [B, h, F]\n",
+ " hidden_state = torch.cat((hidden_state, \n",
+ " futr_exog_futr), dim=-1) # [B, h, rnn_hidden_state] + [B, h, F] -> [B, h, rnn_hidden_state + F]\n",
"\n",
- " # Residual connection with futr_exog\n",
- " if self.futr_exog_size > 0:\n",
- " context = torch.cat((context, futr_exog), dim=-1)\n",
+ " output = self.mlp_decoder(hidden_state) # [B, h, rnn_hidden_state + F] -> [B, seq_len, n_output]\n",
"\n",
- " # Final forecast\n",
- " output = self.mlp_decoder(context)\n",
- " output = self.loss.domain_map(output)\n",
- " \n",
- " return output"
+ " return output[:, -self.h:]"
]
},
{
@@ -322,6 +347,21 @@
"show_doc(GRU.predict, name='GRU.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(GRU, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -343,17 +383,15 @@
"# from neuralforecast.models import GRU\n",
"from neuralforecast.losses.pytorch import DistributionLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
"fcst = NeuralForecast(\n",
- " models=[GRU(h=12,input_size=-1,\n",
+ " models=[GRU(h=12, input_size=24,\n",
" loss=DistributionLoss(distribution='Normal', level=[80, 90]),\n",
" scaler_type='robust',\n",
" encoder_n_layers=2,\n",
" encoder_hidden_size=128,\n",
- " context_size=10,\n",
" decoder_hidden_size=128,\n",
" decoder_layers=2,\n",
" max_steps=200,\n",
diff --git a/nbs/models.informer.ipynb b/nbs/models.informer.ipynb
index c8e30137c..3efdeb344 100644
--- a/nbs/models.informer.ipynb
+++ b/nbs/models.informer.ipynb
@@ -71,7 +71,7 @@
" TransDecoderLayer, TransDecoder,\n",
" DataEmbedding, AttentionLayer,\n",
")\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -83,8 +83,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -259,7 +262,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class Informer(BaseWindows):\n",
+ "class Informer(BaseModel):\n",
" \"\"\" Informer\n",
"\n",
"\tThe Informer model tackles the vanilla Transformer computational complexity challenges for long-horizon forecasting. \n",
@@ -317,10 +320,11 @@
"\t- [Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, Wancai Zhang. \"Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting\"](https://arxiv.org/abs/2012.07436)
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False\n",
+ " RECURRENT = False\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -463,17 +467,11 @@
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
" insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
- "\n",
" futr_exog = windows_batch['futr_exog']\n",
"\n",
- " insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]\n",
- "\n",
" if self.futr_exog_size > 0:\n",
- " x_mark_enc = futr_exog[:,:self.input_size,:]\n",
- " x_mark_dec = futr_exog[:,-(self.label_len+self.h):,:]\n",
+ " x_mark_enc = futr_exog[:, :self.input_size, :]\n",
+ " x_mark_dec = futr_exog[:, -(self.label_len+self.h):, :]\n",
" else:\n",
" x_mark_enc = None\n",
" x_mark_dec = None\n",
@@ -488,7 +486,7 @@
" dec_out = self.decoder(dec_out, enc_out, x_mask=None, \n",
" cross_mask=None)\n",
"\n",
- " forecast = self.loss.domain_map(dec_out[:, -self.h:])\n",
+ " forecast = dec_out[:, -self.h:]\n",
" return forecast"
]
},
@@ -519,6 +517,21 @@
"show_doc(Informer.predict, name='Informer.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(Informer, [\"airpassengers\"])"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -555,7 +568,7 @@
" futr_exog_list=calendar_cols,\n",
" scaler_type='robust',\n",
" learning_rate=1e-3,\n",
- " max_steps=5,\n",
+ " max_steps=200,\n",
" val_check_steps=50,\n",
" early_stop_patience_steps=2)\n",
"\n",
diff --git a/nbs/models.ipynb b/nbs/models.ipynb
index 018525399..e3a3342a0 100644
--- a/nbs/models.ipynb
+++ b/nbs/models.ipynb
@@ -229,10 +229,10 @@
" \"input_size_multiplier\": [-1, 4, 16, 64],\n",
" \"inference_input_size_multiplier\": [-1],\n",
" \"h\": None,\n",
- " \"encoder_hidden_size\": tune.choice([50, 100, 200, 300]),\n",
+ " \"encoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"encoder_n_layers\": tune.randint(1, 4),\n",
" \"context_size\": tune.choice([5, 10, 50]),\n",
- " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n",
+ " \"decoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n",
" \"max_steps\": tune.choice([500, 1000]),\n",
" \"batch_size\": tune.choice([16, 32]),\n",
@@ -314,7 +314,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoRNN.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
"model = AutoRNN(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"model.fit(dataset=dataset)\n",
@@ -372,10 +372,10 @@
" \"input_size_multiplier\": [-1, 4, 16, 64],\n",
" \"inference_input_size_multiplier\": [-1],\n",
" \"h\": None,\n",
- " \"encoder_hidden_size\": tune.choice([50, 100, 200, 300]),\n",
+ " \"encoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"encoder_n_layers\": tune.randint(1, 4),\n",
" \"context_size\": tune.choice([5, 10, 50]),\n",
- " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n",
+ " \"decoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n",
" \"max_steps\": tune.choice([500, 1000]),\n",
" \"batch_size\": tune.choice([16, 32]),\n",
@@ -452,7 +452,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoLSTM.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
"model = AutoLSTM(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -511,10 +511,10 @@
" \"input_size_multiplier\": [-1, 4, 16, 64],\n",
" \"inference_input_size_multiplier\": [-1],\n",
" \"h\": None,\n",
- " \"encoder_hidden_size\": tune.choice([50, 100, 200, 300]),\n",
+ " \"encoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"encoder_n_layers\": tune.randint(1, 4),\n",
" \"context_size\": tune.choice([5, 10, 50]),\n",
- " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n",
+ " \"decoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n",
" \"max_steps\": tune.choice([500, 1000]),\n",
" \"batch_size\": tune.choice([16, 32]),\n",
@@ -591,7 +591,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoGRU.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
"model = AutoGRU(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -650,9 +650,9 @@
" \"input_size_multiplier\": [-1, 4, 16, 64],\n",
" \"inference_input_size_multiplier\": [-1],\n",
" \"h\": None,\n",
- " \"encoder_hidden_size\": tune.choice([50, 100, 200, 300]),\n",
+ " \"encoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"context_size\": tune.choice([5, 10, 50]),\n",
- " \"decoder_hidden_size\": tune.choice([64, 128]),\n",
+ " \"decoder_hidden_size\": tune.choice([32, 64]),\n",
" \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n",
" \"max_steps\": tune.choice([500, 1000]),\n",
" \"batch_size\": tune.choice([16, 32]),\n",
@@ -729,7 +729,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoTCN.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
"model = AutoTCN(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -927,10 +927,10 @@
" \"inference_input_size_multiplier\": [-1],\n",
" \"h\": None,\n",
" \"cell_type\": tune.choice(['LSTM', 'GRU']),\n",
- " \"encoder_hidden_size\": tune.choice([50, 100, 200, 300]),\n",
+ " \"encoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"dilations\": tune.choice([ [[1, 2], [4, 8]], [[1, 2, 4, 8]] ]),\n",
" \"context_size\": tune.choice([5, 10, 50]),\n",
- " \"decoder_hidden_size\": tune.choice([64, 128, 256, 512]),\n",
+ " \"decoder_hidden_size\": tune.choice([16, 32, 64, 128]),\n",
" \"learning_rate\": tune.loguniform(1e-4, 1e-1),\n",
" \"max_steps\": tune.choice([500, 1000]),\n",
" \"batch_size\": tune.choice([16, 32]),\n",
@@ -1007,7 +1007,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoDilatedRNN.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=-1, encoder_hidden_size=8)\n",
"model = AutoDilatedRNN(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -1290,7 +1290,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoMLP.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12, hidden_size=8)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12, hidden_size=8)\n",
"model = AutoMLP(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -1425,7 +1425,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoNBEATS.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12,\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12,\n",
" mlp_units=3*[[8, 8]])\n",
"model = AutoNBEATS(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
@@ -1561,7 +1561,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoNBEATSx.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12,\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12,\n",
" mlp_units=3*[[8, 8]])\n",
"model = AutoNBEATSx(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
@@ -1703,7 +1703,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoNHITS.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12, \n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12, \n",
" mlp_units=3 * [[8, 8]])\n",
"model = AutoNHITS(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
@@ -1841,7 +1841,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoDLinear.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12)\n",
"model = AutoDLinear(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -1976,7 +1976,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoNLinear.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12)\n",
"model = AutoNLinear(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -2119,7 +2119,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoTiDE.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12)\n",
"model = AutoTiDE(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -2257,7 +2257,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoDeepNPTS.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12)\n",
"model = AutoDeepNPTS(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
@@ -2403,7 +2403,7 @@
"source": [
"%%capture\n",
"# Use your own config or AutoKAN.default_config\n",
- "config = dict(max_steps=2, val_check_steps=1, input_size=12)\n",
+ "config = dict(max_steps=1, val_check_steps=1, input_size=12)\n",
"model = AutoKAN(h=12, config=config, num_samples=1, cpus=1)\n",
"\n",
"# Fit and predict\n",
diff --git a/nbs/models.itransformer.ipynb b/nbs/models.itransformer.ipynb
index 5e134cfa0..b226d66dc 100644
--- a/nbs/models.itransformer.ipynb
+++ b/nbs/models.itransformer.ipynb
@@ -27,8 +27,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -69,9 +72,9 @@
"import numpy as np\n",
"\n",
"from math import sqrt\n",
- "\n",
+ "from typing import Optional\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.common._modules import TransEncoder, TransEncoderLayer, AttentionLayer"
]
@@ -195,7 +198,7 @@
"source": [
"#| export\n",
"\n",
- "class iTransformer(BaseMultivariate):\n",
+ "class iTransformer(BaseModel):\n",
"\n",
" \"\"\" iTransformer\n",
"\n",
@@ -222,6 +225,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=128, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=128, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -240,10 +247,11 @@
" \"\"\"\n",
"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = True\n",
+ " RECURRENT = False\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -252,6 +260,7 @@
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" hidden_size: int = 512,\n",
" n_heads: int = 8,\n",
" e_layers: int = 2,\n",
@@ -268,6 +277,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 128,\n",
+ " inference_windows_batch_size = 128,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -286,6 +299,7 @@
" stat_exog_list = None,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -294,6 +308,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" random_seed=random_seed,\n",
@@ -335,8 +353,8 @@
" norm_layer=torch.nn.LayerNorm(self.hidden_size)\n",
" )\n",
"\n",
- " self.projector = nn.Linear(self.hidden_size, h, bias=True)\n",
- " \n",
+ " self.projector = nn.Linear(self.hidden_size, h * self.loss.outputsize_multiplier, bias=True)\n",
+ "\n",
" def forecast(self, x_enc):\n",
" if self.use_norm:\n",
" # Normalization from Non-stationary Transformer\n",
@@ -363,8 +381,8 @@
"\n",
" if self.use_norm:\n",
" # De-Normalization from Non-stationary Transformer\n",
- " dec_out = dec_out * (stdev[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))\n",
- " dec_out = dec_out + (means[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))\n",
+ " dec_out = dec_out * (stdev[:, 0, :].unsqueeze(1).repeat(1, self.h * self.loss.outputsize_multiplier, 1))\n",
+ " dec_out = dec_out + (means[:, 0, :].unsqueeze(1).repeat(1, self.h * self.loss.outputsize_multiplier, 1))\n",
"\n",
" return dec_out\n",
" \n",
@@ -372,14 +390,11 @@
" insample_y = windows_batch['insample_y']\n",
"\n",
" y_pred = self.forecast(insample_y)\n",
- " y_pred = y_pred[:, -self.h:, :]\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
+ " y_pred = y_pred.reshape(insample_y.shape[0],\n",
+ " self.h,\n",
+ " -1)\n",
"\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " if y_pred.ndim == 2:\n",
- " return y_pred.unsqueeze(-1)\n",
- " else:\n",
- " return y_pred\n"
+ " return y_pred"
]
},
{
@@ -409,6 +424,21 @@
"show_doc(iTransformer.predict, name='iTransformer.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(iTransformer, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -448,7 +478,8 @@
" loss=MSE(),\n",
" valid_loss=MAE(),\n",
" early_stop_patience_steps=3,\n",
- " batch_size=32)\n",
+ " batch_size=32,\n",
+ " max_steps=100)\n",
"\n",
"fcst = NeuralForecast(models=[model], freq='M')\n",
"fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
diff --git a/nbs/models.kan.ipynb b/nbs/models.kan.ipynb
index ac7cc5e2b..003a8e3d0 100644
--- a/nbs/models.kan.ipynb
+++ b/nbs/models.kan.ipynb
@@ -61,8 +61,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -80,7 +83,7 @@
"import torch.nn.functional as F\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_windows import BaseWindows"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -318,7 +321,7 @@
"source": [
"#| export\n",
"\n",
- "class KAN(BaseWindows):\n",
+ "class KAN(BaseModel):\n",
" \"\"\" KAN\n",
"\n",
" Simple Kolmogorov-Arnold Network (KAN).\n",
@@ -372,10 +375,11 @@
" \"\"\"\n",
"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True \n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -495,7 +499,7 @@
" \n",
" def forward(self, windows_batch, update_grid=False):\n",
"\n",
- " insample_y = windows_batch['insample_y']\n",
+ " insample_y = windows_batch['insample_y'].squeeze(-1)\n",
" futr_exog = windows_batch['futr_exog']\n",
" hist_exog = windows_batch['hist_exog']\n",
" stat_exog = windows_batch['stat_exog']\n",
@@ -520,7 +524,6 @@
"\n",
" y_pred = y_pred.reshape(batch_size, self.h, \n",
" self.loss.outputsize_multiplier)\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
" return y_pred\n",
" "
]
@@ -552,6 +555,21 @@
"show_doc(KAN.predict, name='KAN.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(KAN, checks=[\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -574,7 +592,6 @@
"from neuralforecast.losses.pytorch import DistributionLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
- "\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
diff --git a/nbs/models.lstm.ipynb b/nbs/models.lstm.ipynb
index 3eb469306..954e53257 100644
--- a/nbs/models.lstm.ipynb
+++ b/nbs/models.lstm.ipynb
@@ -58,7 +58,10 @@
"outputs": [],
"source": [
"#| hide\n",
- "from nbdev.showdoc import show_doc"
+ "import logging\n",
+ "from fastcore.test import test_eq\n",
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -72,9 +75,10 @@
"\n",
"import torch\n",
"import torch.nn as nn\n",
+ "import warnings\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_recurrent import BaseRecurrent\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import MLP"
]
},
@@ -85,7 +89,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class LSTM(BaseRecurrent):\n",
+ "class LSTM(BaseModel):\n",
" \"\"\" LSTM\n",
"\n",
" LSTM encoder, with MLP decoder.\n",
@@ -101,7 +105,7 @@
" `encoder_hidden_size`: int=200, units for the LSTM's hidden state size.
\n",
" `encoder_bias`: bool=True, whether or not to use biases b_ih, b_hh within LSTM units.
\n",
" `encoder_dropout`: float=0., dropout regularization applied to LSTM outputs.
\n",
- " `context_size`: int=10, size of context vector for each timestamp on the forecasting window.
\n",
+ " `context_size`: deprecated.
\n",
" `decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
\n",
" `decoder_layers`: int=2, number of layers for the MLP decoder.
\n",
" `futr_exog_list`: str list, future exogenous columns.
\n",
@@ -129,25 +133,27 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'recurrent'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = True # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int,\n",
- " input_size: int = -1,\n",
- " inference_input_size: int = -1,\n",
+ " input_size: int,\n",
" encoder_n_layers: int = 2,\n",
- " encoder_hidden_size: int = 200,\n",
+ " encoder_hidden_size: int = 128,\n",
" encoder_bias: bool = True,\n",
" encoder_dropout: float = 0.,\n",
- " context_size: int = 10,\n",
- " decoder_hidden_size: int = 200,\n",
+ " context_size: Optional[int] = None,\n",
+ " decoder_hidden_size: int = 128,\n",
" decoder_layers: int = 2,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
+ " recurrent = False,\n",
" loss = MAE(),\n",
" valid_loss = None,\n",
" max_steps: int = 1000,\n",
@@ -157,6 +163,10 @@
" val_check_steps: int = 100,\n",
" batch_size = 32,\n",
" valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 128,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
+ " step_size: int = 1,\n",
" scaler_type: str = 'robust',\n",
" random_seed = 1,\n",
" num_workers_loader = 0,\n",
@@ -167,10 +177,16 @@
" lr_scheduler_kwargs = None,\n",
" dataloader_kwargs = None,\n",
" **trainer_kwargs):\n",
+ " \n",
+ " self.RECURRENT = recurrent\n",
+ " \n",
" super(LSTM, self).__init__(\n",
" h=h,\n",
" input_size=input_size,\n",
- " inference_input_size=inference_input_size,\n",
+ " futr_exog_list=futr_exog_list,\n",
+ " hist_exog_list=hist_exog_list,\n",
+ " stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -180,13 +196,14 @@
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
" valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
+ " step_size=step_size,\n",
" scaler_type=scaler_type,\n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
+ " random_seed=random_seed,\n",
" num_workers_loader=num_workers_loader,\n",
" drop_last_loader=drop_last_loader,\n",
- " random_seed=random_seed,\n",
" optimizer=optimizer,\n",
" optimizer_kwargs=optimizer_kwargs,\n",
" lr_scheduler=lr_scheduler,\n",
@@ -202,75 +219,80 @@
" self.encoder_dropout = encoder_dropout\n",
" \n",
" # Context adapter\n",
- " self.context_size = context_size\n",
+ " if context_size is not None:\n",
+ " warnings.warn(\"context_size is deprecated and will be removed in future versions.\")\n",
"\n",
" # MLP decoder\n",
" self.decoder_hidden_size = decoder_hidden_size\n",
" self.decoder_layers = decoder_layers\n",
"\n",
" # LSTM input size (1 for target variable y)\n",
- " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size\n",
+ " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size\n",
"\n",
" # Instantiate model\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = False\n",
" self.hist_encoder = nn.LSTM(input_size=input_encoder,\n",
" hidden_size=self.encoder_hidden_size,\n",
" num_layers=self.encoder_n_layers,\n",
" bias=self.encoder_bias,\n",
" dropout=self.encoder_dropout,\n",
- " batch_first=True)\n",
- "\n",
- " # Context adapter\n",
- " self.context_adapter = nn.Linear(in_features=self.encoder_hidden_size + self.futr_exog_size * h,\n",
- " out_features=self.context_size * h)\n",
+ " batch_first=True,\n",
+ " proj_size=self.loss.outputsize_multiplier if self.RECURRENT else 0)\n",
"\n",
" # Decoder MLP\n",
- " self.mlp_decoder = MLP(in_features=self.context_size + self.futr_exog_size,\n",
- " out_features=self.loss.outputsize_multiplier,\n",
- " hidden_size=self.decoder_hidden_size,\n",
- " num_layers=self.decoder_layers,\n",
- " activation='ReLU',\n",
- " dropout=0.0)\n",
+ " if not self.RECURRENT:\n",
+ " self.mlp_decoder = MLP(in_features=self.encoder_hidden_size + self.futr_exog_size,\n",
+ " out_features=self.loss.outputsize_multiplier,\n",
+ " hidden_size=self.decoder_hidden_size,\n",
+ " num_layers=self.decoder_layers,\n",
+ " activation='ReLU',\n",
+ " dropout=0.0)\n",
"\n",
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
- " futr_exog = windows_batch['futr_exog']\n",
- " hist_exog = windows_batch['hist_exog']\n",
- " stat_exog = windows_batch['stat_exog']\n",
+ " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
+ " futr_exog = windows_batch['futr_exog'] # [B, seq_len, F]\n",
+ " hist_exog = windows_batch['hist_exog'] # [B, seq_len, X]\n",
+ " stat_exog = windows_batch['stat_exog'] # [B, S]\n",
"\n",
- " # Concatenate y, historic and static inputs\n",
- " # [B, C, seq_len, 1] -> [B, seq_len, C]\n",
- " # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]\n",
+ " # Concatenate y, historic and static inputs \n",
" batch_size, seq_len = encoder_input.shape[:2]\n",
" if self.hist_exog_size > 0:\n",
- " hist_exog = hist_exog.permute(0,2,1,3).squeeze(-1) # [B, X, seq_len, 1] -> [B, seq_len, X]\n",
- " encoder_input = torch.cat((encoder_input, hist_exog), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, hist_exog), dim=2) # [B, seq_len, 1] + [B, seq_len, X] -> [B, seq_len, 1 + X]\n",
"\n",
" if self.stat_exog_size > 0:\n",
- " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog), dim=2)\n",
- "\n",
- " # RNN forward\n",
- " hidden_state, _ = self.hist_encoder(encoder_input) # [B, seq_len, rnn_hidden_state]\n",
+ " # print(encoder_input.shape)\n",
+ " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
+ " encoder_input = torch.cat((encoder_input, stat_exog), dim=2) # [B, seq_len, 1 + X] + [B, seq_len, S] -> [B, seq_len, 1 + X + S]\n",
"\n",
" if self.futr_exog_size > 0:\n",
- " futr_exog = futr_exog.permute(0,2,3,1)[:,:,1:,:] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]\n",
- " hidden_state = torch.cat(( hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, \n",
+ " futr_exog[:, :seq_len]), dim=2) # [B, seq_len, 1 + X + S] + [B, seq_len, F] -> [B, seq_len, 1 + X + S + F]\n",
"\n",
- " # Context adapter\n",
- " context = self.context_adapter(hidden_state)\n",
- " context = context.reshape(batch_size, seq_len, self.h, self.context_size)\n",
+ " if self.RECURRENT:\n",
+ " if self.maintain_state:\n",
+ " rnn_state = self.rnn_state\n",
+ " else:\n",
+ " rnn_state = None\n",
+ " \n",
+ " output, rnn_state = self.hist_encoder(encoder_input, \n",
+ " rnn_state) # [B, seq_len, n_output]\n",
+ " if self.maintain_state:\n",
+ " self.rnn_state = rnn_state\n",
+ " else:\n",
+ " hidden_state, _ = self.hist_encoder(encoder_input, None) # [B, seq_len, rnn_hidden_state]\n",
+ " hidden_state = hidden_state[:, -self.h:] # [B, seq_len, rnn_hidden_state] -> [B, h, rnn_hidden_state]\n",
+ " \n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_futr = futr_exog[:, -self.h:] # [B, h, F]\n",
+ " hidden_state = torch.cat((hidden_state, \n",
+ " futr_exog_futr), dim=-1) # [B, h, rnn_hidden_state] + [B, h, F] -> [B, h, rnn_hidden_state + F]\n",
"\n",
- " # Residual connection with futr_exog\n",
- " if self.futr_exog_size > 0:\n",
- " context = torch.cat((context, futr_exog), dim=-1)\n",
+ " output = self.mlp_decoder(hidden_state) # [B, h, rnn_hidden_state + F] -> [B, seq_len, n_output]\n",
"\n",
- " # Final forecast\n",
- " output = self.mlp_decoder(context)\n",
- " output = self.loss.domain_map(output)\n",
- " \n",
- " return output"
+ " return output[:, -self.h:]"
]
},
{
@@ -300,6 +322,21 @@
"show_doc(LSTM.predict, name='LSTM.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(LSTM, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -326,17 +363,18 @@
"Y_test_df = AirPassengersPanel[AirPassengersPanel.ds>=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
"nf = NeuralForecast(\n",
- " models=[LSTM(h=12, input_size=-1,\n",
- " loss=DistributionLoss(distribution='Normal', level=[80, 90]),\n",
+ " models=[LSTM(h=12, \n",
+ " input_size=24,\n",
+ " loss=DistributionLoss(distribution=\"Normal\", level=[80, 90]),\n",
" scaler_type='robust',\n",
" encoder_n_layers=2,\n",
" encoder_hidden_size=128,\n",
- " context_size=10,\n",
" decoder_hidden_size=128,\n",
" decoder_layers=2,\n",
" max_steps=200,\n",
" futr_exog_list=['y_[lag12]'],\n",
" stat_exog_list=['airline1'],\n",
+ " recurrent=False,\n",
" )\n",
" ],\n",
" freq='M'\n",
@@ -344,19 +382,18 @@
"nf.fit(df=Y_train_df, static_df=AirPassengersStatic)\n",
"Y_hat_df = nf.predict(futr_df=Y_test_df)\n",
"\n",
+ "# Plots\n",
"Y_hat_df = Y_hat_df.reset_index(drop=False).drop(columns=['unique_id','ds'])\n",
"plot_df = pd.concat([Y_test_df, Y_hat_df], axis=1)\n",
"plot_df = pd.concat([Y_train_df, plot_df])\n",
"\n",
"plot_df = plot_df[plot_df.unique_id=='Airline1'].drop('unique_id', axis=1)\n",
"plt.plot(plot_df['ds'], plot_df['y'], c='black', label='True')\n",
- "plt.plot(plot_df['ds'], plot_df['LSTM'], c='purple', label='mean')\n",
"plt.plot(plot_df['ds'], plot_df['LSTM-median'], c='blue', label='median')\n",
"plt.fill_between(x=plot_df['ds'][-12:], \n",
- " y1=plot_df['LSTM-lo-90'][-12:].values, \n",
+ " y1=plot_df['LSTM-lo-90'][-12:].values,\n",
" y2=plot_df['LSTM-hi-90'][-12:].values,\n",
" alpha=0.4, label='level 90')\n",
- "plt.legend()\n",
"plt.grid()\n",
"plt.plot()"
]
diff --git a/nbs/models.mlp.ipynb b/nbs/models.mlp.ipynb
index 46c09406f..848cd037c 100644
--- a/nbs/models.mlp.ipynb
+++ b/nbs/models.mlp.ipynb
@@ -49,8 +49,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -67,7 +70,7 @@
"import torch.nn as nn\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_windows import BaseWindows"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -78,7 +81,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class MLP(BaseWindows):\n",
+ "class MLP(BaseModel):\n",
" \"\"\" MLP\n",
"\n",
" Simple Multi Layer Perceptron architecture (MLP). \n",
@@ -122,10 +125,11 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
- " EXOGENOUS_STAT = True \n",
+ " EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -211,7 +215,7 @@
" def forward(self, windows_batch):\n",
"\n",
" # Parse windows_batch\n",
- " insample_y = windows_batch['insample_y']\n",
+ " insample_y = windows_batch['insample_y'].squeeze(-1)\n",
" futr_exog = windows_batch['futr_exog']\n",
" hist_exog = windows_batch['hist_exog']\n",
" stat_exog = windows_batch['stat_exog']\n",
@@ -235,7 +239,6 @@
"\n",
" y_pred = y_pred.reshape(batch_size, self.h, \n",
" self.loss.outputsize_multiplier)\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
" return y_pred"
]
},
@@ -269,6 +272,22 @@
"show_doc(MLP.predict, name='MLP.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a09d7a35",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(MLP, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -421,6 +440,7 @@
"fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
"forecasts = fcst.predict(futr_df=Y_test_df)\n",
"\n",
+ "# Plot predictions\n",
"Y_hat_df = forecasts.reset_index(drop=False).drop(columns=['unique_id','ds'])\n",
"plot_df = pd.concat([Y_test_df, Y_hat_df], axis=1)\n",
"plot_df = pd.concat([Y_train_df, plot_df])\n",
diff --git a/nbs/models.mlpmultivariate.ipynb b/nbs/models.mlpmultivariate.ipynb
index 71abdfb04..d06f3034b 100644
--- a/nbs/models.mlpmultivariate.ipynb
+++ b/nbs/models.mlpmultivariate.ipynb
@@ -49,8 +49,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -64,8 +67,9 @@
"import torch\n",
"import torch.nn as nn\n",
"\n",
+ "from typing import Optional\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -76,7 +80,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class MLPMultivariate(BaseMultivariate):\n",
+ "class MLPMultivariate(BaseModel):\n",
" \"\"\" MLPMultivariate\n",
"\n",
" Simple Multi Layer Perceptron architecture (MLP) for multivariate forecasting. \n",
@@ -102,6 +106,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -116,10 +124,11 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True \n",
+ " MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -128,6 +137,7 @@
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" num_layers = 2,\n",
" hidden_size = 1024,\n",
" loss = MAE(),\n",
@@ -138,6 +148,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 256,\n",
+ " inference_windows_batch_size = 256,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -157,6 +171,7 @@
" futr_exog_list=futr_exog_list,\n",
" hist_exog_list=hist_exog_list,\n",
" stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -165,6 +180,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" num_workers_loader=num_workers_loader,\n",
@@ -223,15 +242,9 @@
" x = torch.relu(layer(x))\n",
" x = self.out(x)\n",
" \n",
- " x = x.reshape(batch_size, self.h, -1)\n",
- " forecast = self.loss.domain_map(x)\n",
+ " forecast = x.reshape(batch_size, self.h, -1)\n",
"\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.\n",
- " if forecast.ndim == 2:\n",
- " return forecast.unsqueeze(-1)\n",
- " else:\n",
- " return forecast"
+ " return forecast"
]
},
{
@@ -267,76 +280,17 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1bf909e1",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "import logging\n",
- "import warnings\n",
- "\n",
- "from neuralforecast import NeuralForecast\n",
- "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "from neuralforecast.losses.pytorch import MAE, MSE, RMSE, MAPE, SMAPE, MASE, relMSE, QuantileLoss, MQLoss, DistributionLoss,PMM, GMM, NBMM, HuberLoss, TukeyLoss, HuberQLoss, HuberMQLoss"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f7ee8d15",
+ "id": "6c22db80",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
- "# Test losses\n",
+ "# Unit tests for models\n",
"logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
- "warnings.filterwarnings(\"ignore\")\n",
- "\n",
- "Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
- "\n",
- "AirPassengersStatic_single = AirPassengersStatic[AirPassengersStatic[\"unique_id\"] == 'Airline1']\n",
- "Y_train_df_single = Y_train_df[Y_train_df[\"unique_id\"] == 'Airline1']\n",
- "Y_test_df_single = Y_test_df[Y_test_df[\"unique_id\"] == 'Airline1']\n",
- "\n",
- "losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "valid_losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "\n",
- "for loss, valid_loss in zip(losses, valid_losses):\n",
- " try:\n",
- " model = MLPMultivariate(h=12, \n",
- " input_size=24,\n",
- " n_series=2,\n",
- " loss = loss,\n",
- " valid_loss = valid_loss,\n",
- " scaler_type='robust',\n",
- " learning_rate=1e-3,\n",
- " max_steps=2,\n",
- " val_check_steps=10,\n",
- " early_stop_patience_steps=2,\n",
- " )\n",
- "\n",
- " fcst = NeuralForecast(models=[model], freq='M')\n",
- " fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
- " forecasts = fcst.predict(futr_df=Y_test_df)\n",
- " except Exception as e:\n",
- " assert str(e) == f\"{loss} is not supported in a Multivariate model.\"\n",
- "\n",
- "\n",
- "# Test n_series = 1\n",
- "model = MLPMultivariate(h=12, \n",
- " input_size=24,\n",
- " n_series=1,\n",
- " loss = MAE(),\n",
- " scaler_type='robust',\n",
- " learning_rate=1e-3,\n",
- " max_steps=2,\n",
- " val_check_steps=10,\n",
- " early_stop_patience_steps=2,\n",
- " )\n",
- "fcst = NeuralForecast(models=[model], freq='M')\n",
- "fcst.fit(df=Y_train_df_single, static_df=AirPassengersStatic_single, val_size=12)\n",
- "forecasts = fcst.predict(futr_df=Y_test_df_single) "
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(MLPMultivariate, [\"airpassengers\"])"
]
},
{
@@ -374,6 +328,7 @@
" loss = MAE(),\n",
" scaler_type='robust',\n",
" learning_rate=1e-3,\n",
+ " stat_exog_list=['airline1'],\n",
" max_steps=200,\n",
" val_check_steps=10,\n",
" early_stop_patience_steps=2)\n",
@@ -385,6 +340,7 @@
"fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
"forecasts = fcst.predict(futr_df=Y_test_df)\n",
"\n",
+ "# Plot predictions\n",
"Y_hat_df = forecasts.reset_index(drop=False).drop(columns=['unique_id','ds'])\n",
"plot_df = pd.concat([Y_test_df, Y_hat_df], axis=1)\n",
"plot_df = pd.concat([Y_train_df, plot_df])\n",
diff --git a/nbs/models.nbeats.ipynb b/nbs/models.nbeats.ipynb
index 9504770d5..be1c8a93a 100644
--- a/nbs/models.nbeats.ipynb
+++ b/nbs/models.nbeats.ipynb
@@ -66,7 +66,7 @@
"import torch.nn as nn\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_windows import BaseWindows"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -77,9 +77,12 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
"from nbdev.showdoc import show_doc\n",
"from neuralforecast.utils import generate_series\n",
+ "from neuralforecast.common._model_checks import check_model\n",
"\n",
"import matplotlib.pyplot as plt"
]
@@ -231,7 +234,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class NBEATS(BaseWindows):\n",
+ "class NBEATS(BaseModel):\n",
" \"\"\" NBEATS\n",
"\n",
" The Neural Basis Expansion Analysis for Time Series (NBEATS), is a simple and yet\n",
@@ -282,10 +285,11 @@
" \"N-BEATS: Neural basis expansion analysis for interpretable time series forecasting\".](https://arxiv.org/abs/1905.10437)\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
" \n",
" def __init__(self,\n",
" h,\n",
@@ -420,8 +424,8 @@
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " insample_y = windows_batch['insample_y']\n",
- " insample_mask = windows_batch['insample_mask']\n",
+ " insample_y = windows_batch['insample_y'].squeeze(-1)\n",
+ " insample_mask = windows_batch['insample_mask'].squeeze(-1)\n",
"\n",
" # NBEATS' forward\n",
" residuals = insample_y.flip(dims=(-1,)) # backcast init\n",
@@ -435,10 +439,7 @@
" forecast = forecast + block_forecast\n",
"\n",
" if self.decompose_forecast:\n",
- " block_forecasts.append(block_forecast)\n",
- "\n",
- " # Adapting output's domain\n",
- " forecast = self.loss.domain_map(forecast) \n",
+ " block_forecasts.append(block_forecast) \n",
"\n",
" if self.decompose_forecast:\n",
" # (n_batch, n_blocks, h, out_features)\n",
@@ -480,6 +481,22 @@
"show_doc(NBEATS.predict, name='NBEATS.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8de78f60",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(NBEATS, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/nbs/models.nbeatsx.ipynb b/nbs/models.nbeatsx.ipynb
index 9952c3cf9..aaba3b760 100644
--- a/nbs/models.nbeatsx.ipynb
+++ b/nbs/models.nbeatsx.ipynb
@@ -62,7 +62,8 @@
"\n",
"from fastcore.test import test_eq, test_fail\n",
"from nbdev.showdoc import show_doc\n",
- "from neuralforecast.utils import generate_series"
+ "from neuralforecast.utils import generate_series\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -80,7 +81,7 @@
"import torch.nn as nn\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_windows import BaseWindows"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -373,7 +374,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class NBEATSx(BaseWindows):\n",
+ "class NBEATSx(BaseModel):\n",
" \"\"\"NBEATSx\n",
"\n",
" The Neural Basis Expansion Analysis with Exogenous variables (NBEATSx) is a simple\n",
@@ -427,10 +428,11 @@
" \"\"\"\n",
"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = \"windows\"\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(\n",
" self,\n",
@@ -612,8 +614,8 @@
"\n",
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
- " insample_y = windows_batch[\"insample_y\"]\n",
- " insample_mask = windows_batch[\"insample_mask\"]\n",
+ " insample_y = windows_batch[\"insample_y\"].squeeze(-1)\n",
+ " insample_mask = windows_batch[\"insample_mask\"].squeeze(-1)\n",
" futr_exog = windows_batch[\"futr_exog\"]\n",
" hist_exog = windows_batch[\"hist_exog\"]\n",
" stat_exog = windows_batch[\"stat_exog\"]\n",
@@ -637,9 +639,6 @@
" if self.decompose_forecast:\n",
" block_forecasts.append(block_forecast)\n",
"\n",
- " # Adapting output's domain\n",
- " forecast = self.loss.domain_map(forecast)\n",
- "\n",
" if self.decompose_forecast:\n",
" # (n_batch, n_blocks, h)\n",
" block_forecasts = torch.stack(block_forecasts)\n",
@@ -680,6 +679,22 @@
"show_doc(NBEATSx.predict, name='NBEATSx.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ce8cba7d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(NBEATSx, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -806,7 +821,7 @@
"# test seasonality/trend basis protection\n",
"test_fail(NBEATSx.__init__, \n",
" contains='Horizon `h=1` incompatible with `seasonality` or `trend` in stacks',\n",
- " kwargs=dict(self=BaseWindows, h=1, input_size=4))"
+ " kwargs=dict(self=BaseModel, h=1, input_size=4))"
]
},
{
diff --git a/nbs/models.nhits.ipynb b/nbs/models.nhits.ipynb
index e844f4660..98da310c1 100644
--- a/nbs/models.nhits.ipynb
+++ b/nbs/models.nhits.ipynb
@@ -67,7 +67,7 @@
"import torch.nn.functional as F\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_windows import BaseWindows"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -83,7 +83,8 @@
"import matplotlib.pyplot as plt\n",
"from fastcore.test import test_eq\n",
"from nbdev.showdoc import show_doc\n",
- "from neuralforecast.utils import generate_series"
+ "from neuralforecast.utils import generate_series\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -261,7 +262,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class NHITS(BaseWindows):\n",
+ "class NHITS(BaseModel):\n",
" \"\"\" NHITS\n",
"\n",
" The Neural Hierarchical Interpolation for Time Series (NHITS), is an MLP-based deep\n",
@@ -316,10 +317,11 @@
" Accepted at the Thirty-Seventh AAAI Conference on Artificial Intelligence.](https://arxiv.org/abs/2201.12886)\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self, \n",
" h,\n",
@@ -455,8 +457,8 @@
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " insample_y = windows_batch['insample_y']\n",
- " insample_mask = windows_batch['insample_mask']\n",
+ " insample_y = windows_batch['insample_y'].squeeze(-1).contiguous()\n",
+ " insample_mask = windows_batch['insample_mask'].squeeze(-1).contiguous()\n",
" futr_exog = windows_batch['futr_exog']\n",
" hist_exog = windows_batch['hist_exog']\n",
" stat_exog = windows_batch['stat_exog']\n",
@@ -476,9 +478,6 @@
" if self.decompose_forecast:\n",
" block_forecasts.append(block_forecast)\n",
" \n",
- " # Adapting output's domain\n",
- " forecast = self.loss.domain_map(forecast)\n",
- "\n",
" if self.decompose_forecast:\n",
" # (n_batch, n_blocks, h, output_size)\n",
" block_forecasts = torch.stack(block_forecasts)\n",
@@ -516,6 +515,21 @@
"show_doc(NHITS.predict, name='NHITS.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(NHITS, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -611,7 +625,6 @@
"from neuralforecast.losses.pytorch import DistributionLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
- "\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
diff --git a/nbs/models.nlinear.ipynb b/nbs/models.nlinear.ipynb
index b55d42204..fc67b409a 100644
--- a/nbs/models.nlinear.ipynb
+++ b/nbs/models.nlinear.ipynb
@@ -53,7 +53,7 @@
"\n",
"import torch.nn as nn\n",
"\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -65,8 +65,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -76,7 +79,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class NLinear(BaseWindows):\n",
+ "class NLinear(BaseModel):\n",
" \"\"\" NLinear\n",
"\n",
" *Parameters:*
\n",
@@ -113,10 +116,11 @@
"\t- Zeng, Ailing, et al. \"Are transformers effective for time series forecasting?.\" Proceedings of the AAAI conference on artificial intelligence. Vol. 37. No. 9. 2023.\"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -188,11 +192,7 @@
"\n",
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
- " insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
- " #futr_exog = windows_batch['futr_exog']\n",
+ " insample_y = windows_batch['insample_y'].squeeze(-1)\n",
"\n",
" # Parse inputs\n",
" batch_size = len(insample_y)\n",
@@ -204,7 +204,6 @@
" # Final\n",
" forecast = self.linear(norm_insample_y) + last_value\n",
" forecast = forecast.reshape(batch_size, self.h, self.loss.outputsize_multiplier)\n",
- " forecast = self.loss.domain_map(forecast)\n",
" return forecast"
]
},
@@ -235,6 +234,21 @@
"show_doc(NLinear.predict, name='NLinear.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(NLinear, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -254,7 +268,7 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import NLinear\n",
- "from neuralforecast.losses.pytorch import MQLoss, DistributionLoss\n",
+ "from neuralforecast.losses.pytorch import DistributionLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic, augment_calendar_df\n",
"\n",
"AirPassengersPanel, calendar_cols = augment_calendar_df(df=AirPassengersPanel, freq='M')\n",
@@ -264,8 +278,7 @@
"\n",
"model = NLinear(h=12,\n",
" input_size=24,\n",
- " loss=MAE(),\n",
- " #loss=DistributionLoss(distribution='StudentT', level=[80, 90], return_params=True),\n",
+ " loss=DistributionLoss(distribution='StudentT', level=[80, 90], return_params=True),\n",
" scaler_type='robust',\n",
" learning_rate=1e-3,\n",
" max_steps=500,\n",
diff --git a/nbs/models.patchtst.ipynb b/nbs/models.patchtst.ipynb
index 31064cc24..bd6f2a35f 100644
--- a/nbs/models.patchtst.ipynb
+++ b/nbs/models.patchtst.ipynb
@@ -61,7 +61,7 @@
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import RevIN\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
@@ -74,8 +74,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -611,7 +614,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class PatchTST(BaseWindows):\n",
+ "class PatchTST(BaseModel):\n",
" \"\"\" PatchTST\n",
"\n",
" The PatchTST model is an efficient Transformer-based model for multivariate time series forecasting.\n",
@@ -673,10 +676,11 @@
" -[Nie, Y., Nguyen, N. H., Sinthong, P., & Kalagnanam, J. (2022). \"A Time Series is Worth 64 Words: Long-term Forecasting with Transformers\"](https://arxiv.org/pdf/2211.14730.pdf)\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -789,21 +793,11 @@
" def forward(self, windows_batch): # x: [batch, input_size]\n",
"\n",
" # Parse windows_batch\n",
- " insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
- " #futr_exog = windows_batch['futr_exog']\n",
- "\n",
- " # Add dimension for channel\n",
- " x = insample_y.unsqueeze(-1) # [Ws,L,1]\n",
+ " x = windows_batch['insample_y']\n",
"\n",
" x = x.permute(0,2,1) # x: [Batch, 1, input_size]\n",
" x = self.model(x)\n",
- " x = x.reshape(x.shape[0], self.h, -1) # x: [Batch, h, c_out]\n",
- "\n",
- " # Domain map\n",
- " forecast = self.loss.domain_map(x)\n",
+ " forecast = x.reshape(x.shape[0], self.h, -1) # x: [Batch, h, c_out]\n",
" \n",
" return forecast"
]
@@ -835,6 +829,21 @@
"show_doc(PatchTST.predict, name='PatchTST.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(PatchTST, [\"airpassengers\"])"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -872,7 +881,6 @@
" n_heads=4,\n",
" scaler_type='robust',\n",
" loss=DistributionLoss(distribution='StudentT', level=[80, 90]),\n",
- " #loss=MAE(),\n",
" learning_rate=1e-3,\n",
" max_steps=500,\n",
" val_check_steps=50,\n",
diff --git a/nbs/models.rmok.ipynb b/nbs/models.rmok.ipynb
index 017477c13..96dd6e195 100644
--- a/nbs/models.rmok.ipynb
+++ b/nbs/models.rmok.ipynb
@@ -37,8 +37,8 @@
"# Reversible Mixture of KAN - RMoK\n",
"The Reversible Mixture of KAN (RMoK) is a KAN-based model for time series forecasting which uses a mixture-of-experts structure to assign variables to different KAN experts, such as WaveKAN, TaylorKAN and JacobiKAN.\n",
"\n",
- "**Reference**\n",
- "- [Xiao Han, Xinfeng Zhang, Yiling Wu, Zhenduo Zhang, Zhe Wu.\"KAN4TSF: Are KAN and KAN-based models Effective for Time Series Forecasting?\"](https://arxiv.org/abs/2408.11306)"
+ "**References**
\n",
+ "[Xiao Han, Xinfeng Zhang, Yiling Wu, Zhenduo Zhang, Zhe Wu.\"KAN4TSF: Are KAN and KAN-based models Effective for Time Series Forecasting?\"](https://arxiv.org/abs/2408.11306)
"
]
},
{
@@ -55,8 +55,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -73,8 +76,9 @@
"import torch.nn.functional as F\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate\n",
- "from neuralforecast.common._modules import RevIN"
+ "from neuralforecast.common._base_model import BaseModel\n",
+ "from neuralforecast.common._modules import RevINMultivariate\n",
+ "from typing import Optional"
]
},
{
@@ -331,9 +335,11 @@
"source": [
"#| export\n",
"\n",
- "class RMoK(BaseMultivariate):\n",
+ "class RMoK(BaseModel):\n",
" \"\"\" Reversible Mixture of KAN\n",
- " **Parameters**
\n",
+ " \n",
+ " \n",
+ " **Parameters:**
\n",
" `h`: int, Forecast horizon.
\n",
" `input_size`: int, autorregresive inputs size, y=[1,2,3,4] input_size=2 -> y_[t-2:t]=[1,2].
\n",
" `n_series`: int, number of time-series.
\n",
@@ -353,6 +359,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=1024, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=1024, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -366,20 +376,21 @@
" `dataloader_kwargs`: dict, optional, list of parameters passed into the PyTorch Lightning dataloader by the `TimeSeriesDataLoader`.
\n",
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
"\n",
- " Reference
\n",
- " [Xiao Han, Xinfeng Zhang, Yiling Wu, Zhenduo Zhang, Zhe Wu.\"KAN4TSF: Are KAN and KAN-based models Effective for Time Series Forecasting?\"](https://arxiv.org/abs/2408.11306)\n",
+ " **References**
\n",
+ " - [Xiao Han, Xinfeng Zhang, Yiling Wu, Zhenduo Zhang, Zhe Wu.\"KAN4TSF: Are KAN and KAN-based models Effective for Time Series Forecasting?\". arXiv.](https://arxiv.org/abs/2408.11306)
\n",
" \"\"\"\n",
"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
" input_size,\n",
- " n_series,\n",
+ " n_series: int,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
@@ -396,6 +407,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 1024,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -422,6 +437,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" random_seed=random_seed,\n",
@@ -445,35 +464,31 @@
" self.wavelet_function = wavelet_function\n",
"\n",
" self.experts = nn.ModuleList([\n",
- " TaylorKANLayer(self.input_size, self.h, order=self.taylor_order, addbias=True),\n",
- " JacobiKANLayer(self.input_size, self.h, degree=self.jacobi_degree),\n",
- " WaveKANLayer(self.input_size, self.h, wavelet_type=self.wavelet_function),\n",
- " nn.Linear(self.input_size, self.h),\n",
+ " TaylorKANLayer(self.input_size, self.h * self.loss.outputsize_multiplier, order=self.taylor_order, addbias=True),\n",
+ " JacobiKANLayer(self.input_size, self.h * self.loss.outputsize_multiplier, degree=self.jacobi_degree),\n",
+ " WaveKANLayer(self.input_size, self.h * self.loss.outputsize_multiplier, wavelet_type=self.wavelet_function),\n",
+ " nn.Linear(self.input_size, self.h * self.loss.outputsize_multiplier),\n",
" ])\n",
" \n",
" self.num_experts = len(self.experts)\n",
" self.gate = nn.Linear(self.input_size, self.num_experts)\n",
" self.softmax = nn.Softmax(dim=-1)\n",
- " self.rev = RevIN(self.n_series, affine=self.revin_affine)\n",
+ " self.rev = RevINMultivariate(self.n_series, affine=self.revin_affine)\n",
"\n",
" def forward(self, windows_batch):\n",
" insample_y = windows_batch['insample_y']\n",
" B, L, N = insample_y.shape\n",
- " x = self.rev(insample_y, 'norm') if self.rev else insample_y\n",
+ " x = self.rev(insample_y, 'norm')\n",
" x = self.dropout(x).transpose(1, 2).reshape(B * N, L)\n",
"\n",
" score = F.softmax(self.gate(x), dim=-1)\n",
" expert_outputs = torch.stack([self.experts[i](x) for i in range(self.num_experts)], dim=-1)\n",
"\n",
- " y_pred = torch.einsum(\"BLE,BE->BL\", expert_outputs, score).reshape(B, N, -1).permute(0, 2, 1)\n",
+ " y_pred = torch.einsum(\"BLE, BE -> BL\", expert_outputs, score).reshape(B, N, self.h * self.loss.outputsize_multiplier).permute(0, 2, 1)\n",
" y_pred = self.rev(y_pred, 'denorm')\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
+ " y_pred = y_pred.reshape(B, self.h, -1)\n",
"\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " if y_pred.ndim == 2:\n",
- " return y_pred.unsqueeze(-1)\n",
- " else:\n",
- " return y_pred"
+ " return y_pred"
]
},
{
@@ -503,6 +518,21 @@
"show_doc(RMoK.predict, name='RMoK.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(RMoK, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -560,13 +590,6 @@
"ax.legend(prop={'size': 15})\n",
"ax.grid()"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/nbs/models.rnn.ipynb b/nbs/models.rnn.ipynb
index f5e1a67b9..8a92fdfb2 100644
--- a/nbs/models.rnn.ipynb
+++ b/nbs/models.rnn.ipynb
@@ -61,8 +61,10 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "from fastcore.test import test_eq\n",
"from nbdev.showdoc import show_doc\n",
- "from neuralforecast.utils import generate_series"
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -76,9 +78,10 @@
"\n",
"import torch\n",
"import torch.nn as nn\n",
+ "import warnings\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_recurrent import BaseRecurrent\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import MLP"
]
},
@@ -89,7 +92,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class RNN(BaseRecurrent):\n",
+ "class RNN(BaseModel):\n",
" \"\"\" RNN\n",
"\n",
" Multi Layer Elman RNN (RNN), with MLP decoder.\n",
@@ -106,7 +109,7 @@
" `encoder_activation`: str=`tanh`, type of RNN activation from `tanh` or `relu`.
\n",
" `encoder_bias`: bool=True, whether or not to use biases b_ih, b_hh within RNN units.
\n",
" `encoder_dropout`: float=0., dropout regularization applied to RNN outputs.
\n",
- " `context_size`: int=10, size of context vector for each timestamp on the forecasting window.
\n",
+ " `context_size`: deprecated.
\n",
" `decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
\n",
" `decoder_layers`: int=2, number of layers for the MLP decoder.
\n",
" `futr_exog_list`: str list, future exogenous columns.
\n",
@@ -135,26 +138,29 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'recurrent'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = True # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int,\n",
" input_size: int = -1,\n",
" inference_input_size: int = -1,\n",
" encoder_n_layers: int = 2,\n",
- " encoder_hidden_size: int = 200,\n",
+ " encoder_hidden_size: int = 128,\n",
" encoder_activation: str = 'tanh',\n",
" encoder_bias: bool = True,\n",
" encoder_dropout: float = 0.,\n",
- " context_size: int = 10,\n",
- " decoder_hidden_size: int = 200,\n",
+ " context_size: Optional[int] = None,\n",
+ " decoder_hidden_size: int = 128,\n",
" decoder_layers: int = 2,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
+ " recurrent = False,\n",
" loss = MAE(),\n",
" valid_loss = None,\n",
" max_steps: int = 1000,\n",
@@ -164,6 +170,10 @@
" val_check_steps: int = 100,\n",
" batch_size=32,\n",
" valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 128,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
+ " step_size: int = 1,\n",
" scaler_type: str='robust',\n",
" random_seed=1,\n",
" num_workers_loader=0,\n",
@@ -174,10 +184,16 @@
" lr_scheduler_kwargs = None, \n",
" dataloader_kwargs = None, \n",
" **trainer_kwargs):\n",
+ " \n",
+ " self.RECURRENT = recurrent\n",
+ "\n",
" super(RNN, self).__init__(\n",
" h=h,\n",
" input_size=input_size,\n",
- " inference_input_size=inference_input_size,\n",
+ " futr_exog_list=futr_exog_list,\n",
+ " hist_exog_list=hist_exog_list,\n",
+ " stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -187,13 +203,14 @@
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
" valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
+ " step_size=step_size,\n",
" scaler_type=scaler_type,\n",
- " futr_exog_list=futr_exog_list,\n",
- " hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list,\n",
+ " random_seed=random_seed,\n",
" num_workers_loader=num_workers_loader,\n",
" drop_last_loader=drop_last_loader,\n",
- " random_seed=random_seed,\n",
" optimizer=optimizer,\n",
" optimizer_kwargs=optimizer_kwargs,\n",
" lr_scheduler=lr_scheduler,\n",
@@ -208,7 +225,11 @@
" self.encoder_activation = encoder_activation\n",
" self.encoder_bias = encoder_bias\n",
" self.encoder_dropout = encoder_dropout\n",
- " \n",
+ "\n",
+ " # Context adapter\n",
+ " if context_size is not None:\n",
+ " warnings.warn(\"context_size is deprecated and will be removed in future versions.\")\n",
+ "\n",
" # Context adapter\n",
" self.context_size = context_size\n",
"\n",
@@ -217,69 +238,74 @@
" self.decoder_layers = decoder_layers\n",
"\n",
" # RNN input size (1 for target variable y)\n",
- " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size\n",
+ " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size\n",
"\n",
" # Instantiate model\n",
+ " self.rnn_state = None\n",
+ " self.maintain_state = False\n",
" self.hist_encoder = nn.RNN(input_size=input_encoder,\n",
- " hidden_size=self.encoder_hidden_size,\n",
- " num_layers=self.encoder_n_layers,\n",
- " nonlinearity=self.encoder_activation,\n",
- " bias=self.encoder_bias,\n",
- " dropout=self.encoder_dropout,\n",
- " batch_first=True)\n",
- "\n",
- " # Context adapter\n",
- " self.context_adapter = nn.Linear(in_features=self.encoder_hidden_size + self.futr_exog_size * h,\n",
- " out_features=self.context_size * h)\n",
+ " hidden_size=self.encoder_hidden_size,\n",
+ " num_layers=self.encoder_n_layers,\n",
+ " bias=self.encoder_bias,\n",
+ " dropout=self.encoder_dropout,\n",
+ " batch_first=True)\n",
"\n",
" # Decoder MLP\n",
- " self.mlp_decoder = MLP(in_features=self.context_size + self.futr_exog_size,\n",
- " out_features=self.loss.outputsize_multiplier,\n",
- " hidden_size=self.decoder_hidden_size,\n",
- " num_layers=self.decoder_layers,\n",
- " activation='ReLU',\n",
- " dropout=0.0)\n",
+ " if self.RECURRENT:\n",
+ " self.proj = nn.Linear(self.encoder_hidden_size, self.loss.outputsize_multiplier)\n",
+ " else:\n",
+ " self.mlp_decoder = MLP(in_features=self.encoder_hidden_size + self.futr_exog_size,\n",
+ " out_features=self.loss.outputsize_multiplier,\n",
+ " hidden_size=self.decoder_hidden_size,\n",
+ " num_layers=self.decoder_layers,\n",
+ " activation='ReLU',\n",
+ " dropout=0.0)\n",
"\n",
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
- " futr_exog = windows_batch['futr_exog']\n",
- " hist_exog = windows_batch['hist_exog']\n",
- " stat_exog = windows_batch['stat_exog']\n",
+ " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
+ " futr_exog = windows_batch['futr_exog'] # [B, seq_len, F]\n",
+ " hist_exog = windows_batch['hist_exog'] # [B, seq_len, X]\n",
+ " stat_exog = windows_batch['stat_exog'] # [B, S]\n",
"\n",
- " # Concatenate y, historic and static inputs\n",
- " # [B, C, seq_len, 1] -> [B, seq_len, C]\n",
- " # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]\n",
+ " # Concatenate y, historic and static inputs \n",
" batch_size, seq_len = encoder_input.shape[:2]\n",
" if self.hist_exog_size > 0:\n",
- " hist_exog = hist_exog.permute(0,2,1,3).squeeze(-1) # [B, X, seq_len, 1] -> [B, seq_len, X]\n",
- " encoder_input = torch.cat((encoder_input, hist_exog), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, hist_exog), dim=2) # [B, seq_len, 1] + [B, seq_len, X] -> [B, seq_len, 1 + X]\n",
"\n",
" if self.stat_exog_size > 0:\n",
- " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog), dim=2)\n",
- "\n",
- " # RNN forward\n",
- " hidden_state, _ = self.hist_encoder(encoder_input) # [B, seq_len, rnn_hidden_state]\n",
+ " # print(encoder_input.shape)\n",
+ " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
+ " encoder_input = torch.cat((encoder_input, stat_exog), dim=2) # [B, seq_len, 1 + X] + [B, seq_len, S] -> [B, seq_len, 1 + X + S]\n",
"\n",
" if self.futr_exog_size > 0:\n",
- " futr_exog = futr_exog.permute(0,2,3,1)[:,:,1:,:] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]\n",
- " hidden_state = torch.cat(( hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, \n",
+ " futr_exog[:, :seq_len]), dim=2) # [B, seq_len, 1 + X + S] + [B, seq_len, F] -> [B, seq_len, 1 + X + S + F]\n",
"\n",
- " # Context adapter\n",
- " context = self.context_adapter(hidden_state)\n",
- " context = context.reshape(batch_size, seq_len, self.h, self.context_size)\n",
+ " if self.RECURRENT:\n",
+ " if self.maintain_state:\n",
+ " rnn_state = self.rnn_state\n",
+ " else:\n",
+ " rnn_state = None\n",
+ " \n",
+ " output, rnn_state = self.hist_encoder(encoder_input, \n",
+ " rnn_state) # [B, seq_len, rnn_hidden_state]\n",
+ " output = self.proj(output) # [B, seq_len, rnn_hidden_state] -> [B, seq_len, n_output]\n",
+ " if self.maintain_state:\n",
+ " self.rnn_state = rnn_state\n",
+ " else:\n",
+ " hidden_state, _ = self.hist_encoder(encoder_input, None) # [B, seq_len, rnn_hidden_state]\n",
+ " hidden_state = hidden_state[:, -self.h:] # [B, seq_len, rnn_hidden_state] -> [B, h, rnn_hidden_state]\n",
+ " \n",
+ " if self.futr_exog_size > 0:\n",
+ " futr_exog_futr = futr_exog[:, -self.h:] # [B, h, F]\n",
+ " hidden_state = torch.cat((hidden_state, \n",
+ " futr_exog_futr), dim=-1) # [B, h, rnn_hidden_state] + [B, h, F] -> [B, h, rnn_hidden_state + F]\n",
"\n",
- " # Residual connection with futr_exog\n",
- " if self.futr_exog_size > 0:\n",
- " context = torch.cat((context, futr_exog), dim=-1)\n",
+ " output = self.mlp_decoder(hidden_state) # [B, h, rnn_hidden_state + F] -> [B, seq_len, n_output]\n",
"\n",
- " # Final forecast\n",
- " output = self.mlp_decoder(context)\n",
- " output = self.loss.domain_map(output)\n",
- " \n",
- " return output"
+ " return output[:, -self.h:]"
]
},
{
@@ -309,6 +335,21 @@
"show_doc(RNN.predict, name='RNN.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(RNN, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -328,26 +369,24 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import RNN\n",
- "from neuralforecast.losses.pytorch import MQLoss, DistributionLoss\n",
+ "from neuralforecast.losses.pytorch import MQLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
"fcst = NeuralForecast(\n",
" models=[RNN(h=12,\n",
- " input_size=-1,\n",
+ " input_size=24,\n",
" inference_input_size=24,\n",
" loss=MQLoss(level=[80, 90]),\n",
- " scaler_type='robust',\n",
+ " valid_loss=MQLoss(level=[80, 90]),\n",
+ " scaler_type='standard',\n",
" encoder_n_layers=2,\n",
" encoder_hidden_size=128,\n",
- " context_size=10,\n",
" decoder_hidden_size=128,\n",
" decoder_layers=2,\n",
- " max_steps=300,\n",
+ " max_steps=200,\n",
" futr_exog_list=['y_[lag12]'],\n",
- " #hist_exog_list=['y_[lag12]'],\n",
" stat_exog_list=['airline1'],\n",
" )\n",
" ],\n",
diff --git a/nbs/models.softs.ipynb b/nbs/models.softs.ipynb
index 978f3c2c2..588bd8dcb 100644
--- a/nbs/models.softs.ipynb
+++ b/nbs/models.softs.ipynb
@@ -27,8 +27,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -57,8 +60,9 @@
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
+ "from typing import Optional\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import TransEncoder, TransEncoderLayer"
]
},
@@ -134,7 +138,7 @@
"\n",
" # stochastic pooling\n",
" if self.training:\n",
- " ratio = F.softmax(combined_mean, dim=1)\n",
+ " ratio = F.softmax(torch.nan_to_num(combined_mean), dim=1)\n",
" ratio = ratio.permute(0, 2, 1)\n",
" ratio = ratio.reshape(-1, channels)\n",
" indices = torch.multinomial(ratio, 1)\n",
@@ -169,7 +173,7 @@
"source": [
"#| export\n",
"\n",
- "class SOFTS(BaseMultivariate):\n",
+ "class SOFTS(BaseModel):\n",
"\n",
" \"\"\" SOFTS\n",
" \n",
@@ -194,6 +198,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -212,10 +220,11 @@
" \"\"\"\n",
"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = True\n",
+ " RECURRENT = False\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -224,6 +233,7 @@
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" hidden_size: int = 512,\n",
" d_core: int = 512,\n",
" e_layers: int = 2,\n",
@@ -238,6 +248,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 256,\n",
+ " inference_windows_batch_size = 256,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -256,6 +270,7 @@
" stat_exog_list = None,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -264,6 +279,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" random_seed=random_seed,\n",
@@ -299,7 +318,7 @@
" ]\n",
" )\n",
"\n",
- " self.projection = nn.Linear(hidden_size, self.h, bias=True)\n",
+ " self.projection = nn.Linear(hidden_size, self.h * self.loss.outputsize_multiplier, bias=True)\n",
"\n",
" def forecast(self, x_enc):\n",
" # Normalization from Non-stationary Transformer\n",
@@ -316,22 +335,19 @@
"\n",
" # De-Normalization from Non-stationary Transformer\n",
" if self.use_norm:\n",
- " dec_out = dec_out * (stdev[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))\n",
- " dec_out = dec_out + (means[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))\n",
+ " dec_out = dec_out * (stdev[:, 0, :].unsqueeze(1).repeat(1, self.h * self.loss.outputsize_multiplier, 1))\n",
+ " dec_out = dec_out + (means[:, 0, :].unsqueeze(1).repeat(1, self.h * self.loss.outputsize_multiplier, 1))\n",
" return dec_out\n",
" \n",
" def forward(self, windows_batch):\n",
" insample_y = windows_batch['insample_y']\n",
"\n",
" y_pred = self.forecast(insample_y)\n",
- " y_pred = y_pred[:, -self.h:, :]\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
+ " y_pred = y_pred.reshape(insample_y.shape[0],\n",
+ " self.h,\n",
+ " -1)\n",
"\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " if y_pred.ndim == 2:\n",
- " return y_pred.unsqueeze(-1)\n",
- " else:\n",
- " return y_pred"
+ " return y_pred"
]
},
{
@@ -361,6 +377,21 @@
"show_doc(SOFTS.predict, name='SOFTS.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(SOFTS, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -381,9 +412,7 @@
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import SOFTS\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "from neuralforecast.losses.pytorch import MSE\n",
- "\n",
- "\n",
+ "from neuralforecast.losses.pytorch import MASE\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
"\n",
@@ -396,8 +425,7 @@
" d_ff=64,\n",
" dropout=0.1,\n",
" use_norm=True,\n",
- " loss=MSE(),\n",
- " valid_loss=MAE(),\n",
+ " loss=MASE(seasonality=4),\n",
" early_stop_patience_steps=3,\n",
" batch_size=32)\n",
"\n",
diff --git a/nbs/models.stemgnn.ipynb b/nbs/models.stemgnn.ipynb
index b2222fc1c..1e97e9bca 100644
--- a/nbs/models.stemgnn.ipynb
+++ b/nbs/models.stemgnn.ipynb
@@ -53,8 +53,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -68,8 +71,9 @@
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
+ "from typing import Optional\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate"
+ "from neuralforecast.common._base_model import BaseModel"
]
},
{
@@ -171,7 +175,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class StemGNN(BaseMultivariate):\n",
+ "class StemGNN(BaseModel):\n",
" \"\"\" StemGNN\n",
"\n",
" The Spectral Temporal Graph Neural Network (`StemGNN`) is a Graph-based multivariate\n",
@@ -198,6 +202,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int, number of windows in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=1024, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=1024, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='robust', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int, random_seed for pytorch initializer and numpy generators.
\n",
@@ -212,10 +220,11 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False \n",
+ " MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
" \n",
" def __init__(self,\n",
" h,\n",
@@ -224,6 +233,7 @@
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" n_stacks = 2,\n",
" multi_layer: int = 5,\n",
" dropout_rate: float = 0.5,\n",
@@ -236,6 +246,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 1024,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'robust',\n",
" random_seed: int = 1,\n",
@@ -254,7 +268,8 @@
" n_series=n_series,\n",
" futr_exog_list=futr_exog_list,\n",
" hist_exog_list=hist_exog_list,\n",
- " stat_exog_list=stat_exog_list, \n",
+ " stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y, \n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -263,6 +278,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" num_workers_loader=num_workers_loader,\n",
@@ -379,14 +398,8 @@
"\n",
" forecast = forecast.permute(0, 2, 1).contiguous()\n",
" forecast = forecast.reshape(batch_size, self.h, self.loss.outputsize_multiplier * self.n_series)\n",
- " forecast = self.loss.domain_map(forecast)\n",
"\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.\n",
- " if forecast.ndim == 2:\n",
- " return forecast.unsqueeze(-1)\n",
- " else:\n",
- " return forecast"
+ " return forecast"
]
},
{
@@ -423,73 +436,12 @@
"outputs": [],
"source": [
"#| hide\n",
- "import logging\n",
- "import warnings\n",
- "\n",
- "from neuralforecast import NeuralForecast\n",
- "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "from neuralforecast.losses.pytorch import MAE, MSE, RMSE, MAPE, SMAPE, MASE, relMSE, QuantileLoss, MQLoss, DistributionLoss,PMM, GMM, NBMM, HuberLoss, TukeyLoss, HuberQLoss, HuberMQLoss"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# Test losses\n",
+ "# Unit tests for models\n",
"logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
- "warnings.filterwarnings(\"ignore\")\n",
- "\n",
- "Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
- "\n",
- "AirPassengersStatic_single = AirPassengersStatic[AirPassengersStatic[\"unique_id\"] == 'Airline1']\n",
- "Y_train_df_single = Y_train_df[Y_train_df[\"unique_id\"] == 'Airline1']\n",
- "Y_test_df_single = Y_test_df[Y_test_df[\"unique_id\"] == 'Airline1']\n",
- "\n",
- "losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "valid_losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "\n",
- "for loss, valid_loss in zip(losses, valid_losses):\n",
- " try:\n",
- " model = StemGNN(h=12,\n",
- " input_size=24,\n",
- " n_series=2,\n",
- " scaler_type='robust',\n",
- " max_steps=2,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=10,\n",
- " learning_rate=1e-3,\n",
- " loss=loss,\n",
- " valid_loss=valid_loss,\n",
- " batch_size=32\n",
- " )\n",
- "\n",
- " fcst = NeuralForecast(models=[model], freq='M')\n",
- " fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
- " forecasts = fcst.predict(futr_df=Y_test_df)\n",
- " except Exception as e:\n",
- " assert str(e) == f\"{loss} is not supported in a Multivariate model.\"\n",
- "\n",
- "\n",
- "# Test n_series = 1\n",
- "model = StemGNN(h=12,\n",
- " input_size=24,\n",
- " n_series=1,\n",
- " scaler_type='robust',\n",
- " max_steps=2,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=10,\n",
- " learning_rate=1e-3,\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " batch_size=32\n",
- " )\n",
- "fcst = NeuralForecast(models=[model], freq='M')\n",
- "fcst.fit(df=Y_train_df_single, static_df=AirPassengersStatic_single, val_size=12)\n",
- "forecasts = fcst.predict(futr_df=Y_test_df_single) "
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(StemGNN, [\"airpassengers\"])"
]
},
{
@@ -527,13 +479,13 @@
"model = StemGNN(h=12,\n",
" input_size=24,\n",
" n_series=2,\n",
- " scaler_type='robust',\n",
- " max_steps=100,\n",
+ " scaler_type='standard',\n",
+ " max_steps=500,\n",
" early_stop_patience_steps=-1,\n",
" val_check_steps=10,\n",
" learning_rate=1e-3,\n",
" loss=MAE(),\n",
- " valid_loss=None,\n",
+ " valid_loss=MAE(),\n",
" batch_size=32\n",
" )\n",
"\n",
diff --git a/nbs/models.tcn.ipynb b/nbs/models.tcn.ipynb
index dee324513..61551f1f5 100644
--- a/nbs/models.tcn.ipynb
+++ b/nbs/models.tcn.ipynb
@@ -69,7 +69,7 @@
"import torch.nn as nn\n",
"\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_recurrent import BaseRecurrent\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import MLP, TemporalConvolutionEncoder"
]
},
@@ -80,10 +80,11 @@
"outputs": [],
"source": [
"#| hide\n",
- "from nbdev.showdoc import show_doc\n",
- "\n",
"import logging\n",
- "import warnings"
+ "import warnings\n",
+ "from fastcore.test import test_eq\n",
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -93,7 +94,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class TCN(BaseRecurrent):\n",
+ "class TCN(BaseModel):\n",
" \"\"\" TCN\n",
"\n",
" Temporal Convolution Network (TCN), with MLP decoder.\n",
@@ -134,21 +135,22 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'recurrent'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True \n",
- " \n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False) \n",
+ "\n",
" def __init__(self,\n",
" h: int,\n",
" input_size: int = -1,\n",
" inference_input_size: int = -1,\n",
" kernel_size: int = 2,\n",
" dilations: List[int] = [1, 2, 4, 8, 16],\n",
- " encoder_hidden_size: int = 200,\n",
+ " encoder_hidden_size: int = 128,\n",
" encoder_activation: str = 'ReLU',\n",
" context_size: int = 10,\n",
- " decoder_hidden_size: int = 200,\n",
+ " decoder_hidden_size: int = 128,\n",
" decoder_layers: int = 2,\n",
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
@@ -162,6 +164,10 @@
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
" valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 128,\n",
+ " inference_windows_batch_size = 1024,\n",
+ " start_padding_enabled = False,\n",
+ " step_size: int = 1, \n",
" scaler_type: str ='robust',\n",
" random_seed: int = 1,\n",
" num_workers_loader = 0,\n",
@@ -185,6 +191,10 @@
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
" valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
+ " step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" futr_exog_list=futr_exog_list,\n",
" hist_exog_list=hist_exog_list,\n",
@@ -215,7 +225,7 @@
" self.decoder_layers = decoder_layers\n",
"\n",
" # TCN input size (1 for target variable y)\n",
- " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size\n",
+ " input_encoder = 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size\n",
"\n",
" \n",
" #---------------------------------- Instantiate Model -----------------------------------#\n",
@@ -228,11 +238,11 @@
" activation=self.encoder_activation)\n",
"\n",
" # Context adapter\n",
- " self.context_adapter = nn.Linear(in_features=self.encoder_hidden_size + self.futr_exog_size * h,\n",
- " out_features=self.context_size * h)\n",
+ " self.context_adapter = nn.Linear(in_features=self.input_size,\n",
+ " out_features=h)\n",
"\n",
" # Decoder MLP\n",
- " self.mlp_decoder = MLP(in_features=self.context_size + self.futr_exog_size,\n",
+ " self.mlp_decoder = MLP(in_features=self.encoder_hidden_size + self.futr_exog_size,\n",
" out_features=self.loss.outputsize_multiplier,\n",
" hidden_size=self.decoder_hidden_size,\n",
" num_layers=self.decoder_layers,\n",
@@ -242,41 +252,41 @@
" def forward(self, windows_batch):\n",
" \n",
" # Parse windows_batch\n",
- " encoder_input = windows_batch['insample_y'] # [B, seq_len, 1]\n",
- " futr_exog = windows_batch['futr_exog']\n",
- " hist_exog = windows_batch['hist_exog']\n",
- " stat_exog = windows_batch['stat_exog']\n",
+ " encoder_input = windows_batch['insample_y'] # [B, L, 1]\n",
+ " futr_exog = windows_batch['futr_exog'] # [B, L + h, F]\n",
+ " hist_exog = windows_batch['hist_exog'] # [B, L, X]\n",
+ " stat_exog = windows_batch['stat_exog'] # [B, S]\n",
"\n",
- " # Concatenate y, historic and static inputs\n",
- " # [B, C, seq_len, 1] -> [B, seq_len, C]\n",
- " # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]\n",
- " batch_size, seq_len = encoder_input.shape[:2]\n",
+ " # Concatenate y, historic and static inputs \n",
+ " batch_size, input_size = encoder_input.shape[:2]\n",
" if self.hist_exog_size > 0:\n",
- " hist_exog = hist_exog.permute(0,2,1,3).squeeze(-1) # [B, X, seq_len, 1] -> [B, seq_len, X]\n",
- " encoder_input = torch.cat((encoder_input, hist_exog), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, hist_exog), dim=2) # [B, L, 1] + [B, L, X] -> [B, L, 1 + X]\n",
"\n",
" if self.stat_exog_size > 0:\n",
- " stat_exog = stat_exog.unsqueeze(1).repeat(1, seq_len, 1) # [B, S] -> [B, seq_len, S]\n",
- " encoder_input = torch.cat((encoder_input, stat_exog), dim=2)\n",
- "\n",
- " # TCN forward\n",
- " hidden_state = self.hist_encoder(encoder_input) # [B, seq_len, tcn_hidden_state]\n",
+ " # print(encoder_input.shape)\n",
+ " stat_exog = stat_exog.unsqueeze(1).repeat(1, input_size, 1) # [B, S] -> [B, L, S]\n",
+ " encoder_input = torch.cat((encoder_input, stat_exog), dim=2) # [B, L, 1 + X] + [B, L, S] -> [B, L, 1 + X + S]\n",
"\n",
" if self.futr_exog_size > 0:\n",
- " futr_exog = futr_exog.permute(0,2,3,1)[:,:,1:,:] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]\n",
- " hidden_state = torch.cat(( hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2)\n",
+ " encoder_input = torch.cat((encoder_input, \n",
+ " futr_exog[:, :input_size]), dim=2) # [B, L, 1 + X + S] + [B, L, F] -> [B, L, 1 + X + S + F]\n",
+ "\n",
+ " # TCN forward \n",
+ " hidden_state = self.hist_encoder(encoder_input) # [B, L, C]\n",
"\n",
" # Context adapter\n",
- " context = self.context_adapter(hidden_state)\n",
- " context = context.reshape(batch_size, seq_len, self.h, self.context_size)\n",
+ " hidden_state = hidden_state.permute(0, 2, 1) # [B, L, C] -> [B, C, L]\n",
+ " context = self.context_adapter(hidden_state) # [B, C, L] -> [B, C, h]\n",
"\n",
" # Residual connection with futr_exog\n",
" if self.futr_exog_size > 0:\n",
- " context = torch.cat((context, futr_exog), dim=-1)\n",
+ " futr_exog_futr = futr_exog[:, input_size:].swapaxes(1, 2) # [B, L + h, F] -> [B, F, h] \n",
+ " context = torch.cat((context, futr_exog_futr), dim=1) # [B, C, h] + [B, F, h] = [B, C + F, h]\n",
+ "\n",
+ " context = context.swapaxes(1, 2) # [B, C + F, h] -> [B, h, C + F]\n",
"\n",
" # Final forecast\n",
- " output = self.mlp_decoder(context)\n",
- " output = self.loss.domain_map(output)\n",
+ " output = self.mlp_decoder(context) # [B, h, C + F] -> [B, h, n_output]\n",
" \n",
" return output"
]
@@ -308,13 +318,6 @@
"show_doc(TCN.predict, name='TCN.predict')"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Usage Example"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -322,8 +325,19 @@
"outputs": [],
"source": [
"#| hide\n",
+ "# Unit tests for models\n",
"logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
- "warnings.filterwarnings(\"ignore\")"
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(TCN, [\"airpassengers\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Usage Example"
]
},
{
@@ -338,7 +352,7 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import TCN\n",
- "from neuralforecast.losses.pytorch import GMM, MQLoss, DistributionLoss\n",
+ "from neuralforecast.losses.pytorch import DistributionLoss\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds [B, h, n_outputs]\n",
"\n",
- " # Map to output domain\n",
- " forecast = self.loss.domain_map(x + x_skip)\n",
+ " forecast = x + x_skip\n",
" \n",
" return forecast\n"
]
@@ -383,6 +386,21 @@
"show_doc(TiDE.predict, name='TiDE.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(TiDE, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -402,7 +420,7 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import TiDE\n",
- "from neuralforecast.losses.pytorch import GMM, DistributionLoss\n",
+ "from neuralforecast.losses.pytorch import GMM\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds 1:\n",
+ " raise Exception('TimeLLM only supports point loss functions (MAE, MSE, etc) as loss function.') \n",
+ " \n",
+ " if valid_loss is not None and not isinstance(valid_loss, losses.BasePointLoss):\n",
+ " raise Exception('TimeLLM only supports point loss functions (MAE, MSE, etc) as valid loss function.') \n",
+ "\n",
+ "\n",
" # Architecture\n",
" self.patch_len = patch_len\n",
" self.stride = stride\n",
@@ -523,13 +533,10 @@
" return lags\n",
" \n",
" def forward(self, windows_batch):\n",
- " insample_y = windows_batch['insample_y']\n",
- "\n",
- " x = insample_y.unsqueeze(-1)\n",
+ " x = windows_batch['insample_y']\n",
"\n",
" y_pred = self.forecast(x)\n",
" y_pred = y_pred[:, -self.h:, :]\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
" \n",
" return y_pred\n"
]
@@ -575,11 +582,12 @@
"outputs": [],
"source": [
"#| eval: false\n",
+ "import pandas as pd\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import TimeLLM\n",
- "from neuralforecast.utils import AirPassengersPanel, augment_calendar_df\n",
- "\n",
- "AirPassengersPanel, calendar_cols = augment_calendar_df(df=AirPassengersPanel, freq='M')\n",
+ "from neuralforecast.utils import AirPassengersPanel\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
diff --git a/nbs/models.timemixer.ipynb b/nbs/models.timemixer.ipynb
index 9bfdd9cc5..49801a6b8 100644
--- a/nbs/models.timemixer.ipynb
+++ b/nbs/models.timemixer.ipynb
@@ -17,8 +17,8 @@
"\n",
"Seasonal and trend components exhibit significantly different characteristics in time series, and different scales of the time series reflect different properties, with seasonal characteristics being more pronounced at a fine-grained micro scale and trend characteristics being more pronounced at a coarse macro scale, it is therefore necessary to decouple seasonal and trend components at different scales. As such, TimeMixer is an MLP-based architecture with Past-Decomposable-Mixing (PDM) and Future-Multipredictor-Mixing (FMM) blocks to take full advantage of disentangled multiscale series in both past extraction and future prediction phases.\n",
"\n",
- "**Reference**\n",
- "- [Shiyu Wang, Haixu Wu, Xiaoming Shi, Tengge Hu, Huakun Luo, Lintao Ma, James Y. Zhang, Jun Zhou.\"TimeMixer: Decomposable Multiscale Mixing For Time Series Forecasting\"](https://openreview.net/pdf?id=7oLshfEIC2)"
+ "**References**
\n",
+ "[Shiyu Wang, Haixu Wu, Xiaoming Shi, Tengge Hu, Huakun Luo, Lintao Ma, James Y. Zhang, Jun Zhou.\"TimeMixer: Decomposable Multiscale Mixing For Time Series Forecasting\"](https://openreview.net/pdf?id=7oLshfEIC2)
"
]
},
{
@@ -41,10 +41,10 @@
"import torch\n",
"import torch.nn as nn\n",
"\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"from neuralforecast.common._modules import PositionalEmbedding, TokenEmbedding, TemporalEmbedding, SeriesDecomp, RevIN\n",
- "\n",
- "from neuralforecast.losses.pytorch import MAE"
+ "from neuralforecast.losses.pytorch import MAE\n",
+ "from typing import Optional"
]
},
{
@@ -54,8 +54,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -324,7 +327,7 @@
"source": [
"#| export\n",
"\n",
- "class TimeMixer(BaseMultivariate):\n",
+ "class TimeMixer(BaseModel):\n",
" \"\"\" TimeMixer\n",
" **Parameters**
\n",
" `h`: int, Forecast horizon.
\n",
@@ -354,6 +357,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -368,14 +375,15 @@
" `**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
\n",
"\n",
" **References**
\n",
- " [Shiyu Wang, Haixu Wu, Xiaoming Shi, Tengge Hu, Huakun Luo, Lintao Ma, James Y. Zhang, Jun Zhou.\"TimeMixer: Decomposable Multiscale Mixing For Time Series Forecasting\"](https://openreview.net/pdf?id=7oLshfEIC2)\n",
+ " [Shiyu Wang, Haixu Wu, Xiaoming Shi, Tengge Hu, Huakun Luo, Lintao Ma, James Y. Zhang, Jun Zhou.\"TimeMixer: Decomposable Multiscale Mixing For Time Series Forecasting\"](https://openreview.net/pdf?id=7oLshfEIC2)
\n",
" \"\"\"\n",
"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -405,6 +413,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 256,\n",
+ " inference_windows_batch_size = 256,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -431,6 +443,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" random_seed=random_seed,\n",
@@ -522,6 +538,9 @@
" for i in range(self.down_sampling_layers + 1)\n",
" ]\n",
" )\n",
+ " \n",
+ " if self.loss.outputsize_multiplier > 1:\n",
+ " self.distr_output = nn.Linear(self.n_series, self.n_series * self.loss.outputsize_multiplier)\n",
"\n",
" def out_projection(self, dec_out, i, out_res):\n",
" dec_out = self.projection_layer(dec_out)\n",
@@ -678,13 +697,10 @@
"\n",
" y_pred = self.forecast(insample_y, x_mark_enc, x_mark_dec)\n",
" y_pred = y_pred[:, -self.h:, :]\n",
- " y_pred = self.loss.domain_map(y_pred)\n",
+ " if self.loss.outputsize_multiplier > 1:\n",
+ " y_pred = self.distr_output(y_pred)\n",
"\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " if y_pred.ndim == 2:\n",
- " return y_pred.unsqueeze(-1)\n",
- " else:\n",
- " return y_pred"
+ " return y_pred\n"
]
},
{
@@ -714,6 +730,21 @@
"show_doc(TimeMixer.predict, name='TimeMixer.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(TimeMixer, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/nbs/models.timesnet.ipynb b/nbs/models.timesnet.ipynb
index 37e5d46e4..00d65688f 100644
--- a/nbs/models.timesnet.ipynb
+++ b/nbs/models.timesnet.ipynb
@@ -54,7 +54,7 @@
"import torch.fft\n",
"\n",
"from neuralforecast.common._modules import DataEmbedding\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -66,8 +66,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -200,7 +203,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class TimesNet(BaseWindows):\n",
+ "class TimesNet(BaseModel):\n",
" \"\"\" TimesNet\n",
"\n",
" The TimesNet univariate model tackles the challenge of modeling multiple intraperiod and interperiod temporal variations.\n",
@@ -279,10 +282,11 @@
" Haixu Wu and Tengge Hu and Yong Liu and Hang Zhou and Jianmin Wang and Mingsheng Long. TimesNet: Temporal 2D-Variation Modeling for General Time Series Analysis. https://openreview.net/pdf?id=ju_Uqw384Oq\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False \n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -377,13 +381,9 @@
"\n",
" # Parse windows_batch\n",
" insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
" futr_exog = windows_batch['futr_exog']\n",
"\n",
" # Parse inputs\n",
- " insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]\n",
" if self.futr_exog_size > 0:\n",
" x_mark_enc = futr_exog[:,:self.input_size,:]\n",
" else:\n",
@@ -398,7 +398,7 @@
" # porject back\n",
" dec_out = self.projection(enc_out)\n",
"\n",
- " forecast = self.loss.domain_map(dec_out[:, -self.h:])\n",
+ " forecast = dec_out[:, -self.h:]\n",
" return forecast"
]
},
@@ -429,6 +429,21 @@
"show_doc(TimesNet.predict, name='TimesNet.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(TimesNet, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -448,9 +463,7 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.losses.pytorch import DistributionLoss\n",
- "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic, augment_calendar_df\n",
- "\n",
- "AirPassengersPanel, calendar_cols = augment_calendar_df(df=AirPassengersPanel, freq='M')\n",
+ "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
@@ -460,10 +473,9 @@
" hidden_size = 16,\n",
" conv_hidden_size = 32,\n",
" loss=DistributionLoss(distribution='Normal', level=[80, 90]),\n",
- " futr_exog_list=calendar_cols,\n",
" scaler_type='standard',\n",
" learning_rate=1e-3,\n",
- " max_steps=5,\n",
+ " max_steps=100,\n",
" val_check_steps=50,\n",
" early_stop_patience_steps=2)\n",
"\n",
diff --git a/nbs/models.tsmixer.ipynb b/nbs/models.tsmixer.ipynb
index 94a9e4125..4c01a42f3 100644
--- a/nbs/models.tsmixer.ipynb
+++ b/nbs/models.tsmixer.ipynb
@@ -44,8 +44,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -55,12 +58,13 @@
"outputs": [],
"source": [
"#| export\n",
- "import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
+ "from typing import Optional\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate"
+ "from neuralforecast.common._base_model import BaseModel\n",
+ "from neuralforecast.common._modules import RevINMultivariate"
]
},
{
@@ -157,55 +161,6 @@
" return x"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 1.2 Reversible InstanceNormalization\n",
- "An Instance Normalization Layer that is reversible, based on [this reference implementation](https://github.com/google-research/google-research/blob/master/tsmixer/tsmixer_basic/models/rev_in.py).
"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "class ReversibleInstanceNorm1d(nn.Module):\n",
- " \"\"\" \n",
- " ReversibleInstanceNorm1d\n",
- " \"\"\" \n",
- " def __init__(self, n_series, eps=1e-5):\n",
- " super().__init__()\n",
- " self.weight = nn.Parameter(torch.ones((1, 1, n_series)))\n",
- " self.bias = nn.Parameter(torch.zeros((1, 1, n_series)))\n",
- "\n",
- " self.eps = eps\n",
- "\n",
- " def forward(self, x):\n",
- " # Batch statistics\n",
- " self.batch_mean = torch.mean(x, axis=1, keepdim=True).detach()\n",
- " self.batch_std = torch.sqrt(torch.var(x, axis=1, keepdim=True, unbiased=False) + self.eps).detach()\n",
- " \n",
- " # Instance normalization\n",
- " x = x - self.batch_mean\n",
- " x = x / self.batch_std\n",
- " x = x * self.weight\n",
- " x = x + self.bias\n",
- " \n",
- " return x\n",
- "\n",
- " def reverse(self, x):\n",
- " # Reverse the normalization\n",
- " x = x - self.bias\n",
- " x = x / self.weight \n",
- " x = x * self.batch_std\n",
- " x = x + self.batch_mean \n",
- "\n",
- " return x"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -220,7 +175,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class TSMixer(BaseMultivariate):\n",
+ "class TSMixer(BaseModel):\n",
" \"\"\" TSMixer\n",
"\n",
" Time-Series Mixer (`TSMixer`) is a MLP-based multivariate time-series forecasting model. `TSMixer` jointly learns temporal and cross-sectional representations of the time-series by repeatedly combining time- and feature information using stacked mixing layers. A mixing layer consists of a sequential time- and feature Multi Layer Perceptron (`MLP`).\n",
@@ -244,6 +199,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -262,10 +221,11 @@
"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = False\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -274,6 +234,7 @@
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" n_block = 2,\n",
" ff_dim = 64,\n",
" dropout = 0.9,\n",
@@ -286,6 +247,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 256,\n",
+ " inference_windows_batch_size = 256,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -305,6 +270,7 @@
" futr_exog_list=futr_exog_list,\n",
" hist_exog_list=hist_exog_list,\n",
" stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -313,6 +279,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" random_seed=random_seed,\n",
@@ -328,7 +298,7 @@
" # Reversible InstanceNormalization layer\n",
" self.revin = revin\n",
" if self.revin:\n",
- " self.norm = ReversibleInstanceNorm1d(n_series = n_series)\n",
+ " self.norm = RevINMultivariate(num_features = n_series, affine=True)\n",
"\n",
" # Mixing layers\n",
" mixing_layers = [MixingLayer(n_series=n_series, \n",
@@ -349,23 +319,17 @@
"\n",
" # TSMixer: InstanceNorm + Mixing layers + Dense output layer + ReverseInstanceNorm\n",
" if self.revin:\n",
- " x = self.norm(x)\n",
+ " x = self.norm(x, 'norm')\n",
" x = self.mixing_layers(x)\n",
" x = x.permute(0, 2, 1)\n",
" x = self.out(x)\n",
" x = x.permute(0, 2, 1)\n",
" if self.revin:\n",
- " x = self.norm.reverse(x)\n",
+ " x = self.norm(x, 'denorm')\n",
"\n",
" x = x.reshape(batch_size, self.h, self.loss.outputsize_multiplier * self.n_series)\n",
- " forecast = self.loss.domain_map(x)\n",
- "\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.\n",
- " if forecast.ndim == 2:\n",
- " return forecast.unsqueeze(-1)\n",
- " else:\n",
- " return forecast"
+ "\n",
+ " return x"
]
},
{
@@ -401,80 +365,12 @@
"metadata": {},
"outputs": [],
"source": [
- "#| hide\n",
- "import logging\n",
- "import warnings\n",
- "\n",
- "from neuralforecast import NeuralForecast\n",
- "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "from neuralforecast.losses.pytorch import MAE, MSE, RMSE, MAPE, SMAPE, MASE, relMSE, QuantileLoss, MQLoss, DistributionLoss,PMM, GMM, NBMM, HuberLoss, TukeyLoss, HuberQLoss, HuberMQLoss"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# Test losses\n",
+ "# Unit tests for models\n",
"logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
- "warnings.filterwarnings(\"ignore\")\n",
- "\n",
- "Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
- "\n",
- "AirPassengersStatic_single = AirPassengersStatic[AirPassengersStatic[\"unique_id\"] == 'Airline1']\n",
- "Y_train_df_single = Y_train_df[Y_train_df[\"unique_id\"] == 'Airline1']\n",
- "Y_test_df_single = Y_test_df[Y_test_df[\"unique_id\"] == 'Airline1']\n",
- "\n",
- "losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "valid_losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "\n",
- "for loss, valid_loss in zip(losses, valid_losses):\n",
- " try:\n",
- " model = TSMixer(h=12,\n",
- " input_size=24,\n",
- " n_series=2,\n",
- " n_block=4,\n",
- " ff_dim=4,\n",
- " revin=True,\n",
- " scaler_type='standard',\n",
- " max_steps=2,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=5,\n",
- " learning_rate=1e-3,\n",
- " loss=loss,\n",
- " valid_loss=valid_loss,\n",
- " batch_size=32\n",
- " )\n",
- "\n",
- " fcst = NeuralForecast(models=[model], freq='M')\n",
- " fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
- " forecasts = fcst.predict(futr_df=Y_test_df)\n",
- " except Exception as e:\n",
- " assert str(e) == f\"{loss} is not supported in a Multivariate model.\"\n",
- "\n",
- "\n",
- "# Test n_series = 1\n",
- "model = TSMixer(h=12,\n",
- " input_size=24,\n",
- " n_series=1,\n",
- " n_block=4,\n",
- " ff_dim=4,\n",
- " revin=True,\n",
- " scaler_type='standard',\n",
- " max_steps=2,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=5,\n",
- " learning_rate=1e-3,\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " batch_size=32\n",
- " )\n",
- "fcst = NeuralForecast(models=[model], freq='M')\n",
- "fcst.fit(df=Y_train_df_single, static_df=AirPassengersStatic_single, val_size=12)\n",
- "forecasts = fcst.predict(futr_df=Y_test_df_single)"
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(TSMixer, [\"airpassengers\"])"
]
},
{
@@ -504,7 +400,7 @@
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import TSMixer\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "from neuralforecast.losses.pytorch import MAE\n",
+ "from neuralforecast.losses.pytorch import MAE, MQLoss\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
@@ -521,8 +417,7 @@
" early_stop_patience_steps=-1,\n",
" val_check_steps=5,\n",
" learning_rate=1e-3,\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
+ " loss=MQLoss(),\n",
" batch_size=32\n",
" )\n",
"\n",
@@ -536,9 +431,13 @@
"plot_df = pd.concat([Y_test_df, Y_hat_df], axis=1)\n",
"plot_df = pd.concat([Y_train_df, plot_df])\n",
"\n",
- "plot_df = plot_df[plot_df.unique_id=='Airline1'].drop('unique_id', axis=1)\n",
+ "plot_df = plot_df[plot_df.unique_id=='Airline2'].drop('unique_id', axis=1)\n",
"plt.plot(plot_df['ds'], plot_df['y'], c='black', label='True')\n",
- "plt.plot(plot_df['ds'], plot_df['TSMixer'], c='blue', label='Forecast')\n",
+ "plt.plot(plot_df['ds'], plot_df['TSMixer-median'], c='blue', label='median')\n",
+ "plt.fill_between(x=plot_df['ds'][-12:], \n",
+ " y1=plot_df['TSMixer-lo-90'][-12:].values,\n",
+ " y2=plot_df['TSMixer-hi-90'][-12:].values,\n",
+ " alpha=0.4, label='level 90')\n",
"ax.set_title('AirPassengers Forecast', fontsize=22)\n",
"ax.set_ylabel('Monthly Passengers', fontsize=20)\n",
"ax.set_xlabel('Year', fontsize=20)\n",
@@ -569,7 +468,7 @@
"Y_df = AirPassengersPanel[AirPassengersPanel['unique_id']=='Airline1']\n",
"\n",
"plt.plot(Y_df['ds'], Y_df['y'], c='black', label='True')\n",
- "plt.plot(Y_hat_df['ds'], Y_hat_df['TSMixer'], c='blue', label='Forecast')\n",
+ "plt.plot(Y_hat_df['ds'], Y_hat_df['TSMixer-median'], c='blue', label='Forecast')\n",
"ax.set_title('AirPassengers Forecast', fontsize=22)\n",
"ax.set_ylabel('Monthly Passengers', fontsize=20)\n",
"ax.set_xlabel('Year', fontsize=20)\n",
diff --git a/nbs/models.tsmixerx.ipynb b/nbs/models.tsmixerx.ipynb
index cb0ba72b6..691bdbc32 100644
--- a/nbs/models.tsmixerx.ipynb
+++ b/nbs/models.tsmixerx.ipynb
@@ -44,8 +44,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -59,8 +62,10 @@
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"\n",
+ "from typing import Optional\n",
"from neuralforecast.losses.pytorch import MAE\n",
- "from neuralforecast.common._base_multivariate import BaseMultivariate"
+ "from neuralforecast.common._base_model import BaseModel\n",
+ "from neuralforecast.common._modules import RevINMultivariate"
]
},
{
@@ -244,7 +249,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class TSMixerx(BaseMultivariate):\n",
+ "class TSMixerx(BaseModel):\n",
" \"\"\" TSMixerx\n",
"\n",
" Time-Series Mixer exogenous (`TSMixerx`) is a MLP-based multivariate time-series forecasting model, with capability for additional exogenous inputs. `TSMixerx` jointly learns temporal and cross-sectional representations of the time-series by repeatedly combining time- and feature information using stacked mixing layers. A mixing layer consists of a sequential time- and feature Multi Layer Perceptron (`MLP`).\n",
@@ -268,6 +273,10 @@
" `early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
\n",
" `val_check_steps`: int=100, Number of training steps between every validation loss check.
\n",
" `batch_size`: int=32, number of different series in each batch.
\n",
+ " `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
\n",
+ " `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
\n",
+ " `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
\n",
+ " `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
\n",
" `step_size`: int=1, step size between each window of temporal data.
\n",
" `scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
\n",
" `random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
\n",
@@ -286,10 +295,11 @@
"\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'multivariate'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = True\n",
" EXOGENOUS_STAT = True\n",
+ " MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h,\n",
@@ -298,6 +308,7 @@
" futr_exog_list = None,\n",
" hist_exog_list = None,\n",
" stat_exog_list = None,\n",
+ " exclude_insample_y = False,\n",
" n_block = 2,\n",
" ff_dim = 64,\n",
" dropout = 0.0,\n",
@@ -310,6 +321,10 @@
" early_stop_patience_steps: int =-1,\n",
" val_check_steps: int = 100,\n",
" batch_size: int = 32,\n",
+ " valid_batch_size: Optional[int] = None,\n",
+ " windows_batch_size = 256,\n",
+ " inference_windows_batch_size = 256,\n",
+ " start_padding_enabled = False,\n",
" step_size: int = 1,\n",
" scaler_type: str = 'identity',\n",
" random_seed: int = 1,\n",
@@ -329,6 +344,7 @@
" futr_exog_list=futr_exog_list,\n",
" hist_exog_list=hist_exog_list,\n",
" stat_exog_list=stat_exog_list,\n",
+ " exclude_insample_y = exclude_insample_y,\n",
" loss=loss,\n",
" valid_loss=valid_loss,\n",
" max_steps=max_steps,\n",
@@ -337,6 +353,10 @@
" early_stop_patience_steps=early_stop_patience_steps,\n",
" val_check_steps=val_check_steps,\n",
" batch_size=batch_size,\n",
+ " valid_batch_size=valid_batch_size,\n",
+ " windows_batch_size=windows_batch_size,\n",
+ " inference_windows_batch_size=inference_windows_batch_size,\n",
+ " start_padding_enabled=start_padding_enabled,\n",
" step_size=step_size,\n",
" scaler_type=scaler_type,\n",
" random_seed=random_seed,\n",
@@ -351,7 +371,7 @@
" # Reversible InstanceNormalization layer\n",
" self.revin = revin\n",
" if self.revin:\n",
- " self.norm = ReversibleInstanceNorm1d(n_series = n_series)\n",
+ " self.norm = RevINMultivariate(num_features= n_series, affine=True)\n",
"\n",
" # Forecast horizon\n",
" self.h = h\n",
@@ -417,19 +437,19 @@
"\n",
" def forward(self, windows_batch):\n",
" # Parse batch\n",
- " x = windows_batch['insample_y'] # [batch_size (B), input_size (L), n_series (N)]\n",
- " hist_exog = windows_batch['hist_exog'] # [B, hist_exog_size (X), L, N]\n",
- " futr_exog = windows_batch['futr_exog'] # [B, futr_exog_size (F), L + h, N]\n",
- " stat_exog = windows_batch['stat_exog'] # [N, stat_exog_size (S)]\n",
+ " x = windows_batch['insample_y'] # [batch_size (B), input_size (L), n_series (N)]\n",
+ " hist_exog = windows_batch['hist_exog'] # [B, hist_exog_size (X), L, N]\n",
+ " futr_exog = windows_batch['futr_exog'] # [B, futr_exog_size (F), L + h, N]\n",
+ " stat_exog = windows_batch['stat_exog'] # [N, stat_exog_size (S)]\n",
" batch_size, input_size = x.shape[:2]\n",
"\n",
+ " # Apply revin to x\n",
+ " if self.revin:\n",
+ " x = self.norm(x, mode=\"norm\") # [B, L, N] -> [B, L, N]\n",
+ "\n",
" # Add channel dimension to x\n",
" x = x.unsqueeze(1) # [B, L, N] -> [B, 1, L, N]\n",
"\n",
- " # Apply revin to x\n",
- " if self.revin:\n",
- " x = self.norm(x) # [B, 1, L, N] -> [B, 1, L, N]\n",
- " \n",
" # Concatenate x with historical exogenous\n",
" if self.hist_exog_size > 0:\n",
" x = torch.cat((x, hist_exog), dim=1) # [B, 1, L, N] + [B, X, L, N] -> [B, 1 + X, L, N]\n",
@@ -476,26 +496,17 @@
" x = self.mixing_block(x) # [B, h, ff_dim] -> [B, h, ff_dim] \n",
" \n",
" # Fully connected output layer\n",
- " x = self.out(x) # [B, h, ff_dim] -> [B, h, N * n_outputs]\n",
+ " forecast = self.out(x) # [B, h, ff_dim] -> [B, h, N * n_outputs]\n",
" \n",
" # Reverse Instance Normalization on output\n",
" if self.revin:\n",
- " x = x.reshape(batch_size, \n",
- " self.h, \n",
- " self.loss.outputsize_multiplier,\n",
- " -1) # [B, h, N * n_outputs] -> [B, h, n_outputs, N]\n",
- " x = self.norm.reverse(x)\n",
- " x = x.reshape(batch_size, self.h, -1) # [B, h, n_outputs, N] -> [B, h, n_outputs * N]\n",
- "\n",
- " # Map to loss domain\n",
- " forecast = self.loss.domain_map(x)\n",
- "\n",
- " # domain_map might have squeezed the last dimension in case n_series == 1\n",
- " # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.\n",
- " if forecast.ndim == 2:\n",
- " return forecast.unsqueeze(-1)\n",
- " else:\n",
- " return forecast"
+ " forecast = forecast.reshape(batch_size, \n",
+ " self.h * self.loss.outputsize_multiplier,\n",
+ " -1) # [B, h, N * n_outputs] -> [B, h * n_outputs, N]\n",
+ " forecast = self.norm(forecast, \"denorm\")\n",
+ " forecast = forecast.reshape(batch_size, self.h, -1) # [B, h * n_outputs, N] -> [B, h, n_outputs * N]\n",
+ "\n",
+ " return forecast"
]
},
{
@@ -531,113 +542,12 @@
"metadata": {},
"outputs": [],
"source": [
- "#| hide\n",
- "import logging\n",
- "import warnings\n",
- "import pandas as pd\n",
- "\n",
- "from neuralforecast import NeuralForecast\n",
- "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic, generate_series\n",
- "from neuralforecast.losses.pytorch import MAE, MSE, RMSE, MAPE, SMAPE, MASE, relMSE, QuantileLoss, MQLoss, DistributionLoss,PMM, GMM, NBMM, HuberLoss, TukeyLoss, HuberQLoss, HuberMQLoss\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# Test losses\n",
+ "# Unit tests for models\n",
"logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
- "warnings.filterwarnings(\"ignore\")\n",
- "\n",
- "Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
- "\n",
- "AirPassengersStatic_single = AirPassengersStatic[AirPassengersStatic[\"unique_id\"] == 'Airline1']\n",
- "Y_train_df_single = Y_train_df[Y_train_df[\"unique_id\"] == 'Airline1']\n",
- "Y_test_df_single = Y_test_df[Y_test_df[\"unique_id\"] == 'Airline1']\n",
- "\n",
- "losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "valid_losses = [MAE(), MSE(), RMSE(), MAPE(), SMAPE(), MASE(seasonality=12), relMSE(y_train=Y_train_df), QuantileLoss(q=0.5), MQLoss(), DistributionLoss(distribution='Bernoulli'), DistributionLoss(distribution='Normal'), DistributionLoss(distribution='Poisson'), DistributionLoss(distribution='StudentT'), DistributionLoss(distribution='NegativeBinomial'), DistributionLoss(distribution='Tweedie'), PMM(), GMM(), NBMM(), HuberLoss(), TukeyLoss(), HuberQLoss(q=0.5), HuberMQLoss()]\n",
- "\n",
- "for loss, valid_loss in zip(losses, valid_losses):\n",
- " try:\n",
- " model = TSMixerx(h=12,\n",
- " input_size=24,\n",
- " n_series=2,\n",
- " stat_exog_list=['airline1'],\n",
- " futr_exog_list=['trend'],\n",
- " n_block=4,\n",
- " ff_dim=4,\n",
- " revin=True,\n",
- " scaler_type='standard',\n",
- " max_steps=2,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=5,\n",
- " learning_rate=1e-3,\n",
- " loss=loss,\n",
- " valid_loss=valid_loss,\n",
- " batch_size=32\n",
- " )\n",
- "\n",
- " fcst = NeuralForecast(models=[model], freq='M')\n",
- " fcst.fit(df=Y_train_df, static_df=AirPassengersStatic, val_size=12)\n",
- " forecasts = fcst.predict(futr_df=Y_test_df)\n",
- " except Exception as e:\n",
- " assert str(e) == f\"{loss} is not supported in a Multivariate model.\"\n",
- "\n",
- "\n",
- "# Test n_series = 1\n",
- "model = TSMixerx(h=12,\n",
- " input_size=24,\n",
- " n_series=1,\n",
- " stat_exog_list=['airline1'],\n",
- " futr_exog_list=['trend'],\n",
- " n_block=4,\n",
- " ff_dim=4,\n",
- " revin=True,\n",
- " scaler_type='standard',\n",
- " max_steps=2,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=5,\n",
- " learning_rate=1e-3,\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " batch_size=32\n",
- " )\n",
- "fcst = NeuralForecast(models=[model], freq='M')\n",
- "fcst.fit(df=Y_train_df_single, static_df=AirPassengersStatic_single, val_size=12)\n",
- "forecasts = fcst.predict(futr_df=Y_test_df_single) \n",
- "\n",
- "# Test n_series > 1024\n",
- "# See issue: https://github.com/Nixtla/neuralforecast/issues/948\n",
- "n_series = 1111\n",
- "Y_df, S_df = generate_series(n_series=n_series, n_temporal_features=2, n_static_features=2)\n",
- "\n",
- "model = TSMixerx(\n",
- " h=12,\n",
- " input_size=24,\n",
- " n_series=n_series,\n",
- " stat_exog_list=['static_0', 'static_1'],\n",
- " hist_exog_list=[\"temporal_0\", \"temporal_1\"],\n",
- " n_block=4,\n",
- " ff_dim=3,\n",
- " revin=True,\n",
- " scaler_type=\"standard\",\n",
- " max_steps=5,\n",
- " early_stop_patience_steps=-1,\n",
- " val_check_steps=5,\n",
- " learning_rate=1e-3,\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
- " batch_size=32,\n",
- ")\n",
- "\n",
- "fcst = NeuralForecast(models=[model], freq=\"D\")\n",
- "fcst.fit(df=Y_df, static_df=S_df, val_size=12)\n",
- "forecasts = fcst.predict()"
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(TSMixerx, [\"airpassengers\"])"
]
},
{
@@ -667,7 +577,7 @@
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import TSMixerx\n",
"from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
- "from neuralforecast.losses.pytorch import MAE\n",
+ "from neuralforecast.losses.pytorch import GMM\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
@@ -680,13 +590,12 @@
" n_block=4,\n",
" ff_dim=4,\n",
" revin=True,\n",
- " scaler_type='standard',\n",
+ " scaler_type='robust',\n",
" max_steps=500,\n",
" early_stop_patience_steps=-1,\n",
" val_check_steps=5,\n",
" learning_rate=1e-3,\n",
- " loss=MAE(),\n",
- " valid_loss=MAE(),\n",
+ " loss = GMM(n_components=10, weighted=True),\n",
" batch_size=32\n",
" )\n",
"\n",
@@ -702,7 +611,11 @@
"\n",
"plot_df = plot_df[plot_df.unique_id=='Airline1'].drop('unique_id', axis=1)\n",
"plt.plot(plot_df['ds'], plot_df['y'], c='black', label='True')\n",
- "plt.plot(plot_df['ds'], plot_df['TSMixerx'], c='blue', label='Forecast')\n",
+ "plt.plot(plot_df['ds'], plot_df['TSMixerx-median'], c='blue', label='median')\n",
+ "plt.fill_between(x=plot_df['ds'][-12:], \n",
+ " y1=plot_df['TSMixerx-lo-90'][-12:].values,\n",
+ " y2=plot_df['TSMixerx-hi-90'][-12:].values,\n",
+ " alpha=0.4, label='level 90')\n",
"ax.set_title('AirPassengers Forecast', fontsize=22)\n",
"ax.set_ylabel('Monthly Passengers', fontsize=20)\n",
"ax.set_xlabel('Year', fontsize=20)\n",
@@ -733,7 +646,7 @@
"Y_df = AirPassengersPanel[AirPassengersPanel['unique_id']=='Airline1']\n",
"\n",
"plt.plot(Y_df['ds'], Y_df['y'], c='black', label='True')\n",
- "plt.plot(Y_hat_df['ds'], Y_hat_df['TSMixerx'], c='blue', label='Forecast')\n",
+ "plt.plot(Y_hat_df['ds'], Y_hat_df['TSMixerx-median'], c='blue', label='Forecast')\n",
"ax.set_title('AirPassengers Forecast', fontsize=22)\n",
"ax.set_ylabel('Monthly Passengers', fontsize=20)\n",
"ax.set_xlabel('Year', fontsize=20)\n",
diff --git a/nbs/models.vanillatransformer.ipynb b/nbs/models.vanillatransformer.ipynb
index b76cc9ba2..c28b2a4a6 100644
--- a/nbs/models.vanillatransformer.ipynb
+++ b/nbs/models.vanillatransformer.ipynb
@@ -67,7 +67,7 @@
" TransDecoderLayer, TransDecoder,\n",
" DataEmbedding, AttentionLayer,\n",
")\n",
- "from neuralforecast.common._base_windows import BaseWindows\n",
+ "from neuralforecast.common._base_model import BaseModel\n",
"\n",
"from neuralforecast.losses.pytorch import MAE"
]
@@ -79,8 +79,11 @@
"outputs": [],
"source": [
"#| hide\n",
+ "import logging\n",
+ "import warnings\n",
"from fastcore.test import test_eq\n",
- "from nbdev.showdoc import show_doc"
+ "from nbdev.showdoc import show_doc\n",
+ "from neuralforecast.common._model_checks import check_model"
]
},
{
@@ -154,7 +157,7 @@
"outputs": [],
"source": [
"#| export\n",
- "class VanillaTransformer(BaseWindows):\n",
+ "class VanillaTransformer(BaseModel):\n",
" \"\"\" VanillaTransformer\n",
"\n",
" Vanilla Transformer, following implementation of the Informer paper, used as baseline.\n",
@@ -209,10 +212,11 @@
"\t- [Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, Wancai Zhang. \"Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting\"](https://arxiv.org/abs/2012.07436)
\n",
" \"\"\"\n",
" # Class attributes\n",
- " SAMPLING_TYPE = 'windows'\n",
" EXOGENOUS_FUTR = True\n",
" EXOGENOUS_HIST = False\n",
" EXOGENOUS_STAT = False\n",
+ " MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)\n",
+ " RECURRENT = False # If the model produces forecasts recursively (True) or direct (False)\n",
"\n",
" def __init__(self,\n",
" h: int, \n",
@@ -346,14 +350,8 @@
" def forward(self, windows_batch):\n",
" # Parse windows_batch\n",
" insample_y = windows_batch['insample_y']\n",
- " #insample_mask = windows_batch['insample_mask']\n",
- " #hist_exog = windows_batch['hist_exog']\n",
- " #stat_exog = windows_batch['stat_exog']\n",
- "\n",
" futr_exog = windows_batch['futr_exog']\n",
"\n",
- " insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]\n",
- "\n",
" if self.futr_exog_size > 0:\n",
" x_mark_enc = futr_exog[:,:self.input_size,:]\n",
" x_mark_dec = futr_exog[:,-(self.label_len+self.h):,:]\n",
@@ -371,7 +369,7 @@
" dec_out = self.decoder(dec_out, enc_out, x_mask=None, \n",
" cross_mask=None)\n",
"\n",
- " forecast = self.loss.domain_map(dec_out[:, -self.h:])\n",
+ " forecast = dec_out[:, -self.h:]\n",
" return forecast"
]
},
@@ -402,6 +400,21 @@
"show_doc(VanillaTransformer.predict, name='VanillaTransformer.predict')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Unit tests for models\n",
+ "logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
+ "logging.getLogger(\"lightning_fabric\").setLevel(logging.ERROR)\n",
+ "with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\")\n",
+ " check_model(VanillaTransformer, [\"airpassengers\"])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -421,9 +434,7 @@
"\n",
"from neuralforecast import NeuralForecast\n",
"from neuralforecast.models import VanillaTransformer\n",
- "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic, augment_calendar_df\n",
- "\n",
- "AirPassengersPanel, calendar_cols = augment_calendar_df(df=AirPassengersPanel, freq='M')\n",
+ "from neuralforecast.utils import AirPassengersPanel, AirPassengersStatic\n",
"\n",
"Y_train_df = AirPassengersPanel[AirPassengersPanel.ds=AirPassengersPanel['ds'].values[-12]].reset_index(drop=True) # 12 test\n",
@@ -434,7 +445,6 @@
" conv_hidden_size=32,\n",
" n_head=2,\n",
" loss=MAE(),\n",
- " futr_exog_list=calendar_cols,\n",
" scaler_type='robust',\n",
" learning_rate=1e-3,\n",
" max_steps=500,\n",
diff --git a/nbs/utils.ipynb b/nbs/utils.ipynb
index 5b056c144..e8cb8c170 100644
--- a/nbs/utils.ipynb
+++ b/nbs/utils.ipynb
@@ -13,7 +13,16 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n"
+ ]
+ }
+ ],
"source": [
"#| hide\n",
"%load_ext autoreload\n",
@@ -38,12 +47,11 @@
"#| export\n",
"import random\n",
"from itertools import chain\n",
- "from typing import List, Union\n",
+ "from typing import List, Union, Optional, Tuple\n",
"from utilsforecast.compat import DFType\n",
"\n",
"import numpy as np\n",
- "import pandas as pd\n",
- "import utilsforecast.processing as ufp"
+ "import pandas as pd"
]
},
{
@@ -161,7 +169,77 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "---\n",
+ "\n",
+ "[source](https://github.com/Nixtla/neuralforecast/blob/main/neuralforecast/utils.py#L22){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "\n",
+ "### generate_series\n",
+ "\n",
+ "> generate_series (n_series:int, freq:str='D', min_length:int=50,\n",
+ "> max_length:int=500, n_temporal_features:int=0,\n",
+ "> n_static_features:int=0, equal_ends:bool=False,\n",
+ "> seed:int=0)\n",
+ "\n",
+ "*Generate Synthetic Panel Series.\n",
+ "\n",
+ "Generates `n_series` of frequency `freq` of different lengths in the interval [`min_length`, `max_length`].\n",
+ "If `n_temporal_features > 0`, then each serie gets temporal features with random values.\n",
+ "If `n_static_features > 0`, then a static dataframe is returned along the temporal dataframe.\n",
+ "If `equal_ends == True` then all series end at the same date.\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`n_series`: int, number of series for synthetic panel.
\n",
+ "`min_length`: int, minimal length of synthetic panel's series.
\n",
+ "`max_length`: int, minimal length of synthetic panel's series.
\n",
+ "`n_temporal_features`: int, default=0, number of temporal exogenous variables for synthetic panel's series.
\n",
+ "`n_static_features`: int, default=0, number of static exogenous variables for synthetic panel's series.
\n",
+ "`equal_ends`: bool, if True, series finish in the same date stamp `ds`.
\n",
+ "`freq`: str, frequency of the data, [panda's available frequencies](https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases).
\n",
+ "\n",
+ "**Returns:**
\n",
+ "`freq`: pandas.DataFrame, synthetic panel with columns [`unique_id`, `ds`, `y`] and exogenous.*"
+ ],
+ "text/plain": [
+ "---\n",
+ "\n",
+ "[source](https://github.com/Nixtla/neuralforecast/blob/main/neuralforecast/utils.py#L22){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "\n",
+ "### generate_series\n",
+ "\n",
+ "> generate_series (n_series:int, freq:str='D', min_length:int=50,\n",
+ "> max_length:int=500, n_temporal_features:int=0,\n",
+ "> n_static_features:int=0, equal_ends:bool=False,\n",
+ "> seed:int=0)\n",
+ "\n",
+ "*Generate Synthetic Panel Series.\n",
+ "\n",
+ "Generates `n_series` of frequency `freq` of different lengths in the interval [`min_length`, `max_length`].\n",
+ "If `n_temporal_features > 0`, then each serie gets temporal features with random values.\n",
+ "If `n_static_features > 0`, then a static dataframe is returned along the temporal dataframe.\n",
+ "If `equal_ends == True` then all series end at the same date.\n",
+ "\n",
+ "**Parameters:**
\n",
+ "`n_series`: int, number of series for synthetic panel.
\n",
+ "`min_length`: int, minimal length of synthetic panel's series.
\n",
+ "`max_length`: int, minimal length of synthetic panel's series.
\n",
+ "`n_temporal_features`: int, default=0, number of temporal exogenous variables for synthetic panel's series.
\n",
+ "`n_static_features`: int, default=0, number of static exogenous variables for synthetic panel's series.
\n",
+ "`equal_ends`: bool, if True, series finish in the same date stamp `ds`.
\n",
+ "`freq`: str, frequency of the data, [panda's available frequencies](https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases).
\n",
+ "\n",
+ "**Returns:**
\n",
+ "`freq`: pandas.DataFrame, synthetic panel with columns [`unique_id`, `ds`, `y`] and exogenous.*"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"show_doc(generate_series, title_level=3)"
]
@@ -170,7 +248,111 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "C:\\Users\\ospra\\AppData\\Local\\Temp\\ipykernel_16560\\470716697.py:2: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
+ " synthetic_panel.groupby('unique_id').head(4)\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " unique_id | \n",
+ " ds | \n",
+ " y | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0 | \n",
+ " 2000-01-01 | \n",
+ " 0.357595 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 0 | \n",
+ " 2000-01-02 | \n",
+ " 1.301382 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0 | \n",
+ " 2000-01-03 | \n",
+ " 2.272442 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0 | \n",
+ " 2000-01-04 | \n",
+ " 3.211827 | \n",
+ "
\n",
+ " \n",
+ " | 222 | \n",
+ " 1 | \n",
+ " 2000-01-01 | \n",
+ " 5.399023 | \n",
+ "
\n",
+ " \n",
+ " | 223 | \n",
+ " 1 | \n",
+ " 2000-01-02 | \n",
+ " 6.092818 | \n",
+ "
\n",
+ " \n",
+ " | 224 | \n",
+ " 1 | \n",
+ " 2000-01-03 | \n",
+ " 0.476396 | \n",
+ "
\n",
+ " \n",
+ " | 225 | \n",
+ " 1 | \n",
+ " 2000-01-04 | \n",
+ " 1.343744 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " static_0 | \n",
+ " static_1 | \n",
+ " unique_id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0.748805 | \n",
+ " 0.573544 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 0.234966 | \n",
+ " 0.235057 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " unique_id | \n",
+ " ds | \n",
+ " y | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 1.0 | \n",
+ " 1949-01-31 | \n",
+ " 112.0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 1.0 | \n",
+ " 1949-02-28 | \n",
+ " 118.0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 1.0 | \n",
+ " 1949-03-31 | \n",
+ " 132.0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 1.0 | \n",
+ " 1949-04-30 | \n",
+ " 129.0 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 1.0 | \n",
+ " 1949-05-31 | \n",
+ " 121.0 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 1.0 | \n",
+ " 1949-06-30 | \n",
+ " 135.0 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 1.0 | \n",
+ " 1949-07-31 | \n",
+ " 148.0 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 1.0 | \n",
+ " 1949-08-31 | \n",
+ " 148.0 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 1.0 | \n",
+ " 1949-09-30 | \n",
+ " 136.0 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 1.0 | \n",
+ " 1949-10-31 | \n",
+ " 119.0 | \n",
+ "
\n",
+ " \n",
+ " | 10 | \n",
+ " 1.0 | \n",
+ " 1949-11-30 | \n",
+ " 104.0 | \n",
+ "
\n",
+ " \n",
+ " | 11 | \n",
+ " 1.0 | \n",
+ " 1949-12-31 | \n",
+ " 118.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"#We are going to plot the ARIMA predictions, and the prediction intervals.\n",
"fig, ax = plt.subplots(1, 1, figsize = (20, 7))\n",
@@ -291,7 +662,88 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " static_0 | \n",
+ " static_1 | \n",
+ " static_2 | \n",
+ " unique_id | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0.268844 | \n",
+ " 0.875946 | \n",
+ " 0.047605 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 0.995151 | \n",
+ " 0.376025 | \n",
+ " 0.497579 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0.136613 | \n",
+ " 0.060934 | \n",
+ " 0.319290 | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0.084419 | \n",
+ " 0.918999 | \n",
+ " 0.820050 | \n",
+ " 3 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 0.774360 | \n",
+ " 0.685072 | \n",
+ " 0.113191 | \n",
+ " 4 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " unique_id | \n",
+ " ds | \n",
+ " y | \n",
+ " trend | \n",
+ " y_[lag12] | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 140 | \n",
+ " Airline1 | \n",
+ " 1960-09-30 | \n",
+ " 508.0 | \n",
+ " 140 | \n",
+ " 463.0 | \n",
+ "
\n",
+ " \n",
+ " | 141 | \n",
+ " Airline1 | \n",
+ " 1960-10-31 | \n",
+ " 461.0 | \n",
+ " 141 | \n",
+ " 407.0 | \n",
+ "
\n",
+ " \n",
+ " | 142 | \n",
+ " Airline1 | \n",
+ " 1960-11-30 | \n",
+ " 390.0 | \n",
+ " 142 | \n",
+ " 362.0 | \n",
+ "
\n",
+ " \n",
+ " | 143 | \n",
+ " Airline1 | \n",
+ " 1960-12-31 | \n",
+ " 432.0 | \n",
+ " 143 | \n",
+ " 405.0 | \n",
+ "
\n",
+ " \n",
+ " | 284 | \n",
+ " Airline2 | \n",
+ " 1960-09-30 | \n",
+ " 808.0 | \n",
+ " 284 | \n",
+ " 763.0 | \n",
+ "
\n",
+ " \n",
+ " | 285 | \n",
+ " Airline2 | \n",
+ " 1960-10-31 | \n",
+ " 761.0 | \n",
+ " 285 | \n",
+ " 707.0 | \n",
+ "
\n",
+ " \n",
+ " | 286 | \n",
+ " Airline2 | \n",
+ " 1960-11-30 | \n",
+ " 690.0 | \n",
+ " 286 | \n",
+ " 662.0 | \n",
+ "
\n",
+ " \n",
+ " | 287 | \n",
+ " Airline2 | \n",
+ " 1960-12-31 | \n",
+ " 732.0 | \n",
+ " 287 | \n",
+ " 705.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"fig, ax = plt.subplots(1, 1, figsize = (20, 7))\n",
"plot_df = AirPassengersPanel.set_index('ds')\n",
@@ -365,7 +942,18 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAABmcAAAKHCAYAAAB0L5wRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdd3gU5doG8Ht2s9n03js19I4UpXeOKKIgIgpiwWPHXo4axA87niMqNpqKooCAXUADSicQWiiBkF4I6X2z2Z3vj2Unu2Q3bVsC9++6uM6bmXdm3t1sxu+bZ5/nEURRFEFERERERERERERERER2IXP0AoiIiIiIiIiIiIiIiK4lDM4QERERERERERERERHZEYMzREREREREREREREREdsTgDBERERERERERERERkR0xOENERERERERERERERGRHDM4QERERERERERERERHZEYMzREREREREREREREREdsTgDBERERERERERERERkR0xOENERERERERERERERGRHDM4QEREREZFNjR49GoIgQBAERy+FiIiIiIioTWBwhoiIiOgapX9Ybu6fh4cHOnbsiOnTp2PNmjVQqVSOXrJdnT17Fm+99RYmTZqETp06wdvbG87OzggKCsKAAQOwcOFCbN68GbW1tY5eqs3V1tYiMDBQ+mx069bN0UtCXFyc2c+uk5MT/P39MXDgQDz88MM4cOCAo5dL1Czz5883+ZmWyWTw8vJCREQEevfujdmzZ+Ott97CoUOHHLLOnTt3Ii4uDnFxcUhLS3PIGoiIiIjaO0EURdHRiyAiIiIi+2tpFkOnTp2wceNG9OvXzzYLaiPS0tLw/PPPY8OGDdBqtU3O9/f3x/PPP49HH30USqXSDiu0vw0bNmDWrFlG23bv3o3rr7++WcePHj0au3btAgBY6//9iIuLw+LFi5s9f86cOfjiiy/g6upqlesT2cL8+fOxdu3aFh3Tq1cvPPnkk7jnnntstKqGDP/+4uPjMXr0aLtdm4iIiOhq4eToBRARERGR423evNnoZ1EUUVJSgmPHjuGbb77BpUuXkJKSgnHjxuHUqVMIDg520EptKz4+HjNnzkRhYSEAQC6XY9SoURg1ahQiIiLg6emJgoICpKSk4I8//sDJkydRWFiIZ555Bp07d8b06dMd+wJsZOXKlSa3NTc4s3PnTiuvyNjtt9+O2bNnSz/X1dUhOzsbv/zyC7Zv3w4A+Oabb1BVVdXgs07UVj366KMYO3as9LNKpUJJSQmys7Nx8OBB/PPPP6iqqsLJkyexYMECrF+/HuvWrUNAQIADV01EREREzcXgDBERERE1GlR45ZVXMHr0aJw4cQJFRUVYtmwZ3nrrLfstzk6OHDmCqVOnoqamBgAwefJk/O9//0PXrl1Nzn/33Xdx6NAhvPLKK/j999/tuVS7yszMlAIco0ePRnp6OlJTU/H999/jf//7Hzw9PR28QqBbt24mP8OPP/441q5di3vuuQeiKGLLli34448/MGnSJPsvkqiFBgwY0Oi9uaSkBJ988gkWL16MmpoabNu2DdOmTcNff/3FDDEiIiKidoA9Z4iIiIioUX5+fliyZIn0s62zIByhoqICt9xyixSYueeee/DLL7+YDczoDR48GL/99hs++eSTq/Zh6OrVq6XybvPnz8fdd98NAKisrMR3333nyKU1y7x584xKsn3//fcOXA2R9fj4+OD555/Hvn374O3tDQDYv38/nn/+eQevjIiIiIiag8EZIiIiImpS9+7dpXFZWVmT848ePYpHHnkEPXv2hI+PD1xcXBAVFYUZM2Zg3bp1Znu53HzzzVID7Oeee67Ra2zYsEGa27VrV1RUVLTsRRn45JNPkJGRAUDXv2HFihWQyZr/fyovXLiw0WyMCxcu4JlnnkH//v3h5+cHpVKJsLAwTJkyBZ988glqa2tNHvfoo49Kr/H2229vdA379u2DQqGAIAgIDAxETk5Os9dvjiiKWL16NQDA3d0dt956K+bNmyf1KzJV7syU0aNHS6/DlDVr1kj716xZA0CXyfTggw+ia9eu8PT0NNrXUjfddJM0Pn78uNG+5ORkLFu2DLfccgu6dOkCDw8PODs7IygoCCNHjsTrr7+OgoKCZl3nn3/+wYIFC9C9e3d4enrC2dkZISEh6N27N2655RZ89NFHSE1NNXv8L7/8gjvuuAOdO3eGu7s7lEolwsPD0a9fP8yePRurVq1Cbm5uo2uoqanBp59+ihtvvBGRkZFwcXGBt7c3evXqhcceewzJycmNHh8XFyf9LvSB2MOHD+Oee+5Bx44d4eLiAn9/f4wZM8YocNeU3bt344477kBERARcXFwQHh6Of/3rX9iyZQsAXa8n/XXnz5/f5Pn+/vtvPPDAA+jevbt0j4mMjMStt96KTZs2NdrbyNS1cnJy8Morr6B///7w9/c3uY6ysjK89957GDNmDIKDg+Hs7AwvLy906tQJw4cPx5NPPonff//d7N+zrfTr10/6OwWATz/9FNnZ2SbnlpeXY/369Vi4cCEGDx4MPz8/KBQK+Pj4oEePHrj//vtx8OBBs9fSfz4M+z2NGTNGej/1/2JiYhocm5WVhY8//hizZ89Gjx494OnpCYVCgYCAAAwZMgQvvPACMjMzW/9GEBEREbU3IhERERFdkwBI/5qye/duae7EiRPNzqurqxMfe+wxURAEo/Nf+a9fv35ienp6g+MLCgrE8PBwEYAoCIL4xx9/mLxOamqq6O3tLQIQnZ2dxcOHDzf/hV9Bq9VK1wQgfvfdd60+lylvvvmmqFAoGn0/OnToIB49erTBsTU1NWK/fv2keZ9//rnJaxQXF4vR0dHSvJ9++skqa9++fbt0zrvuukvaPnLkSGl7UlJSk+cZNWpUo5+11atXS/tXr14tvvXWW6JcLm/wPq1evVo65tVXX5W2v/rqq41ef9u2bdLcLl26SNvXrl3b6O9F/8/Ly0v8+eefzZ5fo9GICxcubNa5/vWvfzU4vqqqSpw2bVqzjn/44YfNrmPnzp1Gn2VT/+Ryubh06VKz5zB8X+Pj48U333zT5O9C/2/atGlibW1to+//s88+2+g94c477xTPnTsn/Txv3jyz5youLm7WezVy5Ejx0qVLJs+RmppqdK1t27aJfn5+Dc5huI6EhAQxJCSkWb+jQ4cONfp+NGbevHkmP+/N0b9/f+nYN954o8F+lUoluri4NOs1LFy4UFSr1Q3OYfj5aOxfdHS00XHx8fFN/ncB0N3Pv/jiixa9biIiIqL2ij1niIiIiKhJn3zyiTSeMGGC2Xn33XeflN3g5OSE2bNnY8yYMXBzc8OpU6ewatUqZGdn4+jRoxg+fDiOHDmCoKAg6Xh/f3+sW7cO48aNg0ajwd13341jx44hODhYmlNXV4c5c+agtLQUAPDWW29hwIABrX5tSUlJ0rfMPT09MWPGjFaf60qLFy9GXFyc9PP06dMxefJk+Pj4ICUlBWvXrkVycjJSU1MxYsQIHDhwwChLSalUYv369Rg4cCAqKyvx+OOPY/jw4ejRo4fRdR544AGkp6cD0PVZufHGG62yfsPMmHnz5knj+fPn4++//5bmvPfee1a5HqArO/bbb7/Bw8MDd999N6677jo4Ozvj9OnTCAkJadU58/PzpbGPj480rqqqgiAI6Nu3L0aOHIlu3brBz88PgO5b/jt27MDvv/+OsrIy3Hrrrdi7d6/Jz9qHH36ITz/9FIDuM3Tbbbdh4MCBCAwMRG1tLbKyspCQkIAdO3aYXN9LL72En376CQAQGBiI22+/HT179oS/vz9qamqQmpqKgwcPIj4+3uxr/O2333DzzTdDrVZDEASMHz8ekyZNQkREBGpra5GQkIAvv/wSJSUlePHFFwEAL7zwQqPv2xdffIF169YhMDAQ8+fPR58+fSCTybB//3588cUXqK6uxk8//YQ333wTL7/8sslzvP7663j77bcBAIIgYMaMGZg8eTI8PDyQnJyMVatWYd26dairq2t0LYAuc+X666/HqVOnAAAxMTHSe6VUKpGWloZvv/0WR48exd9//43x48dj//79cHFxMXvO8+fP47bbbkN5eTluvfVWjB8/Hn5+fsjKypIyvaqqqjB9+nTk5eUBAAYOHIhbbrkF4eHhcHd3R3FxMU6fPo34+HgcO3asyddhK3PnzkViYiIAXfnJK8ubabVa1NTUIDg4GOPGjUPfvn0RFhYGV1dXFBcXIyEhAd9//z2Ki4vx6aefwsvLS/rd6c2ePRv9+vXD+vXrpbKGS5YsQa9evYzmubm5Gf1cU1MDURQRGxuLMWPGoEePHggICICTkxPy8vLw999/Y8uWLaitrcX999+P4OBgq93HiIiIiNosR0eHiIiIiMgxYPBt5StptVqxuLhY3Llzp3jrrbdK83r06CFWVlaaPN+mTZukeT4+PuKBAwcazCkrKxPHjBkjzZs+fbrJcxl+O3vixImiVquV9r3wwgtGWQiG+1rj448/ls43btw4i85l6ODBg6JMJhMBiEqlUvzxxx8bzFGpVOKcOXOk6/fv39/k6zHMLOndu7dYXV0t7fv000+Njq+pqbHK+gsLC0WlUikCECMjI0WNRiPtKy8vF93d3UUAYmBgYJOZEy3JnAEgdu3a1WRmlaGWZM7cfvvt0twFCxZI20+ePCmeO3eu0WN37Nghurm5Nfr56NmzpwhA9PPza3TdNTU14v79+4221dXVSVlgnTt3FouLi80eX1paKh45cqTB9pycHCnzw9vbW/zzzz9NHp+TkyP26dNHyqA5ffp0gzlXZkaMGjXK5Jr27NkjOjk5iQBEf39/k5+7s2fPis7OziIAUaFQiFu3bm0wp7KyUpwwYYLZjBVDs2fPluY8+eSTJj93Wq1WfO6556R5L730UoM5hpkzAER3d3dxx44dJq8piqK4YcMGae5TTz1ldp4oimJSUpKYn5/f6JzGWJI5s3fvXulYX1/fBvvr6urEX3/91ehv+UoFBQXi8OHDpc9IWlqayXlXZlg1JS0tzWR2oKHExEQxKChIBHQZbpbe24mIiIjaOgZniIiIiK5Rhg8nm/oXFhYmPvbYY2JpaanZ8w0aNEia/+2335qdV1BQIAYEBEhzTZXFqqurMyqd9dZbb4miKIp//vmnFPAIDQ01W7aoJV566SWjUj7WcttttzVaYkhPpVKJ3bp1k+b++uuvJufdeeed0px///vfoijqHgTrAwfu7u7i2bNnrbb+Dz74QLreCy+80GD/XXfdJe3fuHFjo+dqSXBGEASTAYgrNTc489VXXxmVU/r999+bPPeVXn75Zen4rKysBvv1QayZM2e2+Ny5ubnSuZ955pkWHy+Korho0SLpHKYCIIbOnDkjlSl78MEHG+w3fF99fX0b/RszDCz+888/DfY/8sgjjX6G9C5duiT6+vo2Gpw5duyYtP+WW25p9DWKoijecMMNUrDqysDRlcGZ999/v9FzvfHGG43er6zJkuBMTk6O0esyVZasOc6fPy+d4/XXXzc5p6XBmeZauXKldN7du3db7bxEREREbVHzu5wSERER0TVLoVDA3d0dGo3G5P6MjAwkJCQAADp06NBo83p/f388+OCD0s8//PBDgzlyuRzr1q2TSkz95z//wc8//4y5c+dCq9VCJpPh66+/RkBAgCUvCwBQWFgojQ1LXlmitrYWP//8MwDAw8MDjz76qNm5zs7OePrpp6WfN23aZHLeihUr0LlzZ2m8bt06zJ49G1VVVQB0pbW6du1qlfUDxiXN7r777gb7DcucGc611A033ID+/fu36JgzZ85gy5Yt0r9NmzZh+fLlmDx5Mu666y6pOfy0adMwadKkFq/p+uuvl8b79+9vsN/d3R0AcOLEiRY3gzcs/3TkyJEWr00URXz11VcAgNjYWNx0002Nzo+NjcV1110HAPjjjz8anXv33Xc3+jc2btw4aZyUlNRg/9atWwHo/p4fe+wxs+cJCAjA3LlzG13L2rVrpfFzzz3X6FwAuOuuuwAApaWlOHDggNl5rq6uuO+++xo9l/73CwCHDx9u8tqO4uvra/RzUVFRq87TqVMnqYSgqc+7LTX1t0ZERER0NWHPGSIiIiLC5s2bG2yrqqpCWloatm7dioMHD+KNN97AunXrsGPHDnTp0sVoruFDtIkTJ0q9GsyZPHkyXn/99QbHGoqIiMDq1aulPhrTpk2T9j3//PMYO3Zss19fczW17uY6evQoampqAOgeNho+3DVl8uTJ0tjc++Hp6Ylvv/0W119/PWpra40eZs+ZMwfz58+3fOGXJSQkSL0zhgwZgm7dujWYM3bsWERFRSEjIwPbtm1DdnY2wsPDLb72iBEjWnzMd999J/W/MGfWrFlYvXq1yX27d+/Gt99+i4MHD+LChQsoLy+HWq02OTcrK6vBtokTJ2L9+vU4c+YMxo0bh0WLFmHixInw8PBocu1eXl4YOnQo9u/fjz///BM33XQTHn74YYwePRpKpbLJ40+dOoWCggIAQEhICLZs2dLkMXK5HACQmpqKmpoasz1Zhg8f3uh5DH/fxcXFRvsuXryIzMxMAEC3bt2a7Bc0ZswYLF++3Ox+fY8jQRCQmZmJ3NzcRs+n7yMF6N6jkSNHmpzXv3//Jn9P48ePhyAIEEUR//73v3Hu3DnMnj27Qe8nR9MHIZuSk5ODr776Cn/++SdOnTqF4uJiKch7JVOfd0scPXoUX3/9Nfbt24dz586hrKwMKpXKLtcmIiIiamsYnCEiIiIiTJ8+3ey+F198Ee+//z6efPJJZGRk4JZbbkFiYiIUCoU0x/BBaXOyN2JjY6VxTk6O2Xk33XQTHn30UaOHtsOGDcPixYsbPf+2bdvMPmwEdA/T9RkL/v7+0vYrHzC3VkvfD31j8crKykbfj0GDBuGNN97AU089JW3r1KkTVqxYYdmCr2CYCWOYIWNIEATcdddd+L//+z9oNBqsWbMGL730ksXXjoiIsPgccrkcXl5eiI6OxtChQ3HXXXeZDDRUVFTgrrvualZAQ6+srKzBtrfeegu7d+9GVlYWdu/ejd27d8PJyQn9+vXDiBEjMHr0aEycONFsEOSjjz7CuHHjUFJSgp9++gk//fQTlEolBg0ahBEjRmDs2LEYM2YMnJwa/r9vaWlp0njXrl3YtWtXs18LoMuuCAsLM7mvqcw0w+CRPhipZ/g57tSpU5PraGqO/nWKooiZM2c2eT5DjWWQNOfz1r17d/znP//BkiVLUFlZiSVLlmDJkiUICgrCDTfcgJEjR2Ly5MlG9zVHuPL+pc88NPTpp5/iySefbPT+aMjU57016urq8PDDD+Pzzz9vdhDJWtcmIiIiaqsYnCEiIiKiJi1atAhbtmzB33//jaSkJGzcuBF33HGHtL+8vFwaN5UlAsDom+qGx5py5QPPGTNmmHxIbeiBBx5Aenq62f2pqamIiYkBYPzt//Pnzzd63uZq6fsB6N6TysrKFr8fkydPhpeXV8sXaUZ1dTW+/fZbALqSa42VqJs/fz7+7//+DwCwatUqvPjiixZnH7m6urb4mFdffRVxcXEtPu7222/Hr7/+CkD3e/rXv/6F/v37IywsDG5ubtLn7OTJk3j55ZcBwGRpv6ioKCQmJmLp0qX48ssvUVhYiLq6OiQkJCAhIQHvv/8+vLy88Pjjj+Oll15qkBEzYMAAHD16FEuWLMF3332HiooKqFQq7NmzB3v27MGbb76J4OBgPP/883jssccgk9VXpy4pKWnx6zbUWBk2w+u0VGVlpTQ2LN1mTlNzLHmdjb3G5n7eXnvtNVx33XV48803sWfPHgBAfn4+fvjhB6k04/XXX4/33nsPQ4YMafVaLZGamiqNfX19G9wnN2zYYFRSctiwYRg1ahQ6dOgAb29vo8/lAw88gEuXLpktZdlSjz/+OD777DMAujKZkydPxnXXXYeIiAi4u7tLwf78/HwsXLgQgOm/NSIiIqKrCYMzRERERNQsU6ZMkUoLbd++3Sg44+npKY0NH8qaU1FRYfLYK508edKoHwsAvPLKK5g6darVSgoZltE6ePAg6urqmgz+NKWl74fhvMbej9zcXNxzzz1G21asWIFbbrnFqP+HJTZu3IjS0lIAuofahplFjblw4QJ27tyJMWPGWGUdtrZnzx4pMNO7d29s27bNbOktwywxcwICArBs2TK88847OHz4MPbu3Ys9e/bgr7/+QlFREcrKyrBkyRLs2bMH27dvbxD4iI6OxhdffIGPPvoIBw4cwL59+7B7927s3LkTFRUVuHjxIhYtWoRjx44ZlWczDHQ+8cQTeP/991vzdlidYVCyOVkaTf2deHh4oKSkBD4+PlbLcGupG2+8ETfeeCMuXryIf/75B/v27cOuXbtw5MgRiKKIPXv2YMSIEfj1118xfvx4u69v37590thUgOjFF18EoMss27x5s1GpyCvdf//9VltXZmYmPvnkEwC6YHh8fHyD0ph6pnoXEREREV2tWv9VKCIiIiK6phg+pDfs5wAAoaGh0jg5ObnJcxnOMVdSqaqqCrfffrtULunWW28FoMvsmD17doMySobS0tIgiqLZf/qsGQDo2bOnlD1TXl4ufQveEi19P3JycqSAlbn3Q6vVYu7cubh06RIAXQaRIAjQarW46667pL4jljIsaWbPY+1t27Zt0njp0qWN9kQxzEhoilwux3XXXYcnnngCGzZswMWLF/H999/D29sbAPDXX3+Z7PGkp1QqMXLkSDz33HP46aefcOnSJXzyySdSgGjNmjVGTekNy3KdPHmy2eu0NcPPcUpKSpPzL1y40Oh+/essKSlpcP+xt+DgYNx222147733kJCQgLS0NNx2220AALVajUWLFjlkXevWrZPGVwZJU1NTpczA6dOnNxqYKSsra7QUXEvt2LEDWq0WgK5fmLnAjH6dRERERNcKBmeIiIiIqFkMH/5fWapr6NCh0njbtm1N9hT4/fffTR5r6PHHH8epU6cAAHfffTc2btyIWbNmAQBOnDhh1HfFEoIg4IknnpB+fv311802qG6ufv36Sf1F9uzZ02RWQHPejzfeeAN//fUXAGD06NHYsGGDlFWUm5uL+fPnW7RmQFfWTZ8d5evri1dffbVZ//SloTZt2mRxmS17ycvLk8adO3dudK4+w6Y1nJycMHPmTKOya//880+zj3dxccHChQvx0EMPmTy+X79+8PHxkbZbK0hnqeDgYERGRgIATp8+bfR+mxIfH9/o/tGjR0tjawRQrSkqKgrffPMNAgMDAeiCZPb+O9i0aROOHj0KQPeZueuuu4z2t+Tz/vvvv0vBFHMMM7+aut/b62+NiIiIqL1hcIaIiIiImuW3336TxleWFIuKisLgwYMB6L75/P3335s9T3FxsVTiRhAEKSPG0Pfff48vvvgCANClSxd89NFHAIDPPvtMynr5+OOPsXXr1ta/IAP//ve/ER0dDUAX+HnooYeafDhp6LPPPsMff/wh/ezs7Cx9M72iogIffvih2WPVajXeffdd6Wf9N/AN7d27V3q4HxAQgHXr1kEmk+H//u//cN111wEAfvnlF/zvf/9r9ppNWbVqlfSg9Y477kBcXFyz/k2fPh2Arin8N998Y9Ea7MUwwNhYr6G9e/caBc9aq0OHDtK4rq7OasfL5XLMnTsXAKBSqfDSSy9ZsErruvnmmwHosr4++OADs/MKCgrw1VdfNXquefPmSeM333yzzQSh9BQKhVH/qtb8jlvr6NGjuPfee6Wf//3vfxtl7wHN/7zX1tZKfaQaY1hOr6ngc3OvnZKSgrVr1zZ5bSIiIqKrBYMzRERERNSk999/X/q2vkwmw+zZsxvMeeGFF6Txv//9bxw6dKjBnIqKCsyaNUsqzTV9+nR0797daE5aWhoeeOABALogx/r166UHgd7e3vjmm2+knjD33nuvVUocubu7Y9OmTVK2y6pVq3DjjTfi3LlzjR6XkJCAqVOnYuHChaiurjba9+yzz0rfLn/11Vfxyy+/NDherVbj3nvvxenTpwHoGsNPmjTJaE5JSQnmzJkjPexdtWqVVDJKoVDg22+/lfrUPPfcc9K351tKo9EYPRg1fBjeFMO57aW0mT6YCACLFy82WSbv+PHjmDlzZqOZAbm5uXjqqacaLd2lVqulZuiALttFLzExEYsXL0Zubq7Z4ysqKox+N4bHA7peIn5+fgB0gcLnnnsOarXa7Pmqq6uxevVqrF+/3uwca3jkkUekcmzvvvsufvzxxwZzqqqqMGfOnCYzTQYNGiTdd3JycjBp0qQmS2Dt378fzzzzTOsWb+CDDz7Ahg0bUFtba3bOP//8g+PHjwPQlWALCAiw+LpNKSkpwVtvvYVhw4ZJfaKGDx+OpUuXNpjbrVs36T66detWo/40etXV1Zg7d670OhpjGCw8cuRIo3MN/9beeecdFBYWNpiTkZGBm266qVn9iYiIiIiuFpZ1OiUiIiKiq8KWLVsabKuurkZaWhq2bt2KAwcOSNufeuop9OrVq8H8W265BfPnz8eaNWtQXFyM4cOHY86cORg9ejTc3Nxw6tQprFq1CllZWQB0jaH1GTR6dXV1uOOOO6QHjW+++SYGDBhgNGfYsGFYvHgxXnrpJRQWFuLOO+/EX3/91aDBeksNHDgQv/76K2bOnInCwkL89ttv2LZtG0aNGoUxY8YgIiIC7u7uKCwsxPnz57Ft2zacOHHC7PkGDRqEV155BXFxcVCpVJg2bRqmT5+OKVOmwNvbGykpKfjyyy9x5swZAICnpye+/vprCIJgdJ77778f6enpAIDHHnusQa+Ijh074pNPPsGdd94JlUqFO+64AwkJCQ1KzzXlt99+Q05ODgAgNjZWyshpjvHjxyMsLAw5OTk4cuQIjh492iCA0NbMmDEDUVFRyMjIQEJCAmJjY3Hfffehc+fOqKqqwq5du7B+/Xqo1WrMmzfP7Df6VSoVli1bhmXLlmHgwIEYMWIEevToAR8fH1RUVCAlJQXffvut1FOlY8eORsHN0tJSxMXF4bXXXsPw4cMxfPhwxMbGwsvLCyUlJTh9+jS++eYbqTTU0KFDMXbsWKM1hIaGYsOGDfjXv/6FmpoavP3221i3bh1mzpyJPn36wNPTE5WVlUhPT0dCQgL+/PNPVFVVYcmSJTZ6d3ViY2Pxyiuv4OWXX4Zarcb06dMxY8YMTJ48GZ6enjh79ixWr16NtLQ0zJo1S8q4M/e3/PnnnyM5ORlHjhzBkSNHEBsbi5tvvhkjRoxASEgINBoN8vPzceLECfz5559IS0tDp06d8M4771j0Oo4cOYK1a9fC29sbkyZNwoABAxAREQEnJyfk5+cjPj4eP//8s5Rt9+KLL1p0PcPr6kvWAbqsltLSUmRlZeHQoUP4+++/jbJWJk+ejK+//loKMhtydnbGQw89hLfffht1dXUYNWoU5s+fj+uuuw7u7u44deoU1q5di8zMTIwbNw5nz56V7tWmjBw5Es7OzqitrZXe3759+0KpVAIAXF1dMWrUKAC6e/aQIUNw4MABZGRkoFu3bnjggQfQvXt3aDQa7N+/H1999RUqKyul/4YQERERXRNEIiIiIromAWjRP4VCIb766quiVqs1e866ujrx0UcfFQVBaPRcffv2FdPS0hoc//zzz0tzpkyZYvZaGo1GHDt2rDT3tddes9r7kpqaKs6aNUuUyWTNel+CgoLE999/X1SpVCbPt3TpUlGhUDR6jpiYGDExMbHBsZ988onRe1ZTU2N23fPmzZPmLliwoMWve/r06dLxS5cubfHxzzzzjHT8I488YrRv1KhR0j5TVq9eLe1fvXp1s6736quvSse8+uqrLV6vKIpiQkKCGBAQYPb3IpfLxTfffFOMj483e620tLRm/w316tVLPH/+vNHxu3btavbxI0eOFPPz882+niNHjojdunVr1rnkcrn4+eefN/q+xsfHN/r+Nfa+GHrmmWcavSfMnj1bPH36tPTzY489ZvZcFRUV4vz585u8x+j/jRo1qsE5UlNTpf3z5s1r9DWKoijec889zb5Hvv76602erzGGf8fN/de7d+9m/d2oVCpx8uTJTb5fBQUFYnR0tAhAjI6ONnu+//znP2bPc+VxqampYocOHRq99iOPPCJeuHChRb8bIiIiovaMmTNEREREZJJSqYSPjw+6d+8ufcta3+/FHLlcjg8++AALFizAZ599hp07dyIrKwu1tbUIDAzEwIEDMXPmTNxxxx0Nvh2/Y8cOvP322wCAkJAQrF27tkEWiZ5MJsNXX32Fvn37oqCgAIsXL8a4ceMwfPhwi193TEwMvvvuO5w9exabN29GfHw8zp07h4KCAtTU1MDb2xtRUVEYNGgQpk6diqlTp0qlm0x54YUXMGvWLKxYsQI7duxAeno6Kisr4e/vj759++Lmm2/GggULpG+c6yUlJWHRokUAADc3N6xfv77BHEMffvgh9u3bh+TkZKxatQqTJk3CrFmzmvWaL168iJ9//hmA7r3V9zBpiXnz5knfoF+3bh3eeecdk9/gb0sGDhyI48eP47333sPPP/+M9PR0ODk5ISwsDGPGjMEDDzyAAQMGYOfOnWbPER0djYyMDMTHxyM+Ph5HjhxBRkYGysvL4ezsjJCQEPTv3x+33norZs2aJZXk0xs5ciSSk5Ol448fP46srCxUVlbCxcUF4eHhUkmvK7OmrtS/f38kJSVh8+bN2Lp1K/bv34+LFy+isrISHh4eiIyMRO/evTFmzBhMmzYNISEh1ngbm/T2229j2rRp+PDDD7F7924UFBRIn//77rsPt956q1F2nr5Emynu7u5YvXo1nn32WaxZswY7d+5EamoqiouL4ezsjMDAQMTGxmL48OGYMmVKizLAzPnkk08wf/58xMfHY/fu3Th79iwuXbqEuro6eHl5oUuXLhg9ejTuvfdedOnSxeLrmSIIAtzc3ODl5QU/Pz/07NkTAwYMwLhx4zBo0KBmncPZ2Rm//PIL1qxZg7Vr1+LYsWOorq5GYGAgevXqhTlz5mDu3LnNzkJcsmQJ+vbti9WrV+Po0aMoKCgwW/otJiYGiYmJ+O9//4sffvhB6j0TEhKC4cOH495778Xo0aORlpbWrGsTERERXQ0EUWykgDIRERERERGRjS1fvhyPPfYYAGDz5s2YPn26YxdERERERGRjDM4QERERERGRw6jVainrR6FQIDs7G4GBgY5eFhERERGRTVnWNZWIiIiIiIjIjIKCAiQlJZndX1NTgwULFkhzbrvtNgZmiIiIiOiawMwZIiIiIiIisomjR4+if//+GDRoEMaNG4fY2Fh4eXmhvLwcx48fx/r165GbmwtA12vmxIkTCAsLc/CqiYiIiIhsz6npKUREREREREStl5CQgISEBLP7O3TogK1btzIwQ0RERETXDGbOEBERERERkU3U1tbi999/xx9//IF9+/YhPz8fhYWFAICAgAD069cPN910E+bNmwdnZ2cHr5aIiIiIyH4YnCEiIiIiIiIiIiIiIrIjljWzgFarRU5ODjw9PSEIgqOXQ0REREREREREREREDiSKIsrLyxEWFgaZTGZ2HoMzFsjJyUFkZKSjl0FERERERERERERERG1IZmYmIiIizO5ncMYCnp6eAIDU1FT4+fk5eDVE5EhqtRrbtm3DxIkToVAoHL0cInIg3g+ISI/3AyLS4/2AiAzxnkB0dSsrK0NkZKQUPzCHwRkL6EuZeXp6wsvLy8GrISJHUqvVcHNzg5eXF/8PK6JrHO8HRKTH+wER6fF+QESGeE8gujY01QrFfMEzIiIiIiIiIiIiIiIisjoGZ4iIiIiIiIiIiIiIiOyIwRkiIiIiIiIiIiIiIiI7YnCGiIiIiIiIiIiIiIjIjhicISIiIiIiIiIiIiIisiMGZ4iIiIiIiIiIiIiIiOzIydELuBap1WpoNBpHL4OoAblcDoVC4ehlEBEREREREREREV3VGJyxo7KyMhQUFEClUjl6KURmKZVKBAQEwMvLy9FLISIiIiIiIiIiIroqMThjJ2VlZcjOzoaHhwcCAgKgUCggCIKjl0UkEUURarUapaWlyM7OBgAGaIiIiIiIiIiIiIhsgMEZOykoKICHhwciIiIYlKE2y9XVFZ6ensjKykJBQQGDM0REREREREREREQ2IHP0Aq4FarUaKpUK3t7eDMxQmycIAry9vaFSqaBWqx29HCIiIiIiIiIiIqKrDoMzdqDRaACAjdap3dB/VvWfXSIiIiIiIiIiIiKyHgZn7IhZM9Re8LNKREREREREREREZDsMzhAREREREREREREREdkRgzNERERERERERERERER2xOAMERERERERERERERGRHTE4Q0REREREREREREREZEcMzhAREREREREREREREdkRgzNERERERERERERERER2xOAMERERERERERERERHZVWmVGlqt6OhlOAyDM+QQhw4dgiAIuP76683OWbx4MQRBwOuvv27HlRERERERERERERGRLW1OzEK/Jdtw80d7UFxZ6+jlOASDM+QQgwcPxsCBA7F3714kJSU12K/VarF69WrI5XLcc889DlghEREREREREREREdnCd4cyIYrAiexSLPz6MFR1Gkcvye4YnCGHWbhwIQDgiy++aLBv27ZtSE9Px9SpUxEeHm7vpRERERERERERERGRjaRcqpTGB1OL8PymExDFa6vEmZOjF0A605bvxqVylaOX0SyBnkr89OgNFp9nzpw5ePrpp/HVV1/hzTffhFKplPbpAzb333+/xdchIiIiIiIiIiIiorahtFrd4Fn45sRsRPu74YnxXR20KvtjcKaNuFSuQl5ZjaOXYVfu7u648847sWLFCmzevBmzZ88GAOTn5+PHH39EWFgYpk6d6uBVEhEREREREREREZG1pFyqkMZdgz1wLr8Cogj8d8c5RPu74Zb+EQ5cnf0wONNGBHoqm57URlhzrQ8++CBWrFiBzz//XArOrFmzBmq1GgsWLIBcLrfatYiIiIiIiIiIiIjIsc7n1wdnZg+OgkYr4v9+PQ0AeHbjcYR5u2JIR39HLc9uGJxpI6xRJqw96tOnD4YOHYr4+HikpKSgU6dOWLlyJQRBwL333uvo5RERERERERERERGRFRlmznQK8sDILgFIK6zEugMZUGtEPPDVYfzw0HB0CvRw4CptT+boBRA9+OCDEEURK1euxK5du5CcnIwJEyYgJibG0UsjIiIiIiIiIiIiIitKMcic6RzkAUEQsPimnhjVNRCArifNgjWHUFRZ66gl2gWDM+Rws2bNgq+vL9asWYMVK1YAAO6//34Hr4qIiIiIiIiIiIiIrC3lUiUAwFUhR6iXCwDASS7Dh3P6o1uIJwAgvbAKD3yZgBq1xmHrtDUGZ8jhXF1dcffddyM3NxffffcdAgMDcfPNNzt6WURERERERERERERkRao6DdILdcGZTkHukMkEaZ+niwKr5g9G0OWe5wnpxXhm43FotaJD1mprDM5Qm7Bw4UJpPH/+fCgUCgeuhoiIiIiIiIiIiIisLb2wCvpYi6meMmE+rlg1fzBcFXIAwE/HcrBse7I9l2g3DM5Qm9C9e3eEhYUBAO677z4Hr4aIiIiIiIiIiIiIrO28Yb8ZE8EZAOgV7o3ld/SHcDmp5sP48/g+IdMey7MrBmeoTdi7dy9ycnIwatQodO3a1dHLISIiIiIiIiIiIiIrSzEIznQKMh2cAYDxPYLxyo09pJ9f/OEE9p4vsOna7I3BGWoTli5dCgB45JFHHLwSIiIiIiIiIiIiIrKF85cMMmcaCc4AwD3Xd8D84TEAgDqtiJe3nrTl0uyOwRlymL179+Lee+/FkCFD8Msvv2DgwIGYMWOGo5dFRERERERERERERDaQcjk4IxOAaH+3Jue/fGMPdAvxvHxsJSpVdTZdnz212+BMdnY25s6dC39/f7i5uaFfv344fPiwtF8URcTFxSEsLAyurq4YPXo0kpKSjM6hUqnw6KOPIiAgAO7u7rjpppuQlZVl75dyzUpOTsaqVatw+vRpTJs2DT/88ANksnb7kSQiIiIiIiIiIiIiM7RaESn5lQCAKD83KJ3kTR4jlwnoGeYt/ZxeWGWz9dlbu3wSXlxcjOuvvx4KhQK//fYbTp06hffeew8+Pj7SnLfffhvLli3Dhx9+iEOHDiEkJAQTJkxAeXm5NOeJJ57A5s2bsX79euzevRsVFRW48cYbodFoHPCqrj3z58+HKIooKyvDjz/+iKioKEcviYiIiIiIiIiIiIhsILesBtVq3bP3pkqaGYoxyLBJL6y0+rocxcnRC2iNt956C5GRkVi9erW0LSYmRhqLooj//ve/eOmll6QyWWvXrkVwcDC++eYbLFy4EKWlpVi5ciW++uorjB8/HgDw9ddfIzIyEjt27MCkSZPs+pqIiIiIiIiIiIiIiK5W5/Pr+810Cmx+cCY6wF0ap11FmTPtMjjz448/YtKkSZg5cyZ27dqF8PBwPPTQQ7j//vsBAKmpqcjLy8PEiROlY5RKJUaNGoW9e/di4cKFOHz4MNRqtdGcsLAw9OrVC3v37jUZnFGpVFCpVNLPZWVlAAC1Wg21Wm12vWq1GqIoQqvVQqvVWvz6iWxNq9VCFEWo1WrI5U2nFxKke0Bj9wIiujbwfkBEerwfEJEe7wdEZIj3BLpWJeeVSuMYf9dm/w1EeDtL47SC8jb/t9Pc9bXL4MyFCxewYsUKPPnkk3jxxRdx8OBBPPbYY1Aqlbj77ruRl5cHAAgODjY6Ljg4GOnp6QCAvLw8ODs7w9fXt8Ec/fFXeuONN7B48eIG2+Pj4+HmZr55kZOTE0JCQlBRUYHa2toWvVYiR6itrUV1dTX+/vtv1NVdPU227GH79u2OXgIRtRG8HxCRHu8HRKTH+wERGeI9ga41Oy/IoO+0cvHcMfyad6xZx1XVAfpQxpHkTPz6a7ptFmglVVXNy+5pl8EZrVaLQYMGYenSpQCA/v37IykpCStWrMDdd98tzRMEweg4URQbbLtSY3NeeOEFPPnkk9LPZWVliIyMxJgxY+Dv72/2nDU1NcjMzISHhwdcXFyafH1EjlZTUwNXV1eMHDmSn9lmUqvV2L59OyZMmACFQuHo5RCRA/F+QER6vB8QkR7vB0RkiPcEulatW3kIQDEAYO5NE+Dt2vzP/9tJ8SiuUqNCcMPUqSNttELr0Ffcakq7DM6EhoaiR48eRtu6d++OTZs2AQBCQkIA6LJjQkNDpTn5+flSNk1ISAhqa2tRXFxslD2Tn5+P4cOHm7yuUqmEUqlssF2hUDR6I9VoNBAEATKZDDKZrJmvkshxZDIZBEFo8rNNDfE9IyI93g+ISI/3AyLS4/2AiAzxnkDXmgsFuoySAA8lArzMV6IyJdrfHcVVJcgtrYEGMrgo2m4rhub+XbfLSMH111+Ps2fPGm1LTk5GdHQ0AKBDhw4ICQkxSg2sra3Frl27pMDLwIEDoVAojObk5ubi5MmTZoMzRERERERERERERETUMqVVahRU6Pq5dw5yb/HxMf71wZzMouaVDWvr2mXmzKJFizB8+HAsXboUs2bNwsGDB/HZZ5/hs88+A6ArZ/bEE09g6dKl6NKlC7p06YKlS5fCzc0Nc+bMAQB4e3vj3nvvxVNPPQV/f3/4+fnh6aefRu/evTF+/HhHvjwiIiIiIiIiIiIioqvG+UsV0rhToEeLj4/2rw/opBVWoUuwp1XW5UjtMjgzePBgbN68GS+88AJee+01dOjQAf/9739x5513SnOeffZZVFdX46GHHkJxcTGGDBmCbdu2wdOz/pf2/vvvw8nJCbNmzUJ1dTXGjRuHNWvWQC5vuylRRERERERERERERETtSUp+fXCmc1DLgzMxAfWZM+mFlVZZk6O1y+AMANx444248cYbze4XBAFxcXGIi4szO8fFxQXLly/H8uXLbbBCIiIiIiIiIiIiIiJKsWrmzNURnGmXPWeI2oK4uDgIgoA1a9Y4eilEREREREREREREbdZ5SzNnDIIz6YVXR88ZBmfIIdLS0iAIAkaPHu3opRARERERERERERGRDekzZ9yc5Qj1dmnx8b5uCni66AqBMXOGiIiIiIiIiIiIiIioETVqDTKKdNkunQI9IAhCi88hCIKUPZNdXI3aOq1V1+gIDM4QEREREREREREREZFNpBdWQSvqxp0C3Ruf3IhofzcAgFYEsorbf2kzBmfI7uLi4tChQwcAwK5duyAIgvRv/vz5AC5HQmNiUFtbi9deew3dunWDUqnE9OnTpfNUVFTgtddeQ+/eveHm5gYvLy+MGjUKW7ZsaXBNwzJq1dXVeP755xEdHQ2lUonOnTvjrbfegiiKJte7a9cujB49Gh4eHvD398ctt9yCM2fOWPttISIiIiIiIiIiIrrqWNpvRk8fnAGujr4zTo5eAF17+vXrh1tvvRWbNm1CcHAwJk+eLO274YYbpLFWq8X06dPx999/Y9SoUejTpw/8/f0BABcvXsTYsWNx6tQphIeHY8KECaiqqsK+fftwyy234I033sDzzz/f4Nq1tbWYOHEikpKScN1116F79+7YtWsXnn/+eZSXl+P11183mr9161bceuut0Gg0GD58OKKionDw4EEMGTIE06ZNs9E7RERERERERERERHR10PebAXRlzVor2r8+6+Zq6DvD4AzZ3fTp09GvXz9s2rQJ3bp1w5o1a0zOy8zMhFKpxNmzZxEeHm6075577sGpU6fw7LPP4vXXX4dCoQAAXLhwARMnTsR//vMfTJ06FX369DE6bt++fRgxYgSSk5MREBAAAEhISMCwYcPw/vvv4/nnn4eHh+4GUV5ejvvuuw8ajQbffPMN7rjjDgBAXV0d7rvvPqxdu9aabwsRERERERERERHRVcdamTMxBsGZqyFzhmXNqE174403GgRmjh49it9++w3Dhw/Hm2++KQVmAKBjx4547733oNFo8MUXXzQ4n0wmwxdffCEFZgBg0KBBmDJlCqqqqpCQkCBt37BhAwoKCjBhwgQpMAMATk5OeP/996UgDhERERERERERERGZps+ckcsEo+yXlooxKGvGzBmynk9HARX5jl5F83gEAQt32fwygiCYLB22fft2AMDNN98MQRAa7NeXRjt06FCDfTExMejatWuD7fptubm50rbdu3cDAGbNmtVgvq+vLyZOnIgffvihOS+FiIiIiIiIiIiI6Jqj1YpScCbazw3OTq3PFwn0VMJVIUe1WoOMqyBzhsGZtqIiHyjPcfQq2pSgoCAolcoG29PS0gAAzz33HJ577jmzxxcUFDTYFhERYXKuPgtGpVJJ23JydL+PqKgok8eY205EREREREREREREQE5pNWrUWgBARwv6zQC6L/NH+7vhTF45MourUKfRwknefouDMTjTVngEOXoFzWentbq4uJjcrtFoAAAjRoxAx44dzR5vWLpMz1SmjTmiKLb4GCIiIiIiIiIiIiLSsVa/Gb0Yf3ecySuHWiMit7QGkX5uTR/URjE401bYoUzY1UKf/XLbbbfhscces9l1wsLCAADp6ekm92dkZNjs2kRERERERERERETtXcql+t4wnQJb329GLzrAuO9Mew7OtN+cH2rXnJ2dAQB1dXUtPnb8+PEAgC1btlhzSQ3oe9ds2LChwb6SkhJs27bNptcnIiIiIiIiIiIias9skTmjl9bO+84wOEMOERAQAIVCgZSUFKlMWXMNHToU48aNQ3x8PBYtWoSKigqj/VqtFtu2bcPu3bstWuPMmTPh5+eHbdu24fvvv5e2azQaPPXUUw2uS0RERERERERERET1Ui7VP0O1tOcMAET712fKpBdUNjKz7WNwhhzC2dkZkydPRl5eHvr27Yu7774b9913H1avXt2s49etW4c+ffrgv//9L6KjozFu3DjMnj0bI0aMQEhICCZNmoSEhASL1ujl5YXPPvsMMpkMt99+O2644QbMmTMHsbGx2LhxI+68806Lzk9ERERERERERER0NUu5nDkT6KmEt6vC4vMxc4bICr744gvcddddKCwsxDfffIOVK1di167m9d4JDg7G/v37sWzZMnTp0gWHDh3Cli1bkJWVhf79++Ojjz7C3LlzLV7jrbfeiu3bt2PEiBFITEzEb7/9hh49emDfvn3o3LmzxecnIiIiIiIiIiIiuhoVV9aisLIWANDZClkzABDi5QJnJ11YI72wfWfOODl6AXTtCgoKwpdffmlynyiKTR7v6uqKRYsWYdGiRU3OjYmJafSccXFxiIuLM7lv7NixGDt2bIuOISIiIiIiIiIiIrqWGZY06xTk3sjM5pPJBET7ueFcfgXSi6qg1YqQyQSrnNvemDlDRERERERERERERERWZRicsVbmDABEXy5tVlunRV5ZjdXOa28MzhARERERERERERERkVWdzzfMnLFecCbG300ap7Xj0mYMzhARERERERERERERkVWlXKoPnHS2YnAmOqC+RFp6YZXVzmtvDM4QEREREREREREREZFV6TNn3J3lCPFysdp5mTlDRERERERERERERER0hRq1BpnFuqyWTkEeEATBaueO8TfInClg5gwRERERERERERERERHSCishirpxp0DrlTQDgFBvFyjkgnSd9orBGSIiIiIiIiIiIiJqc1IuVeCLfy4gu6Ta0UuhFtKXNAMs7zeTXZGNtUlrkVGWAQBwkssQ6asrbZZeWAVRHwVqZ5wcvQAiIiIiIiIiIiIiIkNbj2bjuU3HUaPWYsfpi1j/wDBHL4laICW/PqOlU6B7IzMbtzNzJ1745wVUqCvwY8qP2HTTJgBAlL8bLhRUolqtwaVyFYKs2NPGXhicISIiIiIiIiIiIqI2oU6jxZu/ncEXu1OlbYkZJdBqRchk1utbQrZ1/pJlmTNaUYtPj3+Kj49+LG1LLk5GlboKbgq3y31nLgEA0ouq2mVwhmXNiIiIiIiIiIiIiMjhiiprMW/1QaPADACo6rTIK6tx0KqoNVIulzWTywRE+bUsc6aitgJPxD9hFJjRyyzPBABE+7tJ29IK2mffGQZniIiIiIiIiIiIiMihknJKMW35buw5XwgAUMgFdAvxlPa31wfw1yKtVsSFAl1wJtrfDc5OzQ9DpJamYs6vcxCfGQ8AkAkydPfrLu1PL0sHgMuZM5e3FVZZY9l2x+AMERERERERERERETnM1qPZuHXFXmSXVAMAAjyU+Pb+oZg7NFqak1rI4Ex7kV1SjRq1FgDQKbD5Jc3iM+Ix55c5SC3VZU55OXthxbgVWNB7gTQnozwDwBWZM+30s8GeM0RERERERERERERkd3UaLd747QxWGpQx6xfpg0/mDkSIt4v0gB9g5kx70tJ+M1pRi0+PfYqPj9WXMevi2wX/G/0/RHpF4lThKWm7PnMmwtcNMgHQiu03c4bBGSIiIiIiIiIiIiKyq6LKWjzyzRHsTSmUts0eHInFN/eE0kkOAIgJMMyOaJ8P4K9F+n4zQNOZMxW1FXhh9wvYmblT2jYpZhJeG/4a3BS633+0V30GVUaZLnPG2UmGcF9XZBZVI62wEqIoQhAE670IO2BZM7rqrFmzBoIgSP98fHwazBEEATExMXZfm97PP/+MF198EePHj4e3tzcEQcDkyZPNzq+qqsKWLVtw7733ok+fPvDy8oK7uzv69u2L1157DRUVFSaPmz59utF7MX/+fBu9IiIiIiIiIiIiouY5ma3rL6MPzCjkAl6f3gtvzOgtBWYAIMzbVepXwsyZ9iOlmZkzF0ov4I5f7pACMzJBhkUDF+Gdke9IgRkAcFe4w9/FH0B9WTOgvu9MeU0diqvUVnwF9sHMGbpq9e3bF/369YObm1vTk+1s7ty5KC0tbfb8b775Bvfffz8AoGfPnpg8eTLKysqwd+9evPrqq/j222+xa9cuBAUFGR03duxY+Pj4IC8vD3/88YdVXwMREREREREREVFL7UspxPzVB6Gq05UsC/RUYsWdAzAoxq/BXJlMQJSfG87nVyC9qAparQiZrH1lR1yLUvLrA2kdA91NzkkqTMK9f9yLSrVurpezF94Z+Q6Ghw83OT/KKwqFNYUoqC5ApboS7gp3RPu74Z9zuv1phZXwc3e27guxMQZn6Ko1ffp0xMXFOXoZJt16663o3r07Bg8ejPLyckybNq3R+c7Ozvj3v/+NRYsWoUuXLtL23Nxc/Otf/0JiYiKeeOIJfPPNN0bHPfbYYwCAnTt3MjhDREREREREREQO9/72ZCkw0z9K118m2MvF7PwYf3ecz69AbZ0WuWU1CPdxtddSqZX0PWeCvZTwclGYnPPpsU+lwEwX3y7435j/IdIz0uw5ozyjkJifCADILM9EN79uUuYMAKQXVmJAlK+1XoJdMDhD5AArV66Uxjt37mxy/t1334277767wfbQ0FB89NFHGD58OH744QfU1tbC2bl9RYiJiIiIiIiIiOjaoNWKOJmjqyYT6u2C9Q8MNSpjZkoHw74zBZUMzrRxRZW1KKqsBdB4v5lThacAAB4KD3w95WujMmamGPadSS9LRze/bog2CM6kFbS/nkTsOUMOcejQIQiCgOuvv97snMWLF0MQBLz++ut2WZMoivj2228xe/ZsdO3aFe7u7vD09MR1112Hjz/+GFqt1uRxFRUVePrppxEZGQlXV1f06NEDH3zwgdSEyta9bfr27QsAUKlUKCwsbGI2ERERERERERGRY2QWV6GqVgMA6BXu3WRgBoDRA/hU9p1p85rTb6ZUVYqLVRcBAF19uzYZmAGASK/6rJqMMl3fmRj/+uPSC9vfZ4OZM+QQgwcPxsCBA7F3714kJSWhZ8+eRvu1Wi1Wr14NuVyOe+65xy5rUqlUmDNnDnx9fdGjRw8MGDAABQUF2LdvHx5++GEcPHgQa9asMTqmpqYG48aNw8GDBxEYGIgbb7wRFRUVeOaZZ5CSkmKXdV+4cAEAoFAo4OfXsDYnERERERERERFRW3A6t1wadw/xbNYxHQKMS1dR25aSXx+cMZc5k1ycLI27+nZt1nmjPeszZzLKdcGZSD83CAIgikBaITNniJpt4cKFAIAvvviiwb5t27YhPT0dU6dORXh4uF3W4+TkhE2bNiEvLw+7d+/G+vXrsWPHDqSlpWHQoEFYu3Yt/v77b6Nj3n33XRw8eBDDhg3D+fPnsWHDBvz22284dOgQvvrqK7us+3//+x8AYPLkyVAqlXa5JhERERERERERUUudzi2Txt1DvZp1TEyAYeZM+3sAf605n9905oxhcCbWL7ZZ543yipLG+swZF4UcoZf7FbXHwB0zZ9qI23++HQXVBY5eRrMEuAbguxu/s/g8c+bMwdNPP42vvvoKb775plFgQR+wuf/++y2+TnM5OTlhxowZDbYHBgbijTfewIQJE7B161aMHDlS2vfpp58CAJYtWwYvr/r/oPTp0wePPvooXnvtNZuu+ddff8XKlSuhUCiwZMkSm16LiIiIiIiIiIjIEmfy6oMz3ZoZnAn1coGzkwy1dVqktcMH8Ncaw7Jm1syccVe4I8A1AAXVBUgvS5e2R/u7I6e0BsVVapRWqeHtpmjlyu2PwZk2oqC6APlV+Y5ehl25u7vjzjvvxIoVK7B582bMnj0bAJCfn48ff/wRYWFhmDp1qt3XdfToUSlzp6qqCqIoorxcl3J57tw5aV5GRgaysrIQERGBoUOHNjjPzJkzbRqcOX36NObOnQtRFPHOO+9IvWeIiIiIiIiIiIjaojN5umdsrgo5ovya7jMCADKZgGg/N5zLr0BGYRU0WhFymWDLZZIFzl8OzngonRDsZbrKT3KRLjgjQEBnn87NPneUZxQKqgtQWFOISnUl3BXuiAlww74Luj7c6UWV6OPmY9kLsCMGZ9qIANcARy+h2ay51gcffBArVqzA559/LgVn1qxZA7VajQULFkAub7opmLXU1tZi/vz5+Pbbb83O0QdpACAnJwcAEBkZaXJuVFSUye3WkJWVhcmTJ6O4uBhPPvkkHn/8cZtdi4iIiIiIiIiIyFIVqjqkX+4LEhvi2aIAS0yAO87lV6BWo0VuaTUifJsX2CH7qlFrkFVcDQDoFOgOQWj4O9ZoNThfch6ArlSZm6L5v8sorygcyT8CQFfarLt/d0T715e9SyusQp8IHwtegX0xONNGWKNMWHvUp08fDB06FPHx8UhJSUGnTp2wcuVKCIKAe++9165rWbZsGb799lv06tUL77zzDgYMGABfX18oFAokJycjNjYWoig2OM7UTcaWCgoKMGHCBGRkZOCee+7Bu+++a9frExERERERERERtdTZvPovPXcP9WzRsR0M+s6kFVQxONNGXbhUCf3jU3MlzTLKM1CjqQHQ/JJmetFe0dI4vTwd3f27I8a//rOQ0c7K3skcvQCiBx98EKIoYuXKldi1axeSk5MxYcIExMTE2HUdmzdvBgB8++23mDx5MoKCgqBQ6GoUXrhwocH80NBQALryZqaY226J8vJyTJkyBWfOnMGMGTPw+eef2z04RERERERERERE1FJG/WZCmtdvRi/a4AE8+860XUb9ZoKa7jfTxbdLi84f6VlfwSizLBMAGmTOtCcMzpDDzZo1C76+vlizZg1WrFgBALj//vvtvo7i4mIApsuUff/99w22RUdHIywsDFlZWThw4ECD/Rs3brTq+lQqFW6++WYkJCRg0qRJ+Pbbb+1a9o2IiIiIiIiIiKi1TufWB2e6h7YsONPB8AF8AYMzbdW5fIPgjJnMmbNFZ6VxrG9si85vlDlTlq7bZhC4S29ngTsGZ8jhXF1dcffddyM3NxffffcdAgMDcfPNN9t9HV276tLoPvnkE6PtGzduxJdffmnymIULFwIAnnrqKaN+NCdPnsTy5cuttjaNRoM77rgD8fHxGDFiBH744Qc4Oztb7fxERERERERERES2dCa3/tlZbEjLyprFGJY1a2cP4K8lxzJLpHHPMNMBuHPF56RxS8uaRXnW9/jOKNdVLXJzdkKgpxJA+8ucYc8ZahMWLlyI//3vfwCA+fPnS+XE7OnZZ5/F77//jueffx4bNmxA165dce7cOSQkJODpp5822dvlmWeewU8//YQ9e/agU6dOGD16NCoqKvDXX3/h/vvvx4cffmgyiLJkyRL88ssvAICyMt23Bvbv34+hQ4dKczZv3iyVTvvwww+lsmsBAQF46KGHTL6Gd999FwEBAZa9EURERERERERERFYkiiLOXO45E+7jCm/Xlj37C/FygdJJBlWdFqnMnGmTtFoRRy8HZwI8nBHh62pynr6smbvCHWEeYS26hpvCDQGuASioLpAyZwAgxt8Nl8pVuFSuQqWqDu7K9hH2aB+rpKte9+7dERYWhpycHNx3330OWcPIkSOxe/duvPTSS0hMTERycjJ69+6NTZs2YcCAASaDM66urvjzzz8RFxeH77//Hlu3bkWHDh2wdOlSzJo1Cx9++CH8/f0bHJeSktKgFFppaanRNpVKJY31JdeA+t44psTFxTE4Q0REREREREREbUpWcTUqVHUAgO6hLcuaAQCZTEC0vxuSL1Ygs6gaGq0IuYx9mNuS1MJKlFarAQD9In1N9skuqy1DTmUOAF3WjExoeWGvKM8oFFQXoKimCBW1FfBw9kC0vzsOpemen6YXVqGHmaydtobBGWoT9u7di5ycHIwaNUoqL2ZLoiia3D506FD8+eefLTrGy8sLy5Ytw7Jly4y2f/fddwCAfv36NThmzZo1WLNmTbPXGxcXh7i4uGbPJyIiIiIiIiIiaisM+810C2ndg/MYf3ckX6xArUaLnJJqRPq5NX0Q2U1iRok07h/lY3KOJSXN9KK9onEk/wgAXWmzHv49EHNF3xkGZ4haYOnSpQCARx55xGrn3LJlC9LS0uDm5oaPP/7Yaue90tGjR9GnTx/IZPWR3hMnTuDZZ58FAMyZM8dm127KBx98gCNHjiAvL89hayAiIiIiIiIiomubvqQZAHRrReYM0LDvDIMzbUtiRn3lH3PBGX1JM6D1wZkoL4O+M2W64Ey0v+Fno/30nWFwhhxm7969WLlyJU6ePImDBw9i4MCBmDFjhtXOf+zYMRw7dgze3t42Dc7Mnj0bZWVl6N27N3x9fZGWloaEhARoNBo8+OCDGDFihM2u3ZS//voLW7duddj1iYiIiIiIiIiIDDNnuoe2PnNGL62wCiO6WLwssiJ95oxMAPpE+Jicc7borDRudXDG0yA4U54BwPizkV7YfnoSMThDDpOcnIxVq1bB09MT06ZNw4cffmiUfaL39NNPo6CgoFnnXLNmDebPn4/58+dbebXmPfroo1i/fj2OHj2K4uJiuLm5Yfjw4bj33nsxb948u63DlC1btjj0+kRERERERERERPrMGaWTzOhBekvEBNRnyqQVtJ8H8NeCqto6nMnTBeC6BnvCQ2k67GBY1qyLb+uia9Fe0dI4vSwdABBlUNYsjcEZoqY1N4iyceNGpKenN+ucLenjYi0PP/wwHn74Ybtfl4iIiIiIiIiIqK2rqq2THpjHhnhCLmvYKL45OhiWNWNwpk05nlUK7eV23f2jfE3O0Wg1OFeiC85EeETAXdG6IF2kZ6Q0zijTZc54uyrg5+6MospapLejsmYN0xSI2pi0tDSIotisf0RERERERERERNR2nM0rh/6xXfeQ1jdqD/Z0gdJJ9zi7PWVHXAv0Jc0A8/1msiqyUF1XDQCI9Ytt9bXcFG4IdA0EUF/WDACiL2fP5JbWoEatafX57YnBGSIiIiIiIiIiIiKyCX1JMwDoFurZ6vPIZIJUEi2zqBoaLb+o3VYczSyWxgPMBGeSi5OlcWv7zehFeen6zhTVFKGitgKAcd+ZjKL2kT3D4AwRERERERERERER2cTp3DJp3M2CzBmgvu9MrUaLnJJqi85F1iGKIo5czpzxdHFCxwAPk/POFp2VxpYGZ4z6zpTr2mFE+7e/nkQMztgRy25Re8HPKhERERERERERWcOZ3PrMme4WZM4AxtkRLG3WNuSU1uBSuQoA0C/SBzIzPYUMM2difVtf1gww7juTWZYJwPiz0V76zjA4Ywcyme5t1mjaR607Iv1nVf/ZJSIiIiIiIiIiailRFHE6T5c5E+rtAh83Z4vOFxNgEJxpJ9kRV7vEjPqSZv0jfczO0wdnXJ1cEe4ZbtE1jTJnykxkzrSTwB2fvNqBQqGAXC5HdTVT7ah9qK6uhlwuh0KhcPRSiIiIiIiIiIioncouqUZ5TR0AoFuIZVkzgHF2RGpB+8iOuNolXi5pBgD9o3xNzqmorUB2RTYAoItvF8gEy8ISUZ5R0jijPAMAM2fIDEEQ4ObmhtLSUmbPUJun0WhQWloKNzc3CILpNEQiIiIiIiIiIqKmGJc0s6zfDFDfcwYA0ttJdsTVzjBzpp+ZzJlzJeeksaUlzQDjsmYZZbrgjI+bAl4uTgCA9KL28dlwcvQCrhVBQUFIS0tDeno6/Pz8oFQq+eCb2hRRFKFSqVBUVAStVougoCBHL4mIiIiIiIiIiNqxM5dLmgFANysEZ4I9XeCikKFGrUUqgzMOp6rT4GSO7nfcIcAdvu6my9YlF9X3m+nq29Xi67op3BDkGoT86nwpc0YQBMQEuON4Vimyi6tRW6eFs1Pbzk1hcMZOnJ2dERERgYKCAuTm5jp6OURmubu7IyQkBM7OltUAJSIiIiIiIiKia9tpw8wZK5Q1k8kExPi740xeOTKLqlCn0cJJ3rYfwF/NTueWo7ZOC6DxfjNni89KY2sEZwAgyisK+dX5KKopQnltOTydPRHtrwvOaEUgq7gKHQM9rHItW2Fwxo7c3NwQFRWFuro61NXVOXo5RA04OTnByYm3BSIiIiIiIiIistzpy5kzzk4ydAhwb2J28+iDM2qNiJySGkQZNIIn+zIsadY/ysfsvOTi+syZLr5drHLtaK9oJFxMAKArbdYzoCdi/A3L3jE4QybwATgRERERERERERFdzaprNUgr0JUe6xrsYbUMl2iDvjNphZUMzjhQYkaJNO4f5WtyjlbU4lyxrudMuEc4PJ0tz6ACrug7U64LzkT71wcA09pB2TvmfBERERERERERERGRVSVfLIdW1I27hVjeb0avQzt7AH81S8zUZc64KGSINVO2Lrs8G1V1VQCslzUD6DJn9NLL0gGgQeZMW8fgDBERERERERERERFZ1ZnLJc0AoHuo9YIzMQbl0VILGJxxlIIKFTKLqgEAfcJ9oDCTGWVY0izWN9Zq14/yipLGGWUZum0GwZkL7eCzweAMEREREREREREREVnV6dxyadzdTFZFa8QYZM60h+yIq9VRo5JmPmbnGQZnuvp2tdr1ryxrBgCBHkr4uTsDAI6kF6O2Tmu169kCgzNERERERERERETUZtRptHh160kM/r8d+OlYjqOXQ610Orc+c6ZbKzJntKIW/z38X4z5fgw2Jm+Utgd7KeGqkAOA1NOG7E9f0gwA+kX6mJ13tvisNLZmcMbVyRVBbkEA6jNnBEHAqK6BAIAKVR0OpRVZ7Xq20C6DM3FxcRAEwehfSEiItF8URcTFxSEsLAyurq4YPXo0kpKSjM6hUqnw6KOPIiAgAO7u7rjpppuQlZVl75dCREREREREREREl2m0Ip7acAxr96XjUrkKH+9McfSSqBVEUcSZPF3mTLBXfTZDS45femApVp5ciYLqAnx6/FNpnyAIiL5cviqjqAp1mradHXG1SjTKnPE1O0+fOePq5GqU7WIN+r4zxapilNXqgoFjugVJ+/86k2/V61lbuwzOAEDPnj2Rm5sr/Ttx4oS07+2338ayZcvw4Ycf4tChQwgJCcGECRNQXl6fSvfEE09g8+bNWL9+PXbv3o2KigrceOON0Gg0jng5RERERERERERE1zSNVsQzG49h69H6bJn0wkqIoujAVVFr5JXVoLRaDQDoFtKyrBlRFPH2obfx3dnv6s9XmYeauhrpZ31pszqtiJySmgbnINvSaEUcyywBAIR6uyDE28XkvCp1FTLLMwEAnX06Qy6TW3UdUZ71fWcyy3TXGdUlEHKZAACIZ3DGNpycnBASEiL9CwzUpSuJooj//ve/eOmllzBjxgz06tULa9euRVVVFb755hsAQGlpKVauXIn33nsP48ePR//+/fH111/jxIkT2LFjhyNfFhERERERERER0TVHqxXxwg/H8cORbKPtVbUaXKpQOWhV1FrGJc2a329GFEW8m/Auvj79dYN92RX1n42YgPq+M6mFLG1mb+fyy1FZq0tycES/Gb0or/rgTHpZOgDA202BgZczeS4UVLbp0ndOjl5Aa507dw5hYWFQKpUYMmQIli5dio4dOyI1NRV5eXmYOHGiNFepVGLUqFHYu3cvFi5ciMOHD0OtVhvNCQsLQ69evbB3715MmjTJ5DVVKhVUqvr/GJSV6W4yarUaarXaRq+UiNoD/T2A9wIi4v2AiPR4PyAiPd4PiBqn1Yp45afT+D5B13JALhPQNcgDpy+XxbpwsQy+Ltb9xr0jXQv3hKSsEmncNdC9Wa9VFEUsP7YcX576EgAgQEA3v244XXQaAJBanIood93D+ChfpXRcysUyDO/gY73FU5MSUgulcZ9wL7O/39MFp6VxJ69OVv/Mh7uFS+PUklTp/CO7+OPg5X4z20/lYv6waKtetynNfZ3tMjgzZMgQfPnll+jatSsuXryI119/HcOHD0dSUhLy8vIAAMHBwUbHBAcHIz1dFz3Ly8uDs7MzfH19G8zRH2/KG2+8gcWLFzfYHh8fDzc3N0tfFhFdBbZv3+7oJRBRG8H7ARHp8X5ARHq8HxA1JIrAxlQZdl/UFfgRIOKuThqUqktwGrqAzE879+Ni4NVX2uxqvifEJ8ugL9qUfy4Rv2YnNjpfFEXsqNmBXapd0rabXW+GrEaG09A94N9+cDsqj+uyIHSJObpH27sOn4J/0UmrvwYy76fz9b/f6sxT+PXXUybn/Vn1pzQuPFuIX1N+teo6LmouSuP9Z/YjIjMCACCvAvSfj417TiOoOMnE0bZTVVXVrHntMjgzZcoUady7d28MGzYMnTp1wtq1azF06FAAusZQhkRRbLDtSk3NeeGFF/Dkk09KP5eVlSEyMhJjxoyBv79/a14KEV0l1Go1tm/fjgkTJkChUDh6OUTkQLwfEJEe7wdEpMf7AZFpoiji/347i90XMwAAMgF459Y+uKlvKP48k4/NaUcBAL4RXTB1XGcHrtS6roV7wgfn9wCohEIuYN4tk6GQN95d49MTn2LXifrAzIuDX8RtXW5DYn4iftjxAwDAM8oTUwdNBQBcLKvB8qS/dZO9gjB16gCbvA4ybfkHut+vk0zAfbdOgovCdGbbxu0bgUu68V2T74KXc8v6DzWluq4ay79fDgDQeGkwdZLu8yGKIr5K+wc5pTW4UCHHqHHj4a60XyhEX3GrKe0yOHMld3d39O7dG+fOncP06dMB6LJjQkNDpTn5+flSNk1ISAhqa2tRXFxslD2Tn5+P4cOHm72OUqmEUqlssF2hUFy1N1IiahneD4hIj/cDItLj/YCI9Hg/IKoniiKW/noaa/fpAjOCALw7sy9mDNB9871TUP1D3OxS1VX5t3O13hNq1BqkXu7z0SXIE24uDZ+nGvrs+Gf49MSn0s8vXPcC7uh+BwCgg28HaXtOZY70foX7OcHNWY6qWg0yiqqvyvexrSqtVuP8Jd3vt0eYFzzdXEzOE0UR50rOAQBC3UPh72795AaFQoFgt2BcrLqIzIpMo8/B2O5B+Hp/BtQaEQfSSzGpZ4jVr9/Yupqj8ZBlO6FSqXD69GmEhoaiQ4cOCAkJMUoLrK2txa5du6TAy8CBA6FQKIzm5Obm4uTJk40GZ4iIiIiIiIiIiMgyoijird/P4vN/UqVtb93aRwrMAECkX30LgXQ2fG9XzudXQHu5Cl23UM9G5646uQrLE5dLPz8z6BnM6T5H+jnANQAuct3D/8zyTGm7IAiI9nfXbS+qQp1Ga63lUxOOG/QT6h/pY3ZeTmUOKtW6v92uvl1ttp5oL10/mRJVCUpVpdL2sd2CpHH8mXybXd8S7TI48/TTT2PXrl1ITU3FgQMHcNttt6GsrAzz5s2DIAh44oknsHTpUmzevBknT57E/Pnz4ebmhjlzdH/Y3t7euPfee/HUU0/hzz//RGJiIubOnYvevXtj/PjxDn51REREREREREREV69l25Pxya4U6ec3ZvTGrEGRRnNcFHIEe+kyLjKKmte/gdqGU7n1JZ26h5gvY7U2aS3eP/y+9POTA5/E3T3vNpojCAIiPHVBu+yKbGi0GmlfjL8ugFenFZFdUm2VtVPTEjNKpHH/KF+z884WnZXGtgzORHrW3zsMA3jDOgZA6aQLf8SfzYcotr2+Ve0yOJOVlYU77rgDsbGxmDFjBpydnbF//35ER+uiZM8++yyeeOIJPPTQQxg0aBCys7Oxbds2eHrWR2rff/99TJ8+HbNmzcL1118PNzc3/PTTT5DLTdfHIyIiIiIiIiIiIsv8b8c5LP/rvPTz69N74Y7rokzOjfbTZUYUVNSiQlVnl/WR5c7klkvj7qGmgzPrTq/DuwnvSj8/PuBx3NPrHpNz9cEZtVaN/Kr6DIiYAHdprC+jRraXmFEsjftH+Zidl1ycLI27+tk+cwYA0svSpbGrsxzDO+lKqV0sUyEpp3l9YOypXfacWb9+faP7BUFAXFwc4uLizM5xcXHB8uXLsXz5crNziIiIiIiIiIiIyDq+2p+O93fUP7BdfFNPzB0abXZ+lL8bDqYVAQAyCqvQI8y6zcTJNs7k1T8EN1XW7NcLv+LNg29KPz/c72Hc1/s+s+e7MjMi1EPXZ7yDf31wJq2gEoi1aNnUDKIo4mhmCQDAz90ZUQblB69kFJyxYeZMlFd9cDejLMNo39huQYg/ewmArrRZr3Bvm62jNdpl5gwRERERERERERG1H6IoYkV8fcbMf/7VHfOGxzR6TLTBg9+MImZGtAeiKOL05bJmAR5KBHgoG8xZdXKVNF7YZyEe7Ptgo+c0V7Yq2r/+85FWyNJ39pBeWIXiKjUAoF+kDwRBMDtXH5xRypWI8jSdHWcNhufOKDcOzowx6Dvz19m213eGwRkiIiIiIiIiIiKyqcyiauSU1gAAhnfyx30jOjZ5TJTBw/d0PnxvF/LLVdLD++4msmZKVaXSQ/tuft3wcL+HmzynueBMB4OyZmmFDN7ZQ2KmQUmzSB+z86rUVVIWS2efznCS2a6Al+Hn48rMmQhfN3QN9gAAHM0sQWGFymbraA0GZ4iIiIiIiIiIiMim9qcWSuNhHf2bdUy0Qdmq9CIGZ9oDfdYMYLrfzOGLhyFC15h9cMjgRjMv9MwFZwI9lXBz1vUPT2PPGbtIzCiRxv2jfM3OSylJkX7PtixpBgAuTi4IcQ8BAKSXpzfYr8+eEUVgV/Ilm66lpRicISIiIiIiIiIiIps6mFokja/r4NesYwzLmmUyONMunM4tl8bdQhpmzhy+eFgaDwwe2KxzhrmHQSboHmNnVWRJ2wVBkAJ4WcXVUGu0rVozNZ8+OCMIQJ9I8/1bzhaflca2Ds4A9aXNSlWlKFWVGu0bG2tQ2uxM2yptxuAMERERERERERER2dSBy5kzzk4y9G2kHJIhHzcFPF105ZBY1qx9OJPXeOZMwsUEaTwwqHnBGYVcgVD3UADGmTMA0CFAF8Cr04rILq5u8Xqp+aprNVJmVJcgD3i5KMzO1ZeuA4BYv1ibry3Ky6DvzBWlzQZG+8Lr8n3k7+RLqGtDQTwGZ4iIiIiIiIiIiMhmckqqkVmke3DeP9IHLgp5s44TBAFRl7NnskuYGdEenLmcOeMkE9Ap0MNoX3ltOc4UnQEAdPHtAh8Xn2afN8IjQjqHYWZEjEHpu1T2nbGpkzmlqNPqSpX1jzRf0gwwDs508eli03UBQLRntDS+srSZk1yGkV0DAQBlNXU4nF6MtsLi4ExVVRWqqsxHrpcvX44RI0age/fumDp1Kn7++WdLL0lERERERERERETtxAGDfjNDmtlvRi/aXxec0WhF5JQwM6ItU9VpkHKpAgDQOcgDzk7Gj54T8xOhFXUBtkHBg1p07gjPCGlsmD0TE2DQl4h9Z2wqMaM+qNE/ysfsPFEUkVykC84EuQW1KAjXWoaZM5llmQ32jzEobRZ/tu30nbEoOPPTTz/B09MTYWFhKC8vb7B/wYIFeOKJJ7B3716cPXsWf/zxB26++Wa8/fbbllyWiIiIiIiIiIiI2okDF+r7zQxtZr8ZvSg/g4fvLG3Wpp3Pr5AyK5oqadbS4EykZ6Q0NgrOGGTOpPHzYVP6fjMA0D/KfOZMXmUeytW6WEGsr+1LmgH1PWeAhpkzADA6NhCCoBvHt6G+MxYFZ/744w+Ioojp06fD09O4wdPu3buxZs0aAICbmxv69+8PFxcXiKKI//znP0hKSrLk0kRERERERERERNQOHEjVBWcUcqHRh7qm6DNnACC9iA/f2zJ9STMA6Bbi2WD/4bzD0nhgcPP6zeiZDc4E1H8+Upk5Y1P64IyH0gmdgzzMzjMsadbVt6utlwUAiPSKhABd9OXKnjMA4O+hRN8IHwDA2YvlyG4jWXgWBWf2798PQRAwZsyYBvs+++wzAEBYWBhOnz6Nw4cP48yZM4iMjIRGo8Gnn35qyaWJiIiIiIiIiIiojcsvq5EemveN8IGrc/P6zehF+9U/fM9gT5E2Td8sHgC6XZE5U6WuQlKh7sv6Hb07wt+1ZeXtzAVnAj2UcL/8mUrn58NmckurkVdWAwDoG+kNuUwwO/ds8VlpbK/gjFKuRIh7CAAgo7xhcAYAxnarL232VxvJnrEoOJOfr3sRXbo0bOrz+++/QxAEPProo4iI0NUEjIyMxKOPPgpRFLFr1y5LLk1ERERERERERERt3P7U+pJmQzq2rKQZAEQZZs6wbFWbdiavPnOme6hx5szR/KPQiBoALS9pBpgPzgiCgOjLpc0yi6uh1mhbfG5qmmFJs36RPo3ONcycifWzT1kzoL60WamqFKWq0gb7DYMzbaW0mUXBmUuXdM1zPDyM05hOnTqFgoICAMBNN91ktG/QIN0fX1pamiWXJiIiIiIiIiIiojbuwIVCaTykQ8uyJQAg1NsVCvnlckUsa9amncnTZc74uzsj0ENptM+o30xIy4MzHs4e8FXqSuJllWcZ7esQoAvOaLQisorbRrmqq83RzBJp3D+y8dKE+uCMQqZAtFe0LZdlJMrLoO9MWcO+Mz3DvBDkqftc7k0pQI1aY7e1mWNRcEYu16WMFRUVGW3/559/AACBgYHo1q2b0T5fX90vr6amxpJLExERERERERERURt38HLmjFwmYEB0y/rN6I+L9NVlz2QUVUEURauuj6wjv7wGBRW1AIBuoZ4QBOOyV4cvtr7fjJ4+eya/Kh8qjUrabth3Jo2lzWwiMaNYGveL8jE7r6auRgqMdPbpDCeZk62XJjEMBJkqbSYIAsbE6rJnatRa7EspbDDH3iwKzoSHhwMAjh49arT9l19+gSAIGDFiRINjSkt1KUUBAQGWXJqIiIiIiIiIiIjasMIKFc7lVwAAeoV7w0PZuge1kZf7zlTVaqQAALUtZ3INSpqFGPebqamrwYmCEwB0paeC3ILQGuGeumfRIkRkl2dL2/VlzQAgrYDBGWtTa7Q4nqV7ph/l54aAK7KiDKWUpEAr6krL2avfjJ5h6buMMtN9Z8a0sb4zFgVnRowYAVEU8eGHH0plzA4dOoTff/8dADBp0qQGx5w+fRoAEBISYsmliYiIiIiIiIiIqA07aNBvZmiHlveb0Ys26DuTUcSH723R6dwyadwt1Dg4c/zScai1agCtK2mmZ67vjL6sGcDgjC3sTSmEqk4XcOnfSNYMAJwtPiuN7R2cMcycMVXWDABu6BIglUn860y+wzPxLArOPPTQQ5DJZEhNTUXHjh0xaNAgjBo1CnV1dfD19cXtt9/e4Ji//voLgiCgX79+llyaiIiIiIiIiIiI2rADBsGZIR1bH5yJ8qsPzqQXsu9MW2TYk6R7qKfRPqN+M8HWD87EGGbO8PNhVRsPZ+H+Lw1+fzGN/x0fv3RcGsf6xdpsXaZEeEZAgC7wYvj5MOShdJJ6X2WXVEuZfY5iUXBmwIABeOeddyAIAioqKnDkyBHU1NRAoVDg888/h6en8R9iaWkpfvnlFwDAhAkTLLk0ERERERERERERtWH7L+h6OsiEph/qNsawbBWDM22PRiti3+XftZeLE7pdUdbM1sGZAA9nqWQee85Yh1qjRdyPSXh6wzHUXs6aGRzji5kDIxo97kDuAQCAQqZA74DeNl+nIaVciVD3UADmM2eAtlXazOKOPIsWLcL48eOxceNG5OXlITQ0FHfccQdiYxtGxnbu3InBgwcDAMaPH2/ppYmIiIiIiIiI6CojiiK0oq4RPLVfJVW1OHtR14ekR5gXvFwULTpe9znQQi6TX1HWjMGZtuZUThlKqnRly4Z3CjD6263V1ErZFOEe4Qj1CG31dcwFZwRBQLS/G5JyypBVXA21RguF3KKchGvapXIVHv7miFFZwrlDo/DKjT3h7GT+fc0sz0RWRRYAoF9QP7gp3MzOtZVIr0jkVOagrLYMJTUl8HHxaTBnbLcgLPn5FABdcObBUZ3svMp6FgdnAKB3797o3bvpSNjNN9+Mm2++2RqXJCIiIiIiIiKidkoURRRW1iKtoBKpBZVIK9T9b2pBFdILKyEAeGdmX0zt3foHueRYB1OLoG/noC8jZEpJTQnSy9ORUZaBtLI0ZJRlIL0sHRnlGdBoNXhhyAuYGlP/PDGdmRFtzu7zBdL4+i4BRvtOFJyASqMCAAwMHmjRdQJdA+Eid0GNpqZB2aqYAHck5ZRBoxWRWVSFjoEeFl3rWnUsswQPfn0YuaU1AABnuQxLpvfE7YOjmjxWnzUDAENDh9psjY2J9oyW1pFenm4yONMhwB0dAtyRWlCJw+nFKK1Sw9utZcFja7EoOLNgwQIAwJQpUzBz5kyrLIiIiIiIiIiIiK4uuaXV2JCQhfP5FVIgprymrtFjVuxMYXCmHTPqN9NBV9KssLoQm89vRkpJihSMKastM3cKAMBnxz/DjC4zEOylxMUyFTKKqm26bmq5PYbBmU7GgbiEPOuUNAN0GTIRnhE4X3Ie2RXZ0IpayARdJkeHK0rfMTjTchsSMvHSlpNSGbNgLyU+mTsQ/aN8m3X8/tz90nhI6BCbrLEpUV71QaSMsgz0Dexrct6Y2CCkFqRCoxXx97lLmNY3zF5LNGJRcGbt2rUAgNtvv90qiyEiIiIiIiIioqvPwq8O43hWaZPz5DIBMgFQa0Sczi1DjVoDF4XcDiskazuQqutBIgjAdZeDMy/ufhF7c/Y2eaxMkEEuyKHWqpFdkY3C6kJE+7njYpkKBRUqVKrq4K60SkEgslCNWoNDabpAXJi3CzoEuBvtN+o3E2JZcAaAFJxRa9XIr8pHiHsIABiVvkstqMQYi6907VBrtHj951NYu6++T8ugaF98PHcAgjxdmnUOrajFwdyDAAAPhQd6+ve0yVqbEuVpEJwpzzA7b2y3IKzakwoAiD+T3z6DM4GBgbh06RKCg4OttR4iIiIiIiIiIrqKnM+vMArMCAIQ7uMqlZaJ8b/8vwHuiPB1xUubT+D7hCzUaUWczC61qJE8OUZZjRqncnQZMbHBnvBxc0ZBdQH25eyT5ggQEOIegiivKER7Ruv+10v3v5Eekfgg8QOsSVoDQFcaK9LPFwcvBwEyiqrQPdSrwXXJ/o6kF0N1OdPi+s4BEIT6fjNqrRrHLh0DAAS7BSPCo/Fm8s1heI7M8kwpOGMYFEpj6btmu1SuwsPrjkh/W0Dz+stcKbk4GcWqYgDA4JDBcJI5Jnga7RUtjdPL0s3Ou66DH9yd5ais1WBn8iVotKJD+pxZ9C716NEDu3btQnp6Ovr162elJRERERERERER0dXij6Q8abxofFc8OLojlE7ms2H6Rfri+wRdU+mjmSUMzrRDh9OKoZX6zeh+f39l/AURuo1zu8/F4wMeh4uT+W/l9w6o7299/NJxRPtPkX5OL2Rwpq0w7DdzwxX9Zk4VnkJ1na4M3cDggUaBm9aK9IyUxpnlmRgcMhiAcXDmTF65xde5FhzNLMGDXx1GXlnL+8tcaX9OfUkzR/WbAXSZVTJBBq2oxbnic2bnOTvJcEOXAPyRdBFFlbU4llWCAc0s32ZNzQ9/mTB37lyIoiiVNyMiIiIiIiIiIjJkGJyZMSC80cAMAPSP8pHGiRklNloV2dL+yyXNAGBIR10Pkj8z/pS23dTppkYDMwDQJ7CPND5ecNyobFVGETMj2grDfjPDGus3Y4WSZkDD4Iyev4cSEb6uAHRBhxq1xirXu1oVVKhw18oDUmAm2EuJ7xYObVVgBjDuN+PI4Iyz3BmxvrEAgPMl51FSU2J27thuQdJ455l8Wy/NJIuCM/fccw/GjRuHrVu3YvHixRBF0VrrIiIiIiIiIiKidi67pFoqadYr3AuRfm5NHAF0DfaEm7MugJOYUWzT9ZFtHLhQXyLpug5+KFWVSv0owj3C0c2vW5PnCHEPQZCb7uHpyYKTiPBVSvvSC6usvGJqjdIqNU5k6/6+Y4M9G/QnMeo3E2zb4AwADOmgCw7V1mlxLLPEKte7Wv16IhflNXUAgAFRPvjp0RvQv5WZI7WaWhzJPwIACHINQgfvDlZbZ2sYBgIP5x82O29El0BpnOigz4tFZc3++ecfPP3007h06RJee+01rF+/Hrfffjv69OkDX19fyOWNfxNi5MiRllyeiIiIiIiIiIjasD9O1mfNTO4Z0qxj5DIBfSN8sO9CIXJKa5BXWoMQ7+Y1pSbHq1TVSQ/sOwd5IMBDiZ9StqFO1D0IHhc1rtnlrfoG9sX29O2oVFdCVNR/sz2jiMGZtmDfhUKpfN31nY1LmtVp65CYnwgA8HfxR4xXjFWuGe4RLpWtahCc6eiHTUd0JREPpBZJWVvU0K8ncqXx0hm9GwTWWuLYpWNS+bqhYUOtUr7OEoOCB+GrU18B0GVvjYsaZ3JeqLcLAjycUVBRi5PZpRBF0e5rtyg4M3r0aKMFJycnY8mSJc06VhAE1NXVWXJ5IiIiIiIiIiJqw343KGk2uVfzgjOArrTZvgu60lhHM4sx2TvU6msj2zicXgzN5Sf2+n4zO9J3SPvHR49v9rn6BPTB9vTtAIC0ilPwVHqiXFXHzJk2Yo9RvxnjQMjZorOoVOvKzw0KGWS1h94KuQIhbiHIqcxpEJwZ2qF+DQdTi648lC67VK6S3p+OAe6IDfa06HxtpaSZ3sDggRAgQISIwxfNZ84IgoAeYd74O/kSiqvUyC2tQZiPqx1XamFZMwAQRbHV/4iIiIiIiIiI6OpUUKHCobTLDwAD3dE5qPkPAA3L67DvTPty4Ip+M1XqKuzJ2QNAl0HRN7Bvs89l2HfmRMEJRF3uO5NdUo06jdZKK6bW2pOiC844yQRc1+GKfjM2KGmmpy9tVl5bjlJVaf12P1eEXs6yO5xeDDU/IyZtO5UnZTxN6R1iceDsQO4BaTwkdIhF57IGb6U3uvh2AQCcLT6L8tpys3N7hXlJ45PZpWbn2YpFmTPx8fHWWgcREREREREREV1Fdpy6CP13c5tb0kyvX6SPNGZwpn0x7DcztIMf9ubshkqjAgCMjRoLmdD874p39+8OJ8EJdWIdjl86jmj/m5CUUwaNVkROSY0UrCH7yympxoVLusyYfpE+8FAaP2ZOyLNdcCbCMwIH8nQBgazyLHgrvQHoMiGGdPDDlqM5qFZrcDyrFAOjW9dH5Wr224n6jMYpvSzLSiyvLcfJgpMAgI7eHaU+UY42KHgQkouToRW1SMxPxMgI0+1VeoV7S+OknDJMbOF/qyxlUXBm1KhR1loHERERERERERFdRVpb0gwAAj2ViPRzRWZRNY5nl0Ct0UIht7gADNlYda0Gx7JKAAAdAtwR5OWCHccMSppFNb+kGQC4Ormiq19XnCo8hZSSFPQzaGuSXlTJ4IwDGZY0u7LfjEarkRqx+yp90cmnk1WvHeEZIY0zyzPRM6Cn9POQjv7YcjQHgC6Li8EZY8WVtVLJyCg/N/Q0yBxpjYS8BGhEDYC2UdJMb1DIIHxz5hsAujWaC84Yvv6kHPtnzvC/akREREREREREZFVlNWrp4W2Ytwt6G3w7ubn6R+oeqtaotTibZ74sDbUdiRnFUGvq+82oNWrsytwFAPB09sTgkMEtPmefAF1pMxEinFyzpO3sO+NYxv1mjIMz50rOSaWkBgYPtHqTdX1ZMwAN+s7o+xwBxllcpLP91EWpJ9SUXpaXNGtr/Wb0BgYPlMaGJfauFOXnBk8XXf7Kyewym6/rSgzOEBERERERERGRVcWfyZce0k9q5QPA/lE+0jgxo9haSyMb2m/QhH1IRz8cyDuACnUFAGB0xGgo5IoWn9Ow70yVcEEaZxQxOOMooihiT4ou+8LNWY6+ET5G+41KmoVYt6QZ0HhwpkOAOwI9lbp1pBWxN9EVfj2ZK42n9LaspBlQ329GJshs8rtuLT8XP3Ty1mVsnSo8hUp1pcl5giCgR6gueyavrAYFFSq7rRGwYnCmrKwMq1atwv33349p06Zh3LhxSE9PN5qTk5ODU6dO4cKFC2bOQkRERERERERE7d3vJw1KmrWyhn//qPpyROw70z4cTC2Uxtd18MeO9PqSZuOix7XqnH0D+0rj3Jqz0ji90PTDVrK9c/kVuFSue4g9pIMfnJ2MHzEfvnhYGhtmMFhLY8EZfd8ZAKis1SApx/7ZEG1VaZVxRmPfiJZnNBrKr8pHSmkKAKBXQC94OntavEZr0geLNKIGR/OPmp13Zd8Ze7JKcOajjz5CVFQU7r//fqxatQq//PILdu7cicpK45vkrl270KtXL/Tq1QtFRUwrIyIiIiIiIiK62lTXarDz7CUAgL+7MwbF+DVxhGndQz3hfLnPzNHMEmstj2xEVaeRgmgRvq4I8XJGfGY8AF3vmOFhw1t13kjPSPgofQAA50tPQSHXbWdZM8fZfc58vxlRFKXgjKezJ7r4dLH69T2dPaXPxJXBGeCK0mYGAcNr3Y7TF6WMxim9Qy0uaabPmgHaVkkzveaWNusV7ri+MxYHZ+Li4vDYY4+hrKwMzs7OGDjQfDT09ttvR2hoKFQqFTZt2mTppYmIiIiIiIiIqI35+9wlVKt1DaIn9gyGXNa6B4BKJzl6Xn5odqGgEsWVtVZbI1nfscxSqOp0JaSGdPBHYn4iimp0X86+IfwGuDq5tuq8giCgd0BvAECxqhih/rqgTEZRFURRtMLKqaUa6zeTUpKCYpWuDOHAoIGQy+Q2WYM+eya/Kh8qjXEpqiEd/aUx+87U+82gpNnU3q3LaDTUVvvN6A0Kri+zZlhq70o9wwwyZ+zcd8ai4ExiYiKWLFkCAJg7dy7y8vJw8OBB8xeTyTBz5kyIoojt27dbcmkiIiIiIiIiImqD/jAoaTaplSXN9PpH1pc2O5pVYtG5yLYOXKjPUBjS0Q9/Zvwp/TwuqnUlzfQM+854++oeMFfValDIgJ3dqTVaHLjcWyjAwxmxwcalrAwzFGzZgyTCMwIAIEJEdkW20b4uQR7wc3cGABxMK4JGyyBeeY0afyfrgmrBXkqje2triKIoBWdc5C5G5QfbikC3QER7RQMAThaeRHVdtcl5HQPc4aLQhUlOtqfMmeXLl0MURQwbNgxffvklvL2brlM3bNgwAMCJEycsuTQREREREREREbUxtXVa7Dh9EQDgqXTC8E4BTRzRuP5RPtKYfWfaNv0DewAYEuOHHRm6fjNOMieMjBhp0bkNgzMyl/oe1yxtZn/Hs0pQoaoDAAzvFNCgNJZRcCbYdsEZw74zWeVZRvsEQcB1l8spltfU4Uwe+878dSYftRpdZtvkniGQtTKjUS+1LBX5VfkAdOXDnOXOFq/RFvSfwTptHY5fOm5yjpNchm4huizN9MIqlNWo7bY+i4Izu3btgiAIeOSRR5p9TExMDAAgOzu78YlERERERERERNSu7L9QiLIa3YPbcd2DGjQKbynj4EyxReci21FrtDicrvv9hHq7oFxMRV6lLoNqaOhQixuF9w7oDQG6h8kVwgVpe0ZRpblDyEZ2n6vPkLrBRL8Zffkod4U7Yv1ibbaOCI8IaWyy70xHg74zLG2G307UZzRO6R1q8fn257TtkmZ6rek7cyrHfsE8i/4LmZurSyOMjW3+H5pSqQQAqFSqJmYSEREREREREVF78nuS9UqaAUC4jysCPXXPko5mlkDL8kRt0vGsUqnP0JAOfvgz03olzQBdA/iO3h0BAAW1qYCg+2Y7M2fsz7DfzPVX9JtJK0tDYY0ueNM/qD+cZE42W4dh5ozJ4EwHg74zqYUN9l9LKlV1iD+ry3IJ8HDG4Bi/Jo5omlG/mbC2G5wZHDJYGjfWd6aXYd+Z9hKccXbWpSup1c1P9dEHdHx8fCy5NBERERERERERtSEarYhtSbqSZkonGUbFBlp8TkEQ0D/SB4CuPNGFggqLz0nWZ/jw+7oOftiRritpJkDAmMgxVrmGvrSZVtRA7qKryJPB4IxdVarqkJipy5CK8XdDuI+r0X57lTQDmg7OdAvxhLerAgBwMLXomg7s7jx7Cao6XUmzST1DILewpFmdtk4KdPgqfdHVt6vFa7SVEPcQhHuEAwCOXzoOlcZ0wkhPw+BMtv36zlgUnImI0KWPJSUlNfuYbdu2AQA6d+5syaWJiIiIiIiIiKgNOZJRjIIK3YOvUV0D4eZsnW/N94+qb1x9hH1n2iTDslFhgWVIK0sDAAwIHgB/V38zR7WMUd8Z1wwAQHoRgzP2dDCtCGqNLshxfeeG/aQMMxMGhdg2OBPoFgilXJdVZyo4I5MJUoZIcZUa5/Kv3cDurydzpfFUK5Q0O1V4CuXqcgDAdaHXQSZYVr7S1vSBwlptLU5cOmFyTtcQDzhdDlqdzGknwZmxY8dCFEWsXr26WfMvXLiAlStXQhAETJgwwZJLExERERERERFRG/LHyfqSZpN7WV7STM+470yJ1c5L1lFn0G8mwEOJs+V7pX3jo8Zb7TqGwRk3T13mDMua2deec/UlzUz2m7mcOePq5Ioe/j1suhaZIJP6zmSXZ0MrahvMGWrYd+YaLW1WXatB/BldSTNfNwWGdLBySbM23G9GzzBQaK7vjNJJji7But5Y5/MrUF2rscvaLArOPPLII3BycsKePXsQFxfX6NyEhARMnDgRFRUVUCqVWLhwoSWXJiIiIiIiIiKiNkIURanfjJNMwLhuwVY7d58Ib+ir8CRmFFvtvGQdp3LLUKGqA6Brwv5nhnX7zeh18u4ENyc3AIDMRZc5U1ChQuXla5Pt7UnRBTgEARjWyTgjKqsiC/lVuiBA38C+UMgUNl+PvrRZrbZWurYho74zBtld15JdyZdQdTnQMKlnCJzklme5HMg9II3bRXAmuOngDAD0CvMCAGhF4EyeffrOWPTb6Nq1K15++WWIooglS5ZgyJAhePvtt6X9v//+O9566y2MGzcOQ4YMQWpqKgRBwJtvvonQUMtTqIiIiIiIiIiIyPGScsqQVVwNQPfQ1tvNeg9m3Zyd0C1E99As+WI5H8a3MYYPvbtHqHG66DQAoKd/T4R6WO/5n1wmR++A3gCAOlkxBCdd6aHMYmbP2ENBhQqnc3UPrHuFecPHzdlov1FJMxv3m9GL8IyQxqZKm/UI84KnUlde8UBqIUTx2us785tBSTNrZDRW11UjMT8RABDuEW70O2irwj3CEeKue+3H8o9BrVGbnNcr3KDvTE47CM4AwMsvv4z//Oc/EAQBhw4dwgsvvABB0H2d4ZlnnsGLL76InTt3Sh/+V155BY899pillyUiIiIiIiIiojbijyTblDTT05c204rA8Sz79QOgpiWk1wdnVMrj0nh8tPVKmukZljaTu+oexrO0mX3sTakvC2aq38zRS0elsa37zejpM2cAIKs8q8F+uUzAoBhdz6qCilqkXKq0y7raClWdBn+e1mUUebk4YXinhr+3lkq8mAi1VhfcaA9ZMwAgCIIUMKzR1CCpMMnkvJ6XM2cAIMlOfWes0q3ntddew/79+zFjxgy4urpCFEWjfwqFAlOmTME///yDV1991RqXJCIiIiIiIiKiNuL3y/1mBAGY0MN6Jc30+kX6SOPETJY2a0tOZuu+Ye7uLMexon+k7dYsaaanz5wBALmrrrRZBoMzdtFYvxkAOF2oy5iSCTKb95vRaypzBgCGdKwvbXYw9doqbbb7XIFUcnBCjxA4O1keCjDqNxPWPoIzQPNKm3UP9cLlnBPpvtYauaXVeOSbw82a69Tqq1xh0KBB2LhxI+rq6nDq1Cnk5+dDo9HA398fPXv2hKurq7UuRUREREREREREbcT5/Aqcy68AAAyK9kWQp4vVr9E/ylcaJ2aUWP381DqlVWpkl+jK2XUJFXE0/ygAXX+YDt4drH693oH1wRnZ5eBMetG1lQ3hCKIoYvd5XXDG2UkmZaPoqTVqnC85DwCI8YqBq5N9ngMbZs6YC85c18FPGh9ILcScIVE2X1db8euJ+ozGqb2tk9FoGJwZEjLEKue0h4HBA6VxwsUE3Nf7vgZz3JVO6BDgjguXKnE2rxxqjRaKVvTo+f1kHnaeLWh6IqwYnJFO6OSEPn36ND2RiIiIiIiIiIjaPcOSZpN6Wr+kGQB0DHCHl4sTymrqkJhRAlEUpbL65DhJufWlfzz9z0Ks1rU1GBdt/awZAAhwDUC4RziyK7Ihd8kGoGFZMzvIKKqSgnCDon3hopAb7b9QekEqddXNr5vd1hXuEQ4BAkSIZoMzvcO94eYsR1WtBgcuFF0z947aOi22n9Ldmz2UTrihi+UlzYprinGm6AwA3e/Z18W3iSPajmivaAS4BqCgugCJFxNRp62Dk6xhaKRXmDcuXKpErUaLcxcr0MOg1Flz/WYQFGuKVcqaERERERERERHRtckewRmZTEC/KH3vCBWyiqttch1qmVMGTbPLZInSeHyU9fvN6On7zggyNWQuecgoYnDG1vRZM4DpfjOni05LY3sGZ5zlzlKjd3PBGYVchoHRuntHXlnNNfN52ZtSgLIaXUmz8d2DoHSSN3FE0w7mHYQIXQC2vfSb0TPsO1NVVyUFma7UK9yyvjP55TU4lN788nkMzhARERERERERUatkl1TjeJbuAVbPMC9E+rnZ7Fr9jfrOlNjsOtR8SfrgjKwK6VXHAABh7mE2fUDfN7CvNJa7ZiC7uBp1Gq3NrkfAnvON95sxfNBtz+AMUF/arKy2DKUq0w/ThxiWNrtwbfSdMczemNwr1CrnNOo3086CM8AVfWfyTPed6RnmLY2Tclred+aPpIsQxebPt6is2YIFC1p8jCAIcHFxgbe3N7p06YKhQ4eie/fuliyDiIiIiIiIiIgc4I+TBg8AbZQ1o9c/ykcaJ2YU46a+YTa9HjVN/81ypddZaEQNAF1JM1uWjeoTUN9OQe6SgZriYcgpqUGUv+0Cg9cyrVbE3pRCAICXixN6hXs3mHO6sD5zpruffZ/zRnpG4mDeQQBAVnkWvJUN1zeko7803p9aiFmDIxvMuZrUabTYdrmkmZuzHKNjA61y3gO5BwAACpkC/YP6W+Wc9jQoxCA4czEB83vNbzCnp0EZs5PZLc+c+e1EbovmWxScWbNmjVVutoMGDcKyZctw/fXXW3wuIiIiIiIiIiKyj9+TDL+dbdvgTD/DzJmMEptei5pWo9Yg5VIlAMDL/wxqLm+3ZUkzQJeZ4SxzRq22FnJXXSmrjKIqBmds5FRuGUqqdP1khnXyh1xm/CxYK2pxtvgsACDEPQQ+Lj52XV+EZ4Q0zqzIRM+Ang3m9InwhtJJBlWd9prInDmQWoTiy7+zMd2CGvQIao2s8iypdFzfwL5wU7S/v7eO3h3h5+KHopoiHLl4BBqtBnKZ8Xvj4+aMcB9XZJdU41RuGbRaETJZ8+IfhRUq7L+gC2RG+rnCdKE9YxaVNYuKikJUVBQCAgIgiqL0z9nZGcHBwQgODoazs7O0HQACAgIQEREBLy8vafuhQ4cwatQorFu3zpLlEBERERERERGRnRRUqHAoTfegs2OgOzoHedj0ej5uzugY6A5A1+tEVaex6fWocWfyyqHRioBQi1rnUwAAfxd/o7JjtqCQK9DdX5edIVMWAPJKpBdV2vSa17LdTZQ0yyrPQqVa9/7bu6QZUF/WTL8WU5ROcgy43LMqu6QaWcVXd9+ZXw2yN6ZaqaSZPmsGaJ8lzQBdRa+BwQMBAOXqciQXJ5ucp+87U1WrQWph8+8t205dhPZySbOJPZr3ZQWLgjNpaWnYvHkzPD094ezsjEWLFiExMRGVlZXIyclBTk4OKisrkZiYiCeeeAIKhQIeHh7YvHkziouLkZmZibfeeguenp7QarW47777kJnZnJgSERERERERERE50o5T9bX1J/cMsWkpK73+kboHrLUabav6AZD16EuaOXkkQwvdt/THRo1t8E10W+gTaFjaLBMZhVf3w3ZHMuw3c72J4MzpIseVNAOuyJwpN/9ceUjHa6PvjEYr4o/LGY0uCpnVS5oBwNCw9hmcASAFZwBdaTNTerWy74xhUGxij+BmHWNRcObixYuYOnUq8vLyEB8fj/feew99+/aFTFZ/WplMhr59+2LZsmWIj49HXl4epk6ditzcXISHh+OZZ57Bzp074erqitraWnz44YeWLImIiIiIiIiIiOxgx+mL0tjWJc30jPvOlNjlmmSa/qGlk8cpaZutS5rpGQVnXDOQzuCMTdSoNVJ2XJi3CzoEuDeYc6bojDR2dOZMo8GZDvV9Zw6mXr3BmUNpRSioqAUAjO4aBHelRV1NAOhK1x3I0wVnPBQe6OnfsHRcezEouL7vzOGLh03O6Rle33cmqZl9Z4ora6XeTBG+ruhh0LumMRYFZ9577z3k5eXhySefxLBhw5qcP2zYMDz55JPIz8/HO++8I23v378/FixYAFEUsX37dkuWRERERERERERENiaKIo5mlgAAfNwU6G2iSbgtGAZn9Ncnx9AHZ2SX+744yZyMGm7bUt+A+tJpctdMpBcxOGMLR9KLUaPWAgCGdw4wmR3n6MwZL2cveCt195/GgjP9o3zgLNc9Cj+QWmiXtTmCYUP6Kb2tEzRPLk5GUY0uoDUoZBCcZJYHfByli28X6fNy+OJhaEVtgzmGmTMnc5oXnNl++qKuzCOAKb2an0lqUXBm69atEAQBkyZNavYxkydPBgD88ssvRtunTJkCQFcqjYiIiIiIiIiI2q6c0hrp29m9w73tUtIMAGKDPeF6ubl1YkaxXa5JDdVptDiTWwbIaiB31pW9ivWNhbPc2S7XD3EPQYCrrsSW3DUTGYXlUr9rsg5RFPHxzhTp5xFdGpY0A4AzhbrMGW+lN0Lc7ZNBd6VID132zMXKi6jV1Jqc46KQo1+kDwAgrbAKF8tq7LU8u1FrtPjtpK6kmbNchrHdgqxy3i9OfCGNh4U2naDRlskEGQYEDQAAlKhKkFKS0mBOkJcLAjyUAHRB6ObcW36//L4DwJTeze/zY1FwJitL12RJqVQ2+xj9XP2xemFhYQCAqipGuomIiIiIiIiI2rITWSXSuG+Ej92u6ySXoXeE7lvNWcXVyC+/+h6wtgcXCiqhqtNC7pINCLoHl70Cetnt+oIgoE+ArrSZIK9BNS6isNL0Q3lqnT9P52P35X4zEb6umNSzYeDlUtUlFNboslC6+XWzW5D2SvrSZiJEZFVkmZ1n2Hdm/wXHZs+cz6/Ay1tOYvupi01PbqYtidnIL1cBAEbFBsLTRWHxOY9cPII/0v4AAPj9P3v3HR1HeTVw+LdNvffuXiX33m3cDbbBBFNCAgkJEBIIgS8khCSQQgk91ACBhEAoAWOKsY1773KRbMu2bEm2erF6l3bn+2Ok2V2ra9V9n3N8zuzuOzOvpN2R/N6597r4sXzQcoeP2d1sS5s12XemrrRZYXkN6YUVzR6vuLKG3Ym5AIR6uzC2Db8THQrOuLm5AXDkSONfRGMOHz5st2+9qir1jePr6+vIlIQQQgghhBBCCCFEJzuRZi31Uh8s6Sp2pc2k70y3OFVX6kfvYl0I7+o+FLZ9Z/TSd6ZDVddaeHK9tVzZ75aNwKUuY81Wd5c0qxfhGaFtp5U0HZyZPMAanDnYjX1n9p3P44bX9/LBgYvc+2EsSbmlDh+z1myxy3S6Z/ZAh49pUSz87fDftMe/GPcLPJ08HT5ud5sQMkHbPpLVRHDGprRZfQnHpmxNyKbGrAapl8SEoNe3PkjpUHBmwoQJKIrC008/zeXLLUcb8/LyeOaZZ9DpdEycaF+D8uzZswAEBXVMupUQQgghhBBCCCGE6Bxx3ZQ5AzAu0npj7zHpO9MtTqWri5UGV5vgTED3BWcMrpdIlb4zHeb9fSkk55UBakBjaUzj5crO5J/Rtof7De+SuTWmPnMGmu87M6GfL8a6hfOD3ZQ5syY2jTv+dYiSqloAzBaFFzefc/i46+IytZ/Z9EH+TOzv18IeLfvq/FecvnwagKG+Q1k1eJXDx+wJhvsOx8PkAaiZM42VLYsO89K2T6U333dmfby1pNmyNpQ0AweDM/fddx+gliibOnUq3377baNfjKIorFu3jmnTppGaqn5Afv7zn9uN2bhxY6NBGyGEEEIIIYQQQvReF3JLeWdXEil1i0ai91MUhbi6zJkgT2dCvF1a3CetJI33T73P+YLzDp/fNnNG+s50j9OZdcEZl3QAXI2uDPR2/E79toj2j0ZXt7RpkMyZDpNXWsUrWxMB0Ongj9eNbLJcWU8MzjSXOePmZNQy/S7klpFbVwKsKyiKwt+3JPLwZye0LIt66+IyOdlCAKA5ZovCq9sStccPzB/S7mPVK6sp45Vjr2iPfzPpNxj0DbOneiOD3sC4oHEA5Ffmk1yc3GBMTLg1c+ZkM5kzpVW17DynljQL8nRmQlTbqoI5FJxZsWIFd999N4qikJSUxIoVKwgODmbRokXcfvvt3H777SxatIjg4GBWrlxJUlISAPfccw/XXXeddpysrCy+/PJLFEVh6dKljkxJCCGEEEIIIYQQPUSt2cIP3z3Ek+sTWPjSTp7ekEBp3d3CovdKuVxOSaX6cxzdipJmiqLwy+2/5Pkjz3PjNzfy5IEnKapq/0JksJcL4T6uAMSlFVFrtrT7WKLtFEXhVEYxOkMZeie1NNRwv+EY9cYunYebyY1+noMA0Dtnc+FyXpeev696YdM5Latj9YRIu0XqK9UHZ1wMLvT36t8V02uUbVmz5jJnAKYM8Ne2D3VRabMas4VHPo/jpS3WDJnbp0bx+2utpeCe++5su4+/Pj6TC7nWTKepA/1b2KNl78S9Q16F+pmaHzWfyaGTHT5mTzIxxKbvTCOlzSJ8XfFyUa9p9WUcG7PtTA7VtervoLaWNAMHgzMA//jHP3jyySdxdnZGURTy8vLYunUrH3/8MR9//DFbt24lLy8PRVFwcnLiqaee4o033rA7hpeXFwkJCSQnJ3P99dc7OiUhhBBCCCGEEEL0APuTLmuNdGvMCm/tTGLe8zv4PDYNi6Vh5Q3RO9iWNBvdipJmp/NPc65AXZS0KBY+OfsJ1669lk/OfEKtpX3BurF12TPl1WbOZTver0G0XnphBUUVNejrsmag6/vN1BsfPAYAnU7hfOGZFkaLlpzOKObTw5cA8HA28n+LhzU5tqS6RAuEDPUd2q1ZFUFuQTjpnYBWBGcG2vad6fzSZiWVNfz434f5LNaa0fPo0uH8ZWUMP5jWTws07zyXy4F2lFqzXJk1c43jWTNpJWn85/R/ADDpTTw88WGHj9nTTAy2Cc5kNwzO6HQ6ouv6zmQXVzWZZbUhPlPbXtJE+b/mOBycAXj00UdJSkri6aefZsGCBQQHB+Pk5ISTkxPBwcHMnz+fp556iqSkJH7729822N/NzY1+/frRr18/jMaujbILIYQQQgghhBCic3x5LKPBc7klVfzfZye44c19UpKql6ovaQZoJYKa823Stw2eK6oq4smDT7J63WoOZx1u8xzGRfpo28dS5X3UleqbYxtcrIvgXd1vpt744LHadmZV+zMPhJoR9Zd1p6mPm//imsEEejo3Of5svvX73Z0lzQD0Or2WPZNWkoZFaTqbbmI/X+qTGzo7cyazqIKb/rGf3YlqBoqTUc9rt43jnjmD0Ol0OBsN/GrhUG38sxvPNNoypDnfncrSAtTjo3yYMdjxrJkXY1+kxlIDwA9G/sCubFxfMcJ/BK5GNTAWmx3b6Pc9Jtym70wj2TPl1bXsOKuWNPN3d2JyO/r8dEhwBiAkJITf/OY3bNq0iYyMDCoqKqioqCAjI4PNmzfz29/+ltDQtjXEEUIIIYQQQgghRO9UWWPmu1Nqk1xPZyObfzWbRSODtddPpBZywxv7eOh/x8kpruyuaYp2iLcJzoxupuQRgNliZmPyRgCMeiNrVqzh2oHXaq8nFiTy4+9+zEM7HiKjtGEwrynjbOr6H7tU2Or9hOPqgzN6V2vmTIx/TLfMZXTgaG27Qp9EebWUTWyv705ls78ucyPKz40fzejf7Hi7fjP+3RucAWvfmWpLNTnlOU2O83QxaaXazmSVUFBW3SnzOZ1RzA2v7+NMVgkAPm4m/vuTKVw3Osxu3A3jwhkSpDanP3qpkK0JTc/9Soqi8Mo2ax+vB+YPabI/UGsdzjrM5oubAfB38eeno37q0PF6KpPepPWdySnPabRXUX3mDFive7Z2ns2losYMwKLoEIyGtodaOiw4I4QQQgghhBBCCFFv25kcrb/MkpgQhgR78vYPJ/LhXVMYGuyhjfviaDrznt/BmzsuUFVr7q7pilYyWxRO1t1BHO7jir9H03fWg1ouJrdCvbN4VvgshvoO5ZlZz/DB0g8Y6T9SG7f54mZWfLmC1469RnlNy43do8O8MBnURUjJwOpap+t+/gYXdTHTw+RBlFdUt8yln1c/jLir83FN5eLlsm6ZR0/R1qyLelW1Zp5an6A9/t2yETgbmy9TlpBvHT/Cb0QzI7uGbXZHy31nrBkOh1I6Pntm17lcVr+1n6y6Gw+i/Nz44mfTmdRIZoVBr7MrH/fcd2cxt7Ls55aEHBIy1aDBmAhv5gwNBNr/PjBbzPzt0N+0xw+MfwAPJ49m9ujdWiptZps5czK9YebM+pNZ2vayUW0vaQYSnBFCCCGEEEIIIUQn+PKY9a76lWPDte2ZQwJY/8Asnlg+Umu2W1Zt5m8bz7DopV1sPp3d5XMVrXc+p5TyajWINiaybSXNlg1cpm2PDRrLx9d+zJ+n/xk/F3XBsspcxVtxb7HiyxVsSN7Q7AKji8nAyLq7mi/kllFUUdOur0e03amMYnTGYvQmdVE42j8ava57lhj1Oj0hzmpZKL2xlOOZyd0yj+6mKAoPfXqcSU9u5Z1dSW3u6fXenhQu5atB0WkD/VkcHdzCHtbMGYPOwBBfx/ucOKq+rBnQaBaErSkDrKW/9p7P69B5/O9wKj/692Ht5oSxkT58cd90BgY2HeRYNDKYsXWlGs9ml/D1ifQmx9ZTFIVXttr0mqnLmnn64NPM+XQObx5/s809vdaeX8vZArVc3Qi/EawctLJN+/c2E0OswZn9mfsbvD4gwANXkxqkvDJzprLGzLYE9e8VHzcTUwda31Ol1aV8du6zVs2hw6+cxcXFpKenc+nSpRb/CSGEEEIIIYQQou8pKq/R6rAHejozbZB9DXyjQc+dMwaw49fz+P6UKK3+/8XL5fz0P0dYE9v8wproPnFphdr26AifZsdWmavYcnELAG5GN+ZEzLF7Xa/Tc8OQG1h3wzruGHkHRp0arMsuz+aRXY/wYcKHzR7ftu/MidTCJseJjpNfVk1mUSV6F+tntLv6zdQb4mM9/9Hs4903kW6UmFPKF8fSySut4sn1Cdz578NNNjC/Uk5JJa9vV0tj6XXwx+UjWyyNVW2uJqkwCYAB3gNwNjSfQdcV2pI5M3WQP051Jai2JuS0O9PkShtPZvHImjgt82VxdDAf/3QqAS1kGOp0Oh5ZYs2eeXHzOaprm+6bA7DjbC7xddkc0WFeXDM8iJzyHD468xEFVQW8ceIN7vruLjJLM5s9Tr2S6hJePfaq9viRSY9g0DefPdXbxQTE4OnkCcCe9D1an516Br2OEaHq65fyy+1uAth1LpeyuhsVFo0MxmRT0mx3+m5eOPJCq+bQIcGZzZs3c8MNNxAQEICvry9RUVEMGDCg2X8DBw7siFPz9NNPo9PpePDBB7XnFEXhiSeeICwsDFdXV+bOncupU6fs9quqquL+++8nICAAd3d3VqxYQVqa/PEnhBBCCCGEEEI4auOpTKrN6sLS8tFhGPSNL/T5uTvx5A2jWHf/LLsyM//el9IV0xTtENeGfjN70vZQUqP2W1jQb4HWfPlKnk6e/N+k/+OLlV8wM3ym9vx/E/7b7KLpuCgfbfuolDbrEvVNsQ2uNsEZ/+4NzowPHqNtny082Y0z6T5XlvbbdS6XpX/fxc5zuS3u+8J357Qsj5snRTEi1KuFPSCxMJFaRd2nJ5Q0A/vgzMXii82O9XA2ajcNpBdWcDqzYT+R9vjwgPW8P5rRnze+PwFXp9YFOKYPCmDWkAAAUvMr+ORw04kNiqLwd5usmfuvUbNm4nPj7cYdzTnKjd/cqPWQac47ce+QX6mWeFvYb6FdVklfZdKbmBU+C1CDU0ezjzYYE2Pze+60TfbMBpuSZktHhdrts/XS1lbPweHgzAMPPMCSJUv4+uuvyc/PR1GUVv9z1OHDh3n77bcZPXq03fPPPvssL774Iq+99hqHDx8mJCSEhQsXUlJSoo158MEHWbt2LZ988gl79uyhtLSU6667DrNZ6tsKIYQQQgghhBCO+PKYtbH7yrFhzYxUjQzz4pO7p2qLgiczirhc2rq7vkXXirOpux8T0Xxw5ttkm5JmA5Y1M1I1wHsAby54k0khkwBIL01v9g748VG+2vaRFAnOdIX60j4Gm8yZmICY7poOAAsHTUJR1ADwpbJTHZYF0Zsct8kcqy/DlFdazR3vHeKp9QlNZmGcTC/if7HqZ8zT2cjDi4a26nxnLp/Rtof7DW/nrDtWhGcEbkY3AA5lHcJsaX6Nd8FIa+m2LadzHD5/YXk1+5MuAxDp58ofrxvZ5I0JTXlksfV7+crW85RXN16WbHdinvYzHx7iyaK6ryUuL04bUx8ML6ku4aEdD/Hn/X+morai0eNdKr7EBwkfAOCkd+LhiQ+3ad692bzIedr2jtQdDV6PDrMGK+uD01W1ZrbUlWD1dDEyY1CANqbKXMWutF2tPr9DwZmPPvqI1157DUVRcHZ25pZbbuG5557j3Xff5V//+lez/9577z1HTk1paSnf//73eeedd/D1tf4yVhSFl19+mccee4xVq1YRExPD+++/T3l5OR999BEARUVFvPvuu7zwwgssWLCAcePG8eGHHxIfH8+WLVscmpcQQgghhBBCCHE1yyqq5ECyukDV39+N0S0s4NfT6XTMHqoucCgK7OngPgDCcdW1FhLqFucHBrjj5WJqcmxJdQk7U3cC4Ofix5TQKa0+z4ywGdr23oy9TY6L8HUl1NsFgNiLBdSYmy8DJBynBmcUrayZr7Mvoe6hze/UycK9/HFV1L5WtcZ0jqdntbBH33PsUiGgliXb9KvZzB0WqL329q4kvvePfaTkldntoygKf/7mNPWxrAfmD2mx/Fa9hPwEbXuEf8/InDHpTcwIV68dhVWFHM893uz4BSOCtO0tCY73OtuSkKOVM1sSHdJiabjGjIrw5tq6LIy80ir+tTelwZgre83cf80Q9HVBoPg8a+bMJ9d+wqJ+i7THn537jNu+vY3EAuu+9Z4/8rzWn+aO6DsI9whvMKavmhE+QyupuT11e4PgbnSY9W+Yk3U3J+w7f5mSumyzhSODcTJaQyz7M/Y3GQRrjLHdMwfeeustACIjI9m2bRuDBg1y5HBt8vOf/5xrr72WBQsW8Ne//lV7Pjk5maysLBYtsr75nJ2dmTNnDvv27eOee+4hNjaWmpoauzFhYWHExMSwb98+Fi9e3Og5q6qqqKqy3rlTXKz+QVJTU0NNjTSeE+JqVn8NkGuBEEKuB0KIenI9EFerr46laot9y0eHUFvb+obE0wf48pa6ns/Oszksiw5qfodeoq9cD05nFGvl6mLCvJr9ejYlb6LaUg3AoqhFKGaFGnPrvv7JQZO17b1pe/neoO81OXZiPx++icuiosbM8YuXtabaonOcSi9EZypAb1Sbx4/0G9mmz3hnGeYzmhPFaeh0Ch8d30FMcNPvmZ6gI68JZVW1nMtWqwUNDfIgxNPEW7eN5f0Dl3hu0zlqzApxaUVc+8punlg+guvrshk3nMziUIpaxqqfnxu3TQpv9XwSLluDM4M8B/WYa9vssNlaCa+tF7cy2m90k2MD3IxEh3lyKqOE+PQiLuWVaMHe9tgQb80YXTA8sN3fkwfmDWTjqSzMFoV/7LzA6vFh+LhZA+EHkvI5clHNFBwc6M6CYf7U1NRgtpg5lae29QhxCyHSPZKnpz/NlOApPBf7HJXmSs4XnufWb2/lV+N+xU1DbkKn03Ew6yDbU7cDEOASwB3D7+gxP8+u4KJzYULwBA5mHSS9NJ0zeWcY7DNYe32Anwsmg44as8LJ9CJqampYF5euvb7oip/1ppRNbTq/Q8GZuLg4dDodjz/+eJcGZj755BOOHj3K4cOHG7yWlaVGx4ODg+2eDw4O5uLFi9oYJycnu4yb+jH1+zfm6aef5k9/+lOD57dv346bm1ubvw4hRN+zeXPLdTyFEFcHuR4IIerJ9UBcbT6IMwDqXbyeBedYv/5cq/ettYBJb6DGomPryXS+db5EO24+7rF6+/Vgb7YOUEsmGYrTWL++6ZJjH5R+oG17Z3izfv36Vp/Holhw17lTppSxP30/X3/7tXZn85VcS61z+mDjfjLCr76SVl2lygzJeQYMntaSZqZ8U5t+tp0losKLE3XbO5L3sn5971in64hrwvkisCjq58NXKdZ+HsHAL0fCfxIN5FTqKKs28+s1J/l0Zxw39LfwQrz1Wr0wsIQtmza26nwWxUJCkRqc8dH7sHvLboe/ho5SYalAjx4LFjac3cCwzGHNjo/S6zhVd/14dc12Zoa07/pRZYadZ9Xvp5dJISN+H1kOtD+aHKBnf46ekspafvv+Vlb0s2YFvnpKT30xrGnexWzcuAGALHMW5bVq0NS/xl97HzjjzN1ud/O/sv+RZcmiylzFM0ee4avjX7HSbSX/Kv2XduzZutns2Lyj/RPvpQKrbDLNtr3NXJe5dq8HuxhIK9NxIbeUNV+vZ8MJ9WftrFcovXCE9cnqOLNiZkuxWpXLCadWnduh4Ex9VGjcuHGOHKZNUlNT+eUvf8mmTZtwcWk6mnll6piiKC2mk7U05tFHH+Whhx7SHhcXFxMZGcm8efPw9/dv5VcghOiLampq2Lx5MwsXLsRkajq1XwjR98n1QAhRT64H4mp0IbeMtP1qGaqYMC9+dOPUNh/j6/yj7EzMo6hGx5CJsxga7NnR0+xyfeV6sOfLU4B6x/DNC6YyoZ9vo+NyK3L545d/BCDcPZy7l9/d5hI/+/ftZ0PKBqqpJnxSOBOCJjQ6bmhOKf97dR8Apa4hLFvWdWtUV5ujlwpRDh3C4GoNzqycspI5EXO6cVaqyRWT+XbtJwCUGVOYNGs+gZ6tK9HVHTrymvD27mQ4rZaqWj49hmUTIuxev72qlr+sP8Oao2pmx5E8PaeLTZRXqz1Zpg/y45HvT2j1ZzS5KJmab+vWhMPGsWx2y/2kutJ3W74jNieWPEseI2aMYID3gCbH9s8sZsMbBwDINgaxbFnj15mWbDiZRe0htd/LdeMiue7ake06Tr1xRZUseHkP1bUW9uQYeeL7MwnxcuFQSj7n9x8BYIC/G4/9YIbW12bt+bVwSN1/YcxClo2w/7ncar6Vvx/7O5+cUz8nCbUJpJSnUGFRS3CN9BvJ7xb/Dr3O4Rb1vc7YsrGs+2odAFnuWSxbbP+921N9is9i01HQkeE5jHLzBQAWRoey8jprdtbBrINUbFO/nzMjZnJCCxk3zaHgTP/+/UlISKC0tNSRw7RJbGwsOTk5TJhg/bCYzWZ27drFa6+9xtmzZwE1OyY01FrzMicnR8umCQkJobq6moKCArvsmZycHKZPn97kuZ2dnXF2bnhhN5lMvfqPKyFEx5HrgRCinlwPhBD15HogribrT1mbKl8/Lrxd7/05w4LYmaj2m9mXVEh0hF+Hza+79fbrwckMtXSSXgdjovwx1TUev9K2xG1YFPVO72UDl+Hk1Lo7iG3NipjFhhT1jvBD2YeYGt54oG94mA9+7k7kl1Vz5GIBBoNR678gOta5HLVnSX2/GYAxwWN6xHs61BSKlyGMYnMGetc0tp7L5AdTh3T3tFrUEdeE+PQSbXtC/4AGx/MxmXhh9ThmDw3isbUnKa2q1QIzeh08vjymTZ/RCyUXtO2RASN7xM/f1ryoecTmxAKwJ2sPQwOGNjl2dKQfYd4uZBRVciCpgCqLDg/nti+Xbz5j7ZG2bFT7fvfZigowcce0fryzO5mqWgtv7krhqRtG8ebOFG3ML64Zgouz9ed2uuC0tj02eGyDOZhMJh6b9hgzImbwh71/oLCq0K43ym+n/BZnp54b0OxM/Xz6Mcx3GGcLznLy8kkKawoJdLNm04yK8OGzWPXGhPf2XtSev25MmN33eUf6Dm17Xr95vM7rLZ7boVDYqlWrANi6dasjh2mT+fPnEx8fz/Hjx7V/EydO5Pvf/z7Hjx9n4MCBhISE2KUFVldXs3PnTi3wMmHCBEwmk92YzMxMTp482WxwRgghhBBCCCGEEI1TFIWvjquLFzodLB8T1q7jzB4aoG3vSsztkLkJx1VUm619LYI9cXVqPDAD8G3St9r2dQOva9f5poZagzH7MvY1OU6n0zGpv3rjbXFlLWezS5ocKxxzKqMYsGBwUT/nQW5BdguY3W1ckJo1pdOZ+ebsgW6eTdc5nloIgLuTgcFBHk2OWzk2nPUPzLLry/T9Kf0YFtK27MSEfGu/mRF+I9q0b1e4JvIabXtH6o5mx+p0OhaMVG/mrzZb2HWu7b9zKmvMbEvIBsDb1cSUgR1zQ8HP5g7WAkWfHk7li6Np7DmvBoGi/NxYOdb+d2xcnpq5Y9AZGOnfdObO3Mi5fL78cyaHWHt7Le2/VPv8XK3mRs7Vtnek7bB7LTrMW9surVJ7bLmaDMwZau2LZ1EsbLu0DQAnvRPTw1oXY3AoOPPwww8TFRXFyy+/zJkzZxw5VKt5enoSExNj98/d3R1/f39iYmLQ6XQ8+OCDPPXUU6xdu5aTJ09y55134ubmxm233QaAt7c3d911Fw8//DBbt27l2LFj3H777YwaNYoFCxZ0ydchhBBCCCGEEEL0JSfSirh4Wa13P22gP8Fe7WusPCjQQ2vKfCg5n8oac4fNUbTf6cxizBa1H8PoCO8mx10svsjJy2qzheF+wxnoM7Bd5wt0C2Sor3rH++nLpymoLGhy7OQB1lLzh5Lz23U+0bJTGcXonC6jM1QBEOMf080zsrdwwDRtO/7yMSqq+/61I6uokqziSgBGR/hoJa6aEuXvxmf3TuMP143kvrmD+N2ytgdXzly2rgEP9xve5v07W6RXJIO81d7ox3OOk1/Z/DVh4Uhr3/Itp7PbfL59F/Ioq3uvLRgRjMnQMWXB/NyduHu2ev00WxQe/sxaIusX8wZjtDlPeU05FwrVjKYhvkNwNbo2e+xg92DeXvg2f5j6B+4YeQd/mPaHDplzbzYvap62fWVQb0SoZ4P+d9cMD7K7SSEuN47cCjW4Nz1sOu4m91ad16F3i7e3Nxs3biQ4OJgZM2bwxhtvUFDQ9C/LrvLII4/w4IMPct999zFx4kTS09PZtGkTnp7WSPBLL73E9ddfz+rVq5kxYwZubm588803GAxN3/khhBBCCCGEEEKIxtVnzQAN7uhtC51Ox+wh6t34VbUWWWzvIeLSCrXtURE+TY5bn2xtDr9sgGO9KGaEzQBAQeFAZtOZEFMGWO9Ul/dL56gxWzibVYLBpqRZdEB0N86ooYmhE60PXJLZfRVk3h1Pta7Djo3yadU+JoOeu2YO4JElw5vNgGuMoiicyVeDM34ufgS5BbWwR/eoz4JQUNiZurPZsVMG+GsZKtvO5lBrtrTpXBtPZmnbS2JC2jbRFvx45gD83dXSZYoaGyfcx5UbxofbjTt1+ZRWSnJUwKhWHdugN7B62Gr+b9L/4enU+3u7OWqk30iCXNX384GMA5TXlGuvuTkZGRRon5W2dJT9z3rrJWtlsfn95rf6vA4FZwYOHMjSpUspKiqioKCA+++/n8DAQEJCQhg4cGCz/wYNGuTIqe3s2LGDl19+WXus0+l44oknyMzMpLKykp07dxITYx/Nd3Fx4dVXX+Xy5cuUl5fzzTffEBkZ2WFzEkIIIYQQQgghrha1ZgvfnMgEwMmgZ0lMaAt7NG+WbWmzdpSZER0vPq1I2x7TROaMoiisT1KDMzp0LB2w1KFzTguzZkI0V9psRKiXtrh6MDkfpX4VU3SY8zmlVJstGFytwZmeljkT5h6Gj5Ma2DW4XmTT6fQW9uj9jtWVNAPsypV1luzybAqq1IDQcL/h6K5MJ+gh7EpUpe5odqyTUc+cYer7prC8htiLrU88qDVb2FyXbePmZGDWkIAW9mgbD2cjv7hmsN1z980b1CA7Jy43TttubXBG2NPpdNr7ptpSzf7M/Xavx4R5advORj3zhlkDk4qisOXiFkAtKzc3Ym6rz+tQcCYlJYWUlBRycnK0iVgsFnJycrTXmvsnhBBCCCGEEEKI3m9/0mXyStVSR/OGB+Lt6lgz5JmDA7QSIrsT85ofLLrEibrMGZNB12SPitP5p0kpTgFgQvAEQtwdu4t8fPB4XAxqibt96fuaDLoY9Dom1vWdySutIuVyeaPjRPup/Wawy5xprq9Fd9DpdEwNm6Ru62vYmnRMK8XXVx2/VKhtj+uC4Ex91gz0zJJm9UYHjsbPRc2o25+5n8raymbHLxxhLW22uQ2lzQ6nFFBQXgPAvGFBuJg6viLTbVOiiPRTy5SF+7jyvQkRDcbE58Vr26MDR3f4HK4WtkG97Ze2271m23dmztBA3OtuCAA4V3COtFL12jgxZCI+Lj6tPqex5SFNu+OOOxzZXQghhBBCCCGEEH3AV8cztO2VY8ObGdk6Pm5OjI7w4URqIWezS8gurmx3DxvhuJLKGpLyygA1S8XZ2PgCZH3WDMCygY6VNANwNjgzIWQCe9P3klORw4XCCwz2Hdzo2En9/dhxVs2yOpR8mQEBrav3L1rnVEYRYEbvon7WIzwi2rQA2VUmhUxgY4r6PizVneN4aiET+vl286w6h9miEJ+uZrSFebsQ1AXXyIT8BG17hF/b+9V0Fb1Oz9zIuXyR+AUVtRUcyjrE7IjZTY6fNywIg16H2aKwOSGbx64d0aqsoO9OWUuaLe7gkmb1nI0GPvrJVNYcTWPl2PBGr7/xuWpwxsPkwQDvAZ0yj6vBlNApuBndKK8tZ1faLswWMwa9+v2eOyyQpzckYFHg1slRdvttubRF214Q1bZ+9g4FZ/71r385srsQQgghhBBCCCF6ucoas1Zz39PZyDXDO6YHwZwhAZyoK9mz61wuN02UUuTd5WR6sdbvYHQTJc3MFjMbkjcAYNQbWdRvUYece0bYDPam7wVgb8beJoMztn1nDibnc/OkqEbHifY5lVGM3jkHnV7NEogJ6FklzepNDLb2nTG6JbP5dHafDc6cyy6hvK4RfWv7zTjqzOXekTkDMDdCDc4AbLu0rdngjLebicn9/difdJmLl8u5kFvK4KDm+7BYLIr2u8/JoGdeXWm0zhDp58aDC4Y2+lpWWRY5FWpVq+iAaPQ6hwplXdWcDE7MCJ/B5oubKagqIC4vjnFB4wAYEuzJtw/Morza3OCaYttv5pqoa9p0TvlpCSGEEEIIIYQQot22ncmhtKoWUO8c7qiyLrOGWhe6pLRZ94qrK2kGMDrcp9ExR7KPkFuhZq7MDJ+Jt3PjQZy2mh42Xdven7G/yXGjIrxxNqrLXIeS8zvk3EJlsSgkZBTblTSL9o/uxhk1bYD3ALydfAAwuKWwOSGzeyfUiY53cb8ZsJY1czO6EeXVswOgU8OmamURd6btxKJYmh2/YKS1tNmmVpQ2i0svIqtYLZc2Y7A/ni6OlfNsL7uSZgFS0sxRzZU2GxHq1SAwc7H4IokFiQCMCRxDkFvbblCR4IwQQgghhBBCCCHa7avj1qbbK8eGddhxx0b6aE3e95zPw9LHe0f0ZHF1pZMARkc2HnRZn2wtaXbtgGs77NwDvQdqi11Hso9QZa5qdJyz0cC4uuyBtIIKMgorOmwOV7vUgnJKqmrRu9oEZwJ6ZnBGp9MxKUTNntEZKkkuOk9yXUm+vsa238zYyM7PDiqqKiKjTC1rN8xvWI/P0HA1ujI1bCoAeRV5nMo71ex4274zW1oRnKnPmgFY0kklzVqjvqQZwKiAUd02j75iVvgs7b29PXV7C6Pts2baWtIMOjg4U1lZyd69e1mzZg0ffPABxcXFHXl4IYQQQgghhBBC9CBF5TVsP6NmSwR4ODN9UECHHdtk0DN9kD8A+WXVWkNy0fXqM2dcTHoGB3o0eL3KXMXmlM2Aekf9nMg5HXZunU7HjLAZ2nlis2ObHDt5gL+2fThFsmc6Sv1nrz5zRoeOkf4ju3NKzZoQPEHbNriltGqhvTeqz5wx6HWMCu+YTLXm1GfNAAzzHdbp5+sI8yLnadstLbRH+bsxLFgtZXYstZDcksYDwQCKorDxpJqVpdfBApvATleLy4vTtkcFSnDGUb4uvlops5TiFJKLkpsdv/WiNTgzP2p+m8/XIcGZ1NRU7rjjDnx8fJg9ezarV6/mzjvvJC0tzW7cu+++y+TJk1m4cCGKIne8CCGEEEIIIYQQvdnGU5lUm9VSMcvHhGLQt9xAuS1sS5vtSszt0GOL1ikoqyY1X81CiQnzxmhouJS0J20PJTUlgLo45Wp07dA5tLa02eT+9n1nRMc4lVEEulr0LmqmwADvAbib3Lt5Vk0bHzxe2za4JbM5oe8FZ0qrajmXo37mhgV74urUMeUkm2MbnBnhP6LTz9cRZkfMRof6e6k1WRALRqpZeooC28/kNDnuXHYpKZfLAZg8wA9/D+cOmG3b1VpqOX35NABh7mEEuHbcDRJXM9ug3s7UnU2OyyrL0oJjQ32HEunV9t54DgdnDh06xLhx4/jwww+prq5GUZQmAy8rVqwgLi6Obdu2sWnTJkdPLYQQQgghhBBCiG701fEMbXvl2PAOP/6cITbBmXMSnOkOtiXNRkU0fnf+t8nfatvLBi7r8DlMDZ2qLbDuzdjb5Ljx/Xww1gUIpe9MxzmVUYzeOROdTm0+31P7zdQb5jtMCx4Z3JI5knKZgrLqbp5Vx4pLK6R++XVsXTm/zpaQn6BtD/cb3iXndFSAa4CWTXK+8DypJanNjrfNgGmu74xdSbPo7itpdqHwAhW1avBcsmY6jl3fmWaCetsubdO221PSDBwMzhQVFbFy5Ury8/MJCQnhjTfeID4+vsnxgYGBLF26FIBvv/22yXFCCCGEEEIIIYTo2bKKKtmfdBmA/v5ujGli4d4RUf5u9PN3AyD2YgGlVbUdfg7RvPi6kmYAYyJ8GrxeUl2i3Vns5+LH1NCpHT4HHxcfLSCQWJBIbnnjgTo3JyMxdeWdzueUklfadFki0XqnMooxuFh7S/XUfjP1DHqDVpZIbyxFMeWx/WzTWRC9UX1JM1D7c3WFM5fVzBmjzshgn8Fdcs6O0NosCFCvcYGeahbMnvO5VFSbGx238ZQ1OLOoG4MzdiXNpN9Mh+nn1Y8B3gMAOJ57nPzKxoP9tv1m5vdre0kzcDA48+qrr5KdnU1AQAD79+/n3nvvJTq6+Qt0fUmzQ4cOOXJqIYQQQgghhBBCdKN1cRnandsrxoaj03VsSbN6s+uyZ2otCgcuXO6Uc4imnUizZs6MbiQAt/XSVqotalbC4v6LMeqNnTKP6eHW0mb7MvY1OW7KAGtpsyPSd8ZhOSWV5JZUYXC1ZhzEBMR044xax7bvjNEtmc19rO/M8UuF2va4LgjOVNZWklys9t4Y5DMIJ4NTp5+zo7Sl74xer2PBCLW0WWWNhb3n8xqMuXi5jIRMtQ/TmEgfwnw6toxjW8TnWpMkRgeO7rZ59EX17xuLYmF32u4GrxdUFnAk+wgAUZ5RDPEZ0q7zOBSc+eabb9DpdDz00ENERUW1ap/64M2FCxccObUQQgghhBBCCCG6kX1Js7BOO8+sIdYa+tJ3puvF1WXOeDob6e/fsM/I+qT12vayAR1f0qyebd+Z5oIzk22CM4eSCzptPleL0xnqIrTeRe0rbdQZe0Uz+InBE7Vtg2sKO8/lUlnTeBZEb6MoipY54+lsZFCgR6efM7EgEYui9hfrLSXN6g30Hkikp9oLJDY7lqKqombH25Y2ayyo992pnlHSDCA+Tw3OGHVGRvj1jj5AvUVLQb0dqTu0z8T8fvPbfYOKQ8GZxMREAGbPnt3qfXx8fAAoLi525NRCCCGEEEIIIYToJhdyS4mv60UyKty7UxcHpw3y1/qI7E5seBez6DzZxZVkF6ulwUZFeKPX2y8+5VXkcTDrIADhHuGMCRzTaXMZHTha6yNyIPOAtih2pYn9/KhfIzuUIplWjjqVUQy6avTOalmwwb6DcTG6dPOsWhbtH42zQS1PZXBLprzazIGkvvF+yCyqJKdE/VyOjmz4uewMtv1mRvj3riCATqfTeoiYFTN70vc0O37G4ABcTOqS+dYz2Vgs9r3VbfvNLI4OpruUVpdyoVBNfhjiO6RXfC57k1EBo/BzUYP9+zL2UWW2L5O55dIWbbu9/WbAweBMRYXacMjdveGdE00pLS0FwMVF3jBCCCGEEEIIIURv1FVZMwCeLibGR/kCkJxXRmp+eaeeT1jF2ZQ0G9VISbONyRu1IMmyAcs6rbQdgElvYnLIZADyK/M5k3+m0XHebiaGBXsCatZHcWVNp83panA6oxiDSwY6nbpAXd/7p6czGUxamSe9UwE6Y2GfKW3WLf1mbD5vvS1zBtpW2szFZGBWXTnNvNJqjtv03couruRoXUm5YcGeDOyCrKWmnLp8CgX1cyklzTqeQW9gdoSakFJRW8HBzIPaa6XVpezP2A9AkFuQQ6UeHQrOBAaqb9TU1NQWRlrFxsYCEBoa6siphRBCCCGEEEII0U021ZV10engutGdG5wBKW3WXeJtFiXHRPg0eN22GfK1A6/t9Pm0trRZfd8ZiwKxF6W0mSNOZRRpJc0AogN6R3AG7PvOGNyS2ZKQjaIozezRO9gHZ3y75Jy2wZneUNbuSuOCxuHl5AXAnvQ91JibD9ouHGnNiNliE9TbZFPSbHFMzyhpBmqWh+h4tkG9Hak7tO3d6bupsajvoflR89Hr2h9icSg4M3myesfChg0bWjXebDbz9ttvo9PpmDlzpiOnFkIIIYQQQgghRDfIL6vmTFYJADFh3oR4d35ljNlDA7Xt3eektFlXOWGbORNunzlTZa4iLjcOUEuaDfIZ1OnzmRE2Q9tuvu+Mv7Z9KDm/U+fUl5VU1pByuRyDq01wppdkzkDD4Ex2cRUn03t/m4Vjl6wBx67InKm11HKu4BygNj73cOq+bJH2MuqNWhZEWU0Zh7MPNzv+muFBWnlE24yr705Zt7u738yJ3BPa9qhACc50hqmhU7XyiLY9ZmxvTHCkpBk4GJy59dZbURSF9957j2PHjjU71mKxcO+993L69GkAbr/9dkdOLYQQQgghhBBCiG5gu9g9xab5emeKCffGx80EwN4LedSaG+83IjqOoihaXyE/dycifF3tXo/LjaPaUg3YN1/vTJFekUR4RABwLOcY5TWNl7ibNMCaTXBYgjPtlpCpBmENdZkzTnonhvgO6c4ptcnogNEYdUYADG4pAGxO6N2lzWrMFu1zGeHrSqCnc6efM6UoReu30RtLmtWr7zsD9lkQjQnwcNbKaSbmlJKSV0ZheTX76/oWRfm5MSLUs5Nm2jJFUYjPVTNnPJ086e/Vv9vm0pe5mdyYGjoVgNyKXE5fPk2VuYpdabsA8HH2YXzweIfO4VBw5sYbb2T69OlUVVUxf/58Xn/9dXJycrTXdTod2dnZfPDBB0ycOJH33nsPnU7HkiVLmDt3rkMTF0IIIYQQQgghRNc7mGxtqj1loH8zIzuOQa9j5mC1tFlJZS0nbMptic6RVlBBfpkafBkd4d2gn8yR7CPa9sSQrgnOAMwIV7Nnai21HM5q/O73IE8XBgSo/ZFPpBVSWWPusvn1JacyikBfgd5ZzVYb7jcck97UzbNqPTeTGyP9RwJgcM5BZyjt9X1nzmaVUFmjBqe7qt9MQn6Ctj3Cf0SXnLMzzAyfqb1/t6dub7HE3YIRNqXNErLZkpCD2aLusyQmpFN7bLUksyyTy5Xq7+JRAaMcKqslmmcb1Nueup39GfupqK0A1LJnRr3RoeM7/JP78ssvGT58OIWFhTzwwAOEhoZqb87x48cTFhbGnXfeyYkTJ1AUhZiYGP773/86elohhBBCCCGEEEJ0g/rMGZ0OJvfvmswZgNlDrKXNdkpps04XZ1PSbPQVJc0AYrNjte2uypwBmBY2TdtutrRZ3XuzxqxwrK6Bt2ibUxnFGFzStcf1gY7exL60WQoJmcWkFTSecdUb2Peb8emSc/b2fjP13E3uTA5RW3RklWVxtuBss+Nt+85sPp3NxpM2/WaigxvbpcvE5cVp29JvpnNdGZzZcnGL9nhBP8dKmkEHBGcCAgI4cuQIP//5z3F2dkZRFO1fVVWVtm00Grn77rvZt28fPj4+Dk9cCCGEEEIIIYQQXauooobTmWrPhuEhXni7dd1d9LOGBmjbuxNzu+y8V6u49EJte3SEj91rNeYaTuSo/Q5C3EMI9wjvsnlNDpmMQWcAWuo7Yw0cSt+Z9jmVUWzXbyYmIKYbZ9M+V/adAdiakNPU8B7PNjgzLsqnS85pG5zpzZkz0HChvTmDAt21DLzDKfnsqvu9E+jpzLhI3+Z27XT1Jc0ARgeO7saZ9H0BrgGMDlC/x4kFiWy6uAkAN6MbU0KnOHz8Dsl5cnNz49VXXyU1NZUPP/yQBx98kNtuu42bb76Z++67j3feeYfk5GT+8Y9/4O7u3hGnFEIIIYQQQgghRBc7kpJPfSWYruo3Uy/U25UhQWoj6hOphRSV13Tp+a82cak2mTMR9pkzpy6fotJcCahZM11Z3sfTyZMxgWMASClOIaM0o9FxdsGZlMuNjhFNq6o1k5hdgt7FGpyJ9o/uxhm1z9igsehQ35/1wZneXNqsPjhj1OuIDmuY0dbRFEXRypoFuAYQ4BrQwh49m11w5lLzwRmdTseCEUEAWBSorlXLyS2ODkav776SZgDxedbgTG8MmvY2tu+b+pJmsyNm42xwvOdThxak8/f357bbbuPFF1/kww8/5OOPP+a1117jrrvuIiwsrCNPJYQQQgghhBCih3t9+3mWvLyLLb14IUzYO2iTgTB1YOuCMx8lfMQNX93A+qT1Dp9/Vl1pM4sCey9IabPOYrEonKxrOh7i5UKQl4vd63b9ZrqwpFm91pQ2i/B1JdRbnffRi4XUmC1dMre+IjG7lFqLgqEuOONqdGWA94BunlXbeTt7M8R3CAAG50zQV3Ig6TLFlb0vuFtcWcOF3FIARoR64WIydPo5M8oyKKkuAdSeQ71diHsII/zU7J+E/ASyyrKaHW/bd6bekujQTplba9VYajh9+TQAER4R+Ll07Y0SVyPb4Ey9+f3md8ixpVuQEEIIIYQQQogOl11cyXPfneVMVgn3fXSUOGng3iccTLJmIExqRb+ZspoynjvyHOcLz/O7Pb/jYOZBh84/26a02a5zUtqssyRfLqOkqhaAUREN784/kmUTnAnp+uDMjLAZ2nZTwRmdTqdlz1TUmLVgU1+UWVRB7MWCFhuct8WBpMvoDKXonQoBGOE3AoO+84MBnUErbaZTMLhepNaisPNs77t+xKUWaZmLjfWbyavIIzY7tkPfB7af9fqgRm83L3Ketr0zdWezYyf088XXpnynt6uJKa28MaGzJBYkUmWuAmBUoPSb6QqDfQYT4RGhPXbSOzErfFaHHLvTgzNVVVVs3bqVTz/9lEOHDnX26YQQQgghhBBC9AA7bRbOq2st3PNBLLklVd04I+Go0qpaTmao/WaGBHng79FyOY+DmQeptaiL/GbFzP/t/D/SStJa2KtpUwb442RUlzJ2J+Z16CKksIpPswYyxlwRnKm11HIs5xigljmK8ozq0rmB2pje21md14HMA9p77EpXQ9+ZgrJqrn1lDze+uY/frInDYnH8M7HvQh7PbjyL3iVde643l05qrO/MloTel9F5PLVA274yOFNeU85t397GnRvv5OGdD1NjcTwz6FTeKZ46+JT1nEFjHT5mT2BX2iyt+dJmRoOeecODtMcLRgRjMnRvroNdv5kA6TfTFXQ6nd37ZnrYdNxNHdO6xaF308WLF3nkkUd45JFHKCwsbPD6gQMHGDRoEIsWLeK2225j2rRpTJo0iUuXLjlyWiGEEEIIIYQQPdzOK7IaMosq+dmHsVrNdtH7HEnJx1y38NvaO4f3pu+1e1xYVcgvt/+S8pryds3B1cnA5LqMnfTCCpLyytp1HNG8EzaZbqMjfOxeS7icQHmt+vPr6n4z9Qx6A1NDpwJQUl3CybyTjY6bchUEZzadziK/rBqA/x1J47Ev4x0K0JxML+Lu/8RSbbZgcE3Vnu8rwRlnjxQAtp/J6bZSdydSC7n3v8eIzWvbZ6e+3wzA2Cgfu9f2ZewjsywTgM0XN/Po7kebDFq2RlJREj/b8jPtsz4vch4zw2e2+3g9yXC/4YS4hwBwKPMQZTXN/x5ZNc6aMXHjhPAOm8f5gvM8uP1BPj/3eZv2i8uL07Ylc6brXDfwOq1/1crBKzvsuA4FZ9auXcvzzz/Ptm3b8PHxsXutpKSE66+/nszMTBRF0f7FxsZy7bXXUlvb/guEEEIIIYQQQoiey2xR2JOo9gPxdDESUtev4sjFAh7/+lR3Tk04wLbfzJQB/i2OVxSFvRlqcMaoN9LPqx8A5wrO8Ye9f2h31susIVLarLPF2WTOjAq3z5zp7n4z9aaHTde292fsb3TMoEAP/NydADhkE1zsSzadss8A+fhQKn/8+mS7Pl/JeWXc8d4hSutK2gUFWD9f0f7Rjk20GwW4BtDfqz8AOpdU0FVTXFnL4W4I2JVU1vDT/xxh65lcPkjUs+f85ZZ3Qr2e1gdnvFyMDPC3v2t/26Vtdo+/S/mO3+35HWaLuc1zzCrL4t7N91JQpWbqTAiewLOzn0Wv6xvdMXQ6HXMj5gJq/5YrbyK40swhAfz3J1P46KdTmD4ooNmxrVVtruaX23/J1ktb+dP+P/Fdynet3jc+T82cMeqNfaIPUG8RHRDN+0vf5435b7Cg34IOO65Dn6rNmzej0+m4/vrrG7z29ttvk5OTA8ADDzzAV199xX333QfA6dOnef/99x05tRBCCCGEEEKIHupEWiFFFWpJlVlDAnj7hxO0UlQfH7rEhwcuduf0RDvZ9ptpTebMxeKLpJeqZZEmBE3glXmvaGVANl3cxD/j/9muecwaEqht764LAoqOU2u2cCpDDc5E+bnhWxfcqGcXnOmGfjP1bIMzzfWdmdTfF4CSylrOZpV0ydy6SllVLbvPq58BD2cj+rpEjA8PXOJP35xuU4Amu7iSH7x7kMt1WTgT+/tgdFU/v55OnkR6Rnbs5LvY+ODxACiYtYygrWdyunwer2xNJKeuxKeCjoc+iyOzqKLF/dIKKsgrVX82YyJ90OutWTc1lhp2pqm9U1wMLpj0ao+UDckb+MPeP7QpQFNYWci9m+/VsnCG+w3n1WtexcXo0upj9AZ2fWfSmu87AzBjcECHBWYA3j/1PpdKrJWlHt/3OClFKS3uV1xdTHKRWppvuO9wnA0tlxcVHWdc0DhmRXRMr5l6DgVnkpKSAJgwYUKD1/73v/+h0+m44YYbePnll1m+fDmvvfYaN910E4qi8PnnbUvZEkIIIYQQQgjRO9g2Wp4zNJDRET48s8paeuOJr0/12RJDfVVFtVnLphgY4E6QZ8sLdfVZMwAzwmcw0Gcgz8x6RisL8uqxV9mVtqvNcxkR6klAXb+b/RcuU1Xb9jvD+4ovjqbx0uZzZBdXdtgxE3NKqaxRyz2NuqLfjNli5li22m/Gz8WPgd4DO+y8bRXiHqKdPz4vnuLq4kbHTbbJ8jqc0reuOzvP5VJda8HoGc+wEXv4w/Uh1FeZ+/e+FP76bUKrAjRF5TX88N1DpBWoQYLhIZ48fVM/LleqgZ9o/+huKV/XkWxLmxnr+s50deZdYnYJ/9qbYvdcQXkNv/joWIsl1mxLmo27ot/M0eyj2vt/XuQ8Xpz7Ika9EYBvkr7hif1PYFFaLuFWXlPOz7f9nAtFFwCI9IzkzQVv4unk2eK+vc2EkAm4Gl0BNbjblf3LMkszeTvubbvnymrKeGjnQ1TUNh+osy3hKCXN+gaHgjP1mTHBwcF2zxcXF3P06FEAfvSjH9m9dssttwBw4sQJR04thBBCCCGEEKKHsu03M3uomuWwanwEd80cAECtReFnH8aSXtjy3cKiZzh6qYDaNvab2ZO+R9ueET4DUBsx/3zszwFQUPjNrt+QVJTUprnodDpm15U2q6gxE3uxoIU9+qaT6UU89L8T/H1rItc8v4M3dpx3OFBlsShsOJmlPR5zRXDmXME5SmrU7JMJwRO6fcG+PnvGrJg5lHmo0TF9ue/MplNZ6EyXcQn/iHNVX/J64j0sm3UCnV7NsHh3TzLPbDzT7MJzRbWZH79/mLPZ6s810s+V9380iQPZ1kbpvbnfTD3b4Iy3XxqgBiK76veQoij88atT2nX0jmlR+Dmr27EXC/jbhjPN7t9cvxnbkmbXRF3D3Mi5PD/neYw6NUDz5fkv+fP+PzcboKkx1/DQjoeIy1X7mQS4BvDWwrcIcO24bJGexNngzKSQSQDkVeRxruBcl537uSPPUWlWA+o3DrmRQd6DAEgsSOTJA082u298bry2PSpAgjN9gUPBmZIS9cJtNtv/8t+7dy9msxmDwcDcuXPtXouMVNMg8/P71i9EIYQQQgghhBBQUFatNRMfFuxJqLer9tqjS4czc7C60HO5rJp7PjhCRfXVm/XQm9iVNGtFv5nK2kqOZKnlr4JcgxjiM0R77e7Rd7Ow30IASmtK+eW2XzaZ9dCU+qAf2AcDrya2X3dZtZlnN55l0Uu72Hw6u813gSuKwnenslj2ym5e2ZqoPT86wsdunG1JM9vF7u5iW9rMNhhoa0SoFx7O6iL1weT8Lr1DvjPVmC1sPZOD0T0RnU79mqrMVezK/ZjQ6Fcwep0AFN7amcTzm842+nXXmC3c999YLcDp7+HEr5Yr/GrPXTx7+FltXIx/7w/OhLmHaU3gq41JgNpXp6uyZ9bFZbK/7joa6efK/y0cwp1DzJgMaoDzn3uS2Xgys8n9bYMzY2w+l4qisC1VDc4Y9UZmhs8EYH7UfP42+28YdAYA1iSu4amDTzX6PrAoFh7b+5iW7ehp8uQfC/7R60vZtaQ114+Oti99H5svbgbU7MOHJj7Ei3Nf1LJ4vrrwFWsT1za5f32/GYDRgaM7d7KiSzgUnPH2Vu+gyMjIsHt+x44dAIwZMwZ3d/crdwPAxaVv1SoUQgghhBBCCAG7z+dRv/YzZ1ig3WtGg55Xbx1HlJ8bACfTi/ntF3F9ZrG0Lztgk3HQmsyZo9lHtTuDZ4TPsMuw0Ol0/HXGXxniqwZsUopT+O2u37apL8LMIda7uXeduzr7zhywCZjVf3svXi7np/85wg/fO8T5nJb7qyiKwtaEbK57dQ/3fBDLGZueLLOHBjKxn6/d+PqAG8DE4O7rN1NvQvAEnPRqT5ymShMZ9Dom1vWdySutIjmvrEvn2FkOJuVTUlmLwc2aeVZfMrCkNg/X8I9x7fcWeud0Xt9+gZe3JNrtb7EoPPJ5HNvP5gIKnt7J9I/5F48fetBuAXhc0Dgt86030+l0WkDRrFSjd1HXMm3LcHaW0qpa/vrtae3x49dF42Iy0M8Tfrd0mPb8rz+LI6WR92eN2cLJdGsfKH8Pa5+R0/mnySpTs92mhE7Bw8lDe21R/0U8M+sZ9Dp1+ffTs5/yzKFn7D4niqLwt0N/Y0PyBkDNKHl1/qsM87POq6+qD2SBfRnOzlJtrubpQ09rjx+a8BBeTl4M9BnI49Me155/8uCTnMlvmEmlKIr22fR29ibKM6rT5yw6n0PBmZgYNXK+dq01omc2m7V+M/PmzWuwT3q62kzsylJoQgghhBBCCCF6vyv7zVzJ192Jd344ETcn9W7er45n8M7utpW1El2rssas3bUd5edmlw3VlD0ZDUua2XIzufHKvFfwdlZv+tydvpvXjr/W6jkFeDgzKlzdNyGzuEN7rvQG1bUWjqSo2Q6h3i58e/8sJtuU79qdmMeSl3fz529OU1RR02B/RVHYeS6X69/Yx13vH+FUhjVzaUykD+//eDLv/2gSRoN12ciiWIjNiQXUhcH64Fp3cjO5aQvumWWZWqPsK03q3/dKm206nQUoGNzV66e7yZ0vVnxht+BsdEvBbcBrOId8wSs7jvFqXVaUoij89dsE1h5Lx+CahHu/tyHsLc4VWYMyQ32H8vK8l3l/yft9phn8+KDx2raHz0UA9p7Pa7Hfi6Ne3ZpIdnEVANcMD2LBSOua6PcnR7J8TBgAJVW1/Oy/R6mssQ9Un8ksoapWnePYK/rN2JU0i7ymwbmXDFjCkzOf1AJ3H535iOeOPKcFaN6Ke4uPznwEgEFn4IU5L/SIrLiuEOUZRYRHBADHso9RVtO5gdv/nP4PKcUpAIwNHMvyQcu1164deC03D7sZUDPgHtrxECXV9gH29NJ08ivV61dMQEy3l5UUHcOh4MwNN9yAoih88MEH/OY3v2HdunXcdtttXLyoXuBWr17dYJ8jR9S7LKKiJLonhBBCCCGEEH2JxaJopZZcTQbtbvUrDQvx5MXVY7THz2w4c9WWpuoNjqcWUl23MGjbv6M5e9PVu5D1Oj1TQ6c2OibCM4Ln5zyvld35Z/w/2ZiysdXzsg3+dXVj7+4Wn15IRd0C7tSB/owM8+LTu6fy+m3jCfdRg2e1FoX39iYz7/kdfHTwEmaLgqIo7D2fx/f+sZ873jvECZtSSTHhXrx350S+vG86c4YGNlj4O194nqIq9e798UHjtbvxu5tt8K+p0kR2fWdSen9wRlEUNp3KRu+Ui95YCqg/k8G+g3lzwZu8Pv91+nn1A0CnU3DyPYT7oOd55ch7vLr9DG/suMC/j+7ANeqfuPV/G72bNag1yHsQz895ns+Wf8b8qPl9agHYNtvLz0+9ebykqtauZFhHO59Twrt71O+vk1HP48tH2r2u0+l4etUoBgaqlYcSMot54utTdmOOp1r7ajUXnJkX2fAmeYDrBl7HX2b8RQvQfHD6A16KfYlPz3zK68df18b9ecafmRM5p41fYe+l0+m060etUttk36qOkFWWxdtxbwPq78XHpj7W4Br6yKRHiPaPBiC1JJU/7v2jXZaTXUmzAClp1lc49Jv0nnvuYcSIESiKwvPPP8/KlSv5/PPPAVi+fDkTJzZMcV27di06na5BLxohhBBCCCGEEL1bQlYxeaXq3cHTBvnjbDQ0OXZJTCgPXDMYAIsC9390tNFyLqL7HUyyLWnWcr+ZzNJMkorUu/lHBYzSsmMaMzV0Kg9PfFh7/Me9f2y0nEtjbMvm7Uq8ukqbHbD5mUytKzOn0+m4dnQoWx6aw4MLhuBiUpd88suq+d3aeFa8toeb3z7A9/95UOsxAjA8xJO3fjCBb34xk2uGBze5GN/TSprVa01polER3jgb1e9HX8iciU8vIqu40q6kWX1zc4DZEbNZu2ItD094GHeTuuivM1TiErKONxN/zmunH8G9/5sY3c9r+/T36s/fZv2NNSvWsLj/4h4TfOtIA7wH4Ous3jRQqksE1KBzZ5U2UxSFx78+Ra1FXWC/d/ZA+vk3bP/g4WzkH7dPwNWk/s785HAqn8emaa8fswkejY3y0bYvFV/ifKH6MxwdOJpAt4bZqvVWDl7Jn6b/SXv8r1P/4q8H/6o9/r+J/8eKQSva9gX2AV1V2uy5w89RUVsBwOqhqxnuN7zBGCeDEy/MfQEvJy8AtlzawgenP9Bej8uN07ZHBYzqtLmKruXQldbZ2ZmtW7eyatUqjEYjiqJgMpn4wQ9+wAcffNBg/K5duzh9Wq2xuHDhQkdOLYQQQgghhBCih7Ht/dFYSbMrPbhgKAtGqOVdiitr+el/jlBS2bAEk+heB5OtvU1akzlju8DVml4Vt4+4XVsUrKit4JfbfqmVbmnOuEgfPF3URu+7E3MxW66e3kW2/WamXhEwc3Uy8OCCoWx9eC7XjQ7Vnj+VUWwXmBgS5MHrt41n/QOzWBwd0mKGxJFsa3BmQkjPKXs00Hug1ug9NjuWytqGJe6cjQbG1S1qpxVUkF5Y0ZVT7HCbTmUDNBmcATAZTNwZcyfrbljH9YOv1543OOdg9LD2n4nwiODJmU+yduValg1chkHfdFC9t7PtO1NlKUPvrPZq6azMzfXxWew9r35Ww31c+dncwU2OHRrsyZM3xGiPf/9lPGey1HKD9Zk9JoOOkaFe2pjtqdu17cZKml3phiE38Mdpf2zw/I9jfswd0Xe0uH9fNDlkMka9+ntkT/qeTumBtz9jP5subgLAz8WPX4z7RZNjwz3CeWrmU9rjl2Jf4njOccA+c0aCM32Hw2HwkJAQPv/8c4qLi0lPT6e4uJj3338fT0/PBmMjIyPZvn0727ZtY9KkSY0cTQghhBBCCCFEb7XzXI623ZrgjF6v46WbxzA4SG1gnJhTymvbz7ewl+hK1bUWjl5SsyzCvF2I8G2530x9STOAmWEzmxmp0ul0/HHaH4nxVxcmM8oyeOXoKy3uZzTomTk4AIDC8hri0gpb3KcvuLLfTJSfW6Pjwn1cee228Xx691RG2CzoDgxw5++3jGXjg7O5dnQoen3LZasURSE2W+0342HyYLhvw7u+u4tOp2NGmBoErDJX2QWRbE0eYA1iHbhwudExvUVj/WYauxMfIMA1gL/M+AsfX/sxQU5Dtefd9YH8efqf+fqGr1kxaIW2QN3X2fZTCQ/NANRMpPqsz45SVlXLX789rT3+4/KRuDo1H/haNT6CWyerbSAqayz87MOjpBdWkJSrZpWODPXCxWQ9hl2/maiWgzMANw29icemPGY955BVPDj+wVbt2xe5mdyYEKS+J9JL07lYfLFDj19jruHpQ09rjx8c/2Cz2aQAcyLncFfMXYBabu3hnQ+TU55DwuUEQO2V4+Pi06HzFN2nw3IUnZ2dCQ0NxcnJqckxAwYMYM6cOcyZM6dP1awUQgghhBBCiKtdaVWttmDcz9+N/gENS7c0xtPFxDs/nIhTXePxz4+kaf1NRPeLTy+ksqau38xA/xb/L19jqeFA5gEAfJx9GOk/stnx9ZwNzrw872U8TGqgbn3yekqrS1vczzYIeLX0Lbqy30xLP5MpA/1Zd/9M3vz+eP5x+wQ2/Wo2K8eGY2hFUKZecnGyls00Lmhcj8uusM3Qsg0O2qoP5AF8dyqr0+fUWZLzyjiXXdqg30xLwZWYgBg23/IZv4x5mh8P/QO7bt3IDUNuwKQ3dcW0ewzbHlhOXtbeLrsTO/b68dr282QWqVlcc4cFsmhkcKv2e3z5SKLD1GBqcl4ZP3j3oPaabb+ZyxWXOZZzDFDLtQ3wHtDqud0y/BY+ue4TXrvmNR6f9vhVv0Zrd/3o4NJmHyZ8SHKR2nNodOBoVg5e2ar9fjHuF1r5yJzyHO767i6qLdUAjAqUrJm+pO8VkBRCCCGEEEII0eX2nc/T6uq3JmvG1oAAdxZFqwtXl8uq2Xw6u8PnJ9rHtrdJa0qaxeXGUVqjLhhPC5vWpkX8YPdgrh14LaCWN1ufvL7FfWZfhcGZxvrNtMSg17F0VChLYkIwGtq+FGTXbyak5/SbqTcldAoGnfpe25O+p9ExE/r5EuDhDMCOc7mUVtV22fw60ubTamCpuZJmTdHr9PxkwnX8atpqnIxN31zdlw3yGUR/r/4A5NacQWcoATq278yF3FL+uVv9+TgZ9DyxPLrVARAXk4E3vz9BK9lYnzUD9v1mdqbtREH9nduakmZXivaPZk7knD7ZW6itbIMzTV0/2iOrLIs3T7wJgA4dj015rNXfb6PeyLOzn8XfRc34SylO0V6TkmZ9i8OfwPLycsrLy5t8/dVXX2XWrFmMGDGCZcuWsW7dOkdPKYQQQgghhBCih7FdGG9rcAbglklR2vYnhy91yJyE4w7a9CiZckVvk8bYlTQLb7mk2ZVWDVmlbX+R+EWL48N8XBkarGbbnEgtpKCsus3n7G2a6zfTWWxLhdXfzd2TeDl5MSZwDKAuYqaVpDUYY9DrWBKjBoGray1sP5PTYExv0Jp+M6JpOp2Ohf3UPtgKCu6+aqmoXYl5WDqgb5WiKDzx9SlqzOqx7p49sNWZpPWi/N144aYxDZ4fG+mrbbenpJlo3BCfIQS5BgFqILqxvlXt8cKRF6ioVftbrR62utWZpPUC3QJ5bs5zDQI6owNGd8j8RM/gUHDmm2++wdPTk7CwMEpKShq8/uMf/5gHH3yQffv2cfbsWb777jtWrlzJs88+68hphRBCCCGEEEL0IIqiaMEZJ4O+XQvG0wf5E+mn9jPZcz6P1PymbwIUXaPWbCE2RQ3OBHk609+/8d4mtmzvOp4eNr3N5xzpP5IRfiMAOHX5FGfyz7S4z+whajDQoqjvnb6stf1mOpKiKMRmqf1mXI2ujPAf0ennbA/bu9/3ZexrdMyymFBte8PJzE6fU0fLLaki9lIBoODsqZZKaq7fjGjcov6LtG3vQDU4k19WzcmMIoePvfFkFrsT1etQuI8rP583uH1zjA7h7tkDtcc+bibtGlxeU87+jP0ABLkGERMQ4+Csr246nY7p4ervq0pzJUezjzp8zEOZh9iYshFQS3zeP+7+dh1nUsgku31NehPD/IY5PD/RczgUnPnuu+9QFIXrr78eT09Pu9f27NnDv//9bwDc3NwYN24cLi4uKIrC73//e06dOtXIEYUQQgghhBBC9DZJeWWkFah3h04a4Iu7c9sbS+v1Oi17RlHgf0dSO3SOou1OZhRTVq32NmlNv5m8ijwS8tWFzhF+IwhwDWh2fFO+N/R72vbn5z5vcfycYVdPabO29pvpCKklqeRUqFkm44LG9dgeJa0pTTR5gB9+7mo5r+1ncqmoe393JkVROJlexGvbEln1xl5GPf4dL24+165jbUnIRlFA75SLRa/eJN2afjPC3jDfYUR6RgJQwll0BrUUo6Olzcqra/nLutPa4z9cNwJXJ7XcnqIoJBYk8t7J97hz453M/mw268rXoShNZ+v8evEwrhmuZnTcNjlK+7zvzdir9R+ZFzVPSpN1ALvrR4Zjpc1qLDU8dfAp7fGD4x/E29m73cf7ccyPWTZgGQA3DL4BJ8PVWZKwr3Lo03vgwAF0Oh3z5s1r8Nrbb78NQFhYGAkJCcTGxnLmzBkiIyMxm8289dZbjpxaCCGEEEIIIUQPYbugVZ/F0B7fmxChNSn/7EgatWaLw3MT7XfQpnxWa/rN1N/JDe3Lmqm3dMBSXI1qFtX6pPVaWZimTOrvh4tJXd7YdS632cXO3q49/WYc1dNLmtUb4TcCPxf1e3Iw8yA15poGY4wGvdaYvaLGzM5znVParLSqlo0nM/nN53FMeWor1726h+c3nePopUJKqmp5ZWsi+y9cbvlAV9h0qv39ZoSVTqdjQb8FAChYMHqqAZVdiY4FZ17ffp6MIrUk1qwhAcwa6sXO1J38Zf9fWLxmMau+XsVLsS8Rmx1LaU0pB6oPsD1te5PHMxn0vHvHRA49Np9Hllizo+xKmrWj34xoaFroNC3ItS+98cy71voo4SMuFF0A1P4wNwy5waHj6XV6/jb7b2xfvZ3fT/29Q8cSPY9DwZmcHPWX2JAhQxq8tnHjRnQ6Hffffz8REREAREZGcv/996sp7zt3OnJqIYQQQgghhBA9hF2/mWHtD84Ee7kwb5h6l3BWcWWfz4Lo6Wz7zbQmEGCbrWB7F3JbeTp5sqifWnaopKaEzRc3NzvexWRgWl0pvZySKs5kNSy73ld0S7+ZLJvgTEjPDc7odXotKFheW87x3OONjls6ylrabH18VoecW1EUzueoTeBve+cA4/68iXs/PMqnR1LJKalqdJ/H1sZTVdv6zJ3Sqlr2nld//u7eF7XnJTjTPvXXGABPfzXj7+ilQooqGgb1WiM5r4x3diWjM13GxX8f+tB/MvvT2fxi2y/437n/kVnWeBm9Z488S2l1aZPH1el0BHm6aI9rLDXsTFPXVD1MHvLz7yDezt6MChgFwIWiC2SWtq/sYU55Dm8cfwMAHToem/JYh2U2BbgGdEm2pOhaDr07cnPVP5Q9PDzsnj99+jR5eWp9xRUrVti9NnGi+os8JSXFkVMLIYQQQgghhOgBKmvMHExWFwyDvZwZFuzZwh7Nu3VypLb98SEpbdZdzBaFw3XBGX93JwYFejQ73qJYtMwZd5M7YwPHOnR+29Jma86taXH8nKF9v7RZd/SbAWvmjLPBmRj/nt3bwjYouDd9b6Njpg/yx9tVLc22NSGbyhrHSpsdScnnmhd2suDFnfz12wT2XbisNYMHcDHpuWZ4EH9ZGc2uX89jfJQPoJaDfHPHhVafZ+fZXKrNFkDB6K5mzki/mfaL9o8m1F0N1NU6nQN9OWaLwr529q36w/rvMEa9gMfg5zAFfc3R3ANa6TFQe4VMD5vObyb9hnU3rGNGqPpezanI4bXjr7X6PLHZsZRUqwHoWRGzMBl6ZpnB3sju+pHR+PWjJW8cf4PyWrVn3o1DbyQ6ILpD5ib6LoeCMwaDWjcxPz/f7vndu3cDEBgYyPDh9r8kfH19AaisrHTk1EIIIYQQQggheoBDyflU1qjlx+YMDXT4rs45QwMJ9nIGYPvZHLKL5f+O3SEhs5iSqlpA7dPR0s814XICBVVq4GBKyBSHFwzHBI5hoLfaDPtozlGSipKaHT+nLuMKHO8b0VN1R7+Z9NJ07Y7/MYFjevxC8PSw6eiw9uVojMmgZ2FdabOyarPWvL09FEXh91+eJDmvzO75SD9X7pjWj3/9aBLH/7iI9+6cxA+m9SfK342nVo3CWFe+8Y3tF7iQ23TWhK1Np9UsH71TLlWK2rhe+s20n06nY2G/hQAomLXSZu0J7p5MLyK29H0MLtl2zwe7BXPT0Jt4Zd4r7LllD28tfIvbR95OP69+/HbSbzGhfp4+SviIk3knW3Uuu5JmUVLSrCPNDJupbTcV3G1OanEqX57/ElCzmh4Y90BHTU30YQ4FZ8LDwwE4fvy43fPffvstOp2OWbNmNdinqEj9BRIQ0L7GgEIIIYQQQggheg67kmZDg5oZ2TpGg57VE9XsGbNF4fPYNIePKdrOtqRZa/rNdFRJs3o6nY4bh9yoPf7i3BfNju/v76Zlkhy5mE9pXWCpL7HtUdJV/WZis2O17Z7cb6aen4sfI/xHAHAm/wy55Y0vtC8bFaJtb4hvX/kigH0XLmtl9CL9XPn9tSPY8tAcdv16Hn9aGcO8YUG4mAx2+wwP8eIns9TAY7XZwmNr41vsk1Rda2HbGbW1gLt3iva8lLRyTH1wBsDZWw2O7GxH36ont2y2ZjMZ/Hhw/IOsWbGGzd/bzB+n/ZF5UfNwM9lnuoV7hHONixpcUVD48/4/U2tp/rqlKIoWnDHpTXbBBOG4kf4j8XH2AeBA5gFqLG0rcfePuH9gVtQA+g9H/hBfF9+OnqLogxwKzsyaNQtFUXjttde0MmaHDx9m48aNACxevLjBPgkJah3HkJCQBq8JIYQQQgghhOhd6oMzeh3MHNwxN+HVB2cAPjl8CYul7zZ476kO2vQ2mdKK3ia2WQodEZwBWD5oOSa9emf51xe+ptpc3eRYnU6nlTarMSvtarbe0x1Isu0BJP1mmjIjzPr+25fReGPvGYMD8HRWM042J2S3qfeLrX/utmZ0/WbJcH4yayCDgzxazGr65fwhRPi6AurPdc3R9GbHH0y+TEmlunAfFGQNWEtwxjGjA0cT5KbeVGBwSwR9JZlFlSTmtC6bCeB4aiHHir7WHj8w/j7uGnUXQ32Htvg+mO48ncE+gwFIyE/go4SPmh1/Ov802eVqds6U0Cl4ODVfblK0jUFvYFrYNABKa0qJy41r9b7JRcmsS1oHgJeTF7ePvL1T5ij6HoeCM/fddx96vZ7k5GQGDhzIxIkTmTNnDrW1tfj6+nLzzTc32Gfbtm3odDrGjh3ryKmFEEIIIYQQQnSztIJyztctYo2L8sXbrWNKHkX6uTFriBroSc2vYH9S31to78ksFoVDKWogwMfN1GIfoaKqIk7kngBggPcAwj3CO2Qevi6+zI+aD0BBVQHbU7c3O96+70xOh8yhp6iutXDkovoz6Y5+Mya9SWuW3dPNDG+5NJGz0cCCutJmJZW17Dvf9mvM+ZxStteV0Av3cWVJdOtvQnZ1MvDX6639e5789jT5ZU0HHzedqi+XpVCuPwdIv5mOoNfpraXNdGaMHuoN5W0pjfi3zQcxeqnXPxe9JzcMXdHCHlYGnYHfT/69VorvteOvNduIXkqadb7WXD8a848T/8CiqCVe74y+E08nx/rviauHQ8GZ8ePH89xzz6HT6SgtLeXo0aNUVlZiMpl455138PS0fyMWFRXx7bffArBw4cLGDimEEEIIIYQQopfYdc7aq8F2Ybwj3DIpStv++NClDj22aN65nBIKy9VyLpP6+6HXN3/398HMg9qilG3WQke4cai1tNmac2uaHTttkD8mgzrX9pQm6sni0gq13k5d1W8muyyb1JJUAEYFjMLF6NLp5+wIowNH42lS16P2Ze7DbGk8K2ZpjDWYsr4dpc3e25usbf9oRn+MhrYtsc0dFsTyMWEAFJTX8NT6hEbHWSwKm0+rwRln1zxKawsB6TfTUWxLmxm94oHW952JvZhPbME6dDr1s/n9EbfganRt0/lHB4xm9bDVAFTUVvDUwaeavHbVB2d06JgXOa9N5xGtMz1surZtW66zOecLzrMheQMAPs4+3Dbitk6Zm+ibHArOAPzqV7/i2LFj/OEPf+CnP/0pf/zjH4mLi+OGG25oMHbHjh1MmjSJ2bNns2DBAkdPLYQQQgghhBCiG9lmJ3R0cGbByCD83J0A9a7x5u4qFx3rYFLb+s3YljSzveu4I0wOmaxl4uzP3E9aSdM9iNydjUzqr843Nb+ClMvlHTqX7nTAJntsWleVNMvufSXNAIx6I1PDpgJqVtepy6caHTd7aCDuTmo/mM0J2dSYLa0+R35ZNV8cVd+L7k4GVk+KbGGPxv3huhF4uqgBls9j09h3Ia/BmPj0IrKKKwEY0s96zZ0cMrld5xT2xgaOJcBVzdQ0eZwDfRWHkvMpr265b9Xzm+Nx8jkEgB4j3x95a7vm8Mvxv9TmsCNtB1svbW0w5lLxJc4XngfUAGT9eNGxAlwDtIy0hPwE8ioafiav9OaJN1FQA2o/jvkx7ib3Tp2j6FscDs4AjBo1ij/96U+89dZbPPHEEwwbNqzRcStXrmT79u1s376dgAC5iAghhBBCCCFEb1VjtrC3rhSQr5uJmHDvDj2+s9HAjePVRflqs0VbCBWd72CybeP55gMBiqJodxc7G5yZEDyhQ+ei1+m5cYg1e2bt+bXNjp9tW9rsbN8pbdYt/WZsgzPBvSc4A/YZXE2VJnIxGZg3XO03UlheYxcAa8lHBy9qmUw3T4rCy6V9JR2DPF347VJrabLfrz1JZY19ps+m01natptXirYt/WY6hkFv0MonoqvF6H6GarOlxffDgaTLHMnbgs6oBoGXDVxKoFv7blLwdPLkt5N/qz1++uDTlFbb972xLesoJc06l+31Y3/G/mbHns0/y6aLmwDwc/Hj5mENW3wI0ZwOCc4IIYQQQgghhLi6HL1YQGmVemfxrCGBGFoofdUeN9uUNvvkcGqfKlPVUymKwqFkNRDg6WJkRKhXs+PPF54np1wNgkwMntgppa9WDl6JQadmOHyZ+CW1lqbvaLfvO9P6vhE9mW2/mTBvFyL92lY2qb2OZKnBGaPOyJjAMV1yzo4yI9y6uLono+nSRMtGhWrb6+Ozmhxnq6rWzPv7LwKg16klzRxx66Qoxkf5AJCUV8abOy7YvV7fb0anU8iqVrOAPEweDPNr/MZo0Xb2pc1OAvZlO6+kKAovbD6Dk5/1vXVH9A8dmsOifouYFT4LgJyKHF499qrd67bZNNdESnCmM9leP2wzQxvzxvE3tO2fjPoJbqau6Qcm+g4JzgghhBBCCCGEaDPbhe+OLmlWb3CQB5P6+wJq8+2jlwo65TzC6kJuKXmlagm5Sf39Wgy62WYl2C5odaQgtyBmRVgXLZtr0jw8xJMgT2cA9iddbpCF0Bt1R7+ZvIo8UopTAIgOiO51C44h7iEM9hkMwMm8kxRVFTU6bu6wQFxM6tLYplNZ1LaitNm6E5nkllQBsDg6hEg/x743er2Op1aNwlj3WXtzxwXO56hZE0m5pSTWbcf0q6KgSg3SjQ+WfjMdaULwBHyd1d81Ro8zoKtuNri778JljubuR++sBnAmBU/SSmG1l06n47Gpj+FiUAPcH5/5mJN5aqAoryKP4znHARjoPZD+3v0dOpdo3tjAsVppsn3p+7Sealc6dfkU21LVPkBBrkHcNPSmLpuj6Ds6PDiTkpLCkSNH2L17N7t27Wr2nxBCCCGEEEKI3mlXonXhatbQzitbfYtN9szHh1I77TxCdcCBfjOdFZwB+N6Q72nbnyd+3uQ4nU6nBQsraywcTslvcmxvYVteSUqatV59aSKLYmF/ZuOlidycjMwbppY2u1xWzaEW3i+KovDPPcna45/MGtAhcx0e4sVPZw8E1DKOj62NR1EUNp3O1sb0j8jUticFS0mzjmTUG7VSYTp9DUaPsyTnlXHxclmDsYqi8MKmszj57daeuyP6jg6ZR7hHOPeNvU89Dwp/2v8nai217EzdqfU0kZJmnc9kMDElZAoABVUFJFxOaHScXdbM6J90Suao6Ps6JDhz9uxZ7rjjDnx9fRk0aBBTpkxh7ty5zJs3r8l/11wjFxMhhBBCCCGE6I1yS6o4mV4MQHSYF0GenbcgsWxUqNYwe11cBsWVNZ12LoFW0gxgSguBgPKacmKzYwEIcw9jgFfHLFQ3Zkb4DILc1EX03Wm7tVJqjZkzzLbvTO8vbdYd/WZis2K17YkhvTM4Mz18urbdXLbVUpvSZhtaKG22P+kyCZnqtW9MpA/jo3wdnKXVA9cM0UrWHUzO5/PYNDadss6n1um8ti39Zjreon6LtG2jZ31ps4bXj53ncjmefRqjexIA/bz6aZl9HeH2kbcz1HcoAGfyz/DfhP9q2RkgJc26il1pxPSGpRHjcuPYlaYmHoS4h9j1RhOiLRwOznz55ZeMHz+eDz/8kKKiIhRFafU/IYQQQgghhBC9z+7Ezi9pVs/VycD1Y8MBNRPi6+MZnXq+q5miKBxMVrM03JwMRIc132/mSPYRaixqsGxG+IxOLbdl1Bu5fvD1AJgVM1+d/6rJsTMHB1Bfjc02w6s36rZ+M3WZM3qdnrGBY7vknB1tQvAErUTU3vS9Ta5DXTM8CCejujy28VQWFkvT61Xv7rbJmpk5oEPf865OBv56/Sjt8V+/TeBYaiEAQ4M9OF1wDJB+M51lUugkvJzUa57RIwF0NQ1KmymKwoubz9n1mvnBiB+g13VcYSKT3sTj0x5Hh/reev346xzIOACoJR6jA6I77FyiaS31nXn9+Ova9t2j78bJ4NQl8xJ9j0NXj9TUVG6//XYqKioICwvj5Zdf5u233wbUVOKtW7fy+eef89vf/pawsDAAZs6cyZYtW9i2bVtzhxZCCCGEEEII0UN1Rb8ZW7dMjtS2Pzl8qdPPd7W6eLmc7GK1l8aEfr6YDM0vGdjeTdyZJc3qrRqySluwXJO4psk+AD5uToyN9AHgXHYpGYUVnT63ztId/WYKKgs4X6hmaYzwG4GHk0enn7MzOBuctQyT3IpczhWca3Sch7NRu47lllQR20Rvq6TcUraeUTO2wrxdWBoT0uFznjM0kBVj1PWzoooa6uNJU4bWkl8p/WY6k0lvspY2M1RjcE9k34XLVNVa+1ZtTcghPisVo/cJALydvFk+aHmHz2V04GhuHnYzABW1FVRb1D5g8yLndWggSDQt3COcAd5qNuiJ3BN2fauOZh9lX8Y+bdz1g67vjimKPsKhT/Qrr7xCeXk5np6eHDx4kAceeIBp06Zpr8+bN49Vq1bx1FNPkZiYyC233MLevXt59913mTNnjsOTF0IIIYQQQgjRtcwWRSv14uFsZHy/jivr05ToMG9GR3gDcDK9mJPpjTf3Fu1XVWtmzdE07XFrymfVl4oy6oxaff7OFO4RzrQwdc0hvTSdg5kHmxw7Z2iQtt1YaaLeoqv7zdRYavj6wtfa497ab6ZeS3e/11s2yhpoWR+f2eiY9/Zas2bunNEfYwvBy/b6/XUj8HKxD774+lv7bUm/mc6zsN9CbdvkGU95tZnYFDVYZ7GoWTMm3/3odGrA5qZhN+FmcuuUuTww/gECXe1vfpB+M13Ltm+V7e8b26yZe0bfg8lg6vK5ib7Dod8kW7ZsQafTcd9992mZMU1xdXXlww8/ZNy4cXzyySesWbPGkVMLIYQQQgghhOgGJ9OLKCivK2U12L/F7IqOcvMkyZ6pV1Vr5oGPj7HytT08vT6B/RcuU2NuPIukOYqicCQln9+tjWfyk1t5dZu1p8WUAX7N7nup+BKXStSfw9igsV2WXbFqyCpt+4vEL5ocZ9d3plcHZ5ruN2O2mHl83+PcvO5m/nbob+zL2Ee1ubrN51AUhVN5p3jm0DMs+GwBzx95Xnutt/abqTczfKa23VzfmfkjgjEZ1KykjScbljYrKKvm81g1eOnuZODmSVGdMFtVkKcLv106Qnsc6u1CRuVJ7bH0m+k8U0On4mnyBMDomQC6Wu36sel0Fqez8nDyURfpjXojtw6/tdPm4unkyW8n/9b62OQpgbku1lhw91DmIQ5lHQIgyjOqUzKnxNXFob+iU1JSAJg+3dpkzTbFtra21v5kej0PPPAAiqLw3nvvOXJqIYQQQgghhBDdwHahe3YXlDSrt2JMGK4mAwBfHcugvLq2hT36rs9j0/j6RAYn0op4a1cSt75zgPF/3sx9/43lsyOp5JRUNrt/Sl4ZL24+x5zndvC9f+zno4OXKKqo0V4fF+WjlQVrim0WQleUNKt3TeQ1+Dqr2VpbL22loLLxElSjwr3xdVPvZt6TmNeu4FVHKquqJakYatswj5b6zWy5tIUvEr/g9OXTfJjwIfdsvoeZn8zkgW0P8Pm5z8kqa765fUZpBu/EvcPKr1Zyy7e38N+E/2qlswAG+wxmaujUNnyVPU+UZxQRHhEAHM05SllNWaPjvFxMzBwcAEBmUSXH0wrtXv/o0CWtvNxNEyPxdm3fnfKVtZUcyznWYhDtlkmRXDc6FKNexy/nD+Zw9mFA+s10NieDE3Mi1Uo/OkMlBrfz7DyXi8Wi8NLmREzex9AZywFY2n8pQW5BzR3OYQv7LWT10NUYdAbuGSMZGl1tYvBEnA3OgFrGU1EUu6yZe8fcKyUGhcMcegeVlam/1CIjrXcwublZ0/mKiorw97e/syM6Wm1cdeLECUdOLYQQQgghhBCiG9gFZ4Z0XXDG08XEdaND+Sw2jZKqWr6Ny+SmiZEt79gHfXMio8FzJVW1rI/PYn28uiA/OsKbucOCuGZ4EKPDvSmqqGFdfCZrj6Zx9FJhg/1dTQaWxIRww7hwZgwOwKBvvreJbRZCfemXrmAymFgxaAXvn35fK8F1R/QdDcYZ9DpmDQnk6xMZlFTVcjy1kEn9m88G6iybTmXx+y9PklNiZHvRIT64a2qrFvdb6jezMXljg30qaivYnrqd7anbARjmO4xZEbOYHTGbUQGjqKitYPPFzXxz4RuOZB9psL+T3om5kXNZPmg5M8JnYNL37sVgnU7HjPAZfHr2U2ottRzKPMS8qHmNjl06KpTtZ9Xr24b4TMZHqUHA6loL7+9LqTse/HjGgHbNZX/Gfv60/0+kl6Yz2Gcw/1z0T/xdGy9Vp9frePXWcZgtCpdKUngyQfrNdJWF/RayLmkdACaveM5kDue9vcmczS7CbaC1z9YPRv6g0+ei0+n4w7Q/8OiUR+Xn3g1cjC5MDJ7I3oy95JTn8GHChxzNOQrAAO8BLBuwrJtnKPoChz7Z3t7e5OfnU1lpvSvHNhhz4cKFBsGZ4uJiAPLy8hw5tRBCCCGEEEKILna5tIqjdc2yBwW6E+nXObX2m3LL5Cg+qyst9Onh1KsyOJNdXMnBZHWhtr+/Gw8uGMq2MznsPJdrl/0Sl1ZEXFoRr2xNxM/diZLKGmrM9qWadDqYMSiAVePDWRwdgrtz65YIKmorOJB5AIAA14Auv5N/1dBVvH/6fUAtbfbDkT9sELgAtbn613WBrJ1nc7s8OHO5tIonvjltF0yLSyvmjvcO8cFdk/F0aT7w0Vy/mdLqUnal7QLAz8WPRyc/yu703exJ32OX/XK24CxnC87yz/h/4uXkRZW5iipzVYNzTQiewPKBy1nYfyFeTl7t+np7qpnhM/n07KeAmvHVVHBm0chgfqfXUWtRWB+fxe+WjUCn0/FtfAY5JVXamCj/tl33SqpLeOHIC6xJtJb3P194np9s+gnvLX4PX5fG+3bpdDqMBh2Hsw5rz0lZq843PWw6bkY3ymvLMXqehkwzT284g8H9HAZnNXg3KWQSI/xHtHCkjiOBme4zI3yGlin6wpEXtOfvG3MfBr2hu6Yl+hCHypoNG6b+AZaUlKQ95+npSb9+/QDYtGlTg322bNkCgI+PjyOnFkIIIYQQQgjRxbadyUGpW99fODKk+cGdYHyUD0OD1d4mRy4WkJhd0uVz6G7fxmVqP4MVY8O5flw4r9w6jtjfL+Dze6fx83mDGBlqv7ieX1ZtF5gZHuLJo0uHs/+38/nwJ1NYNT6i1YEZgAMZB7QF/jkRc9DruqbvUL2B3gMZHzQegKSiJI7nHm903KyhAdp2V/adURSFr09ksPClXXaBGYNO/RkcTy3kzn8dprSq+dJ8zfWb2Z66nWqLWhprcf/FLBmwhCdnPsn21dv5+NqP+dmYnxHjH2O3T3F1sV1gpr9Xf+4fdz8bb9zIv5f8mxuH3tjnAjMAk0Mma4vb9aWJGuPj5sS0Qer3Ob2wgvj0IhRF4Z+7k7UxP5k1sE3n3pG6g+u/vN4uMFM/l/OF5/nppp9SWFnY7DHqS5qB9JvpCi5GF+ZE1Jc2q8DgfgGzRcHJ35o188ORP+yu6YkuZlu206yYAbXk46L+i7prSqKPcegvqGnTpgFw4MABu+evu+46FEXhueeeY9u2bdrzn3/+OS+//LKaVjqj69KehRBCCCGEEEI4bktCtra9cGTn1tpvjE6ns2vE/fnRtC6fQ3f7Js662L98dKi2bTTomdjfj18vHs76X87iwKPzeWbVKBaNDMbTxUiotws/nTWA9Q/MYuODs7lnziBCvF3aNYcdaTu07XmRjWchdLYbh96obX91/qtGxwR5umiBqvj0IvJKG2aMdLTs4kp++p9YHvj4GPllavDEx83E898bxcOjzPjUlTOLvVjAj/91uMneSS31m9mQvEHbXjpgqbat1+mJCYjhvrH38fF1H7N99Xb+OuOvLO6/GG9nbwJcA7h1+K18fO3HfH3919w9+m7CPcI79HvQ07iZ3LRgXnppOpdKLjU5dtko62dqfXwWB5LyOZWhVoAZHeHNxH6NZ7lcqaCygN/s+g33b7ufnIocANxN7vxh6h/4cuWXBLmq18+zBWe5e/PdFFUVNXocRVG0zBnpN9N1FvZfqG0bPU+id87A6H4egH5e/ZgdMbu7pia62ACvAYS5h9k99/OxP+/ymxJE3+XQO2nZsmUoisIXX3yB2WzWnv/1r3+Nm5sbpaWlLFy4kMDAQLy8vLj55pupqKhAr9fz61//2uHJCyGEEEIIIYToGpU1ZnadU8tT+7s7MTaydYuUHe36sWEY6/qhfHUsA7Ol8bvg+6LU/HKO1fWLGR7iyZBgzybHhni7cMvkKN7+4UTin1jM/kfn89i1IxkZ5lhmhNliZkfqDgBcja5MCZ3i0PHaa0HUAlyNasBiU8qmRkt1AcwZZu2LtCex88qrK4rC/w6nsuDFnXZBzGWjQtj8qzmsHBNKuDv8+84JWr+ZQyn53PXvI1RUmxscr7l+M4WVhezP2A9AiHsIYwLHNDmvANcAVg5eyfNznmfPLXvYvno7v5vyO2ICYhotBddX2d79vid9T5PjFo0Mpr7d0saTmby7x1op5q6ZA1r8nimKwsaUjVz/1fWsT16vPT8zfCZfrvyS1cNW08+rH+8ufpcAVzWzKyE/gXs230NxdXGD4yUXJWtl6qTfTNeZGT5Tu74YPU/h5L9be+32EbfLwvxVpL5vVb3hfsO5JuqabpyR6GscuprMnTuXxx9/nB/96Eekp6drz0dFRfHZZ5/h7e2NoihcvnyZ0tJSFEXB2dmZd955h6lTpzo8eSGEEEIIIYQQXWPfhTwqatRF5PkjglpsGN9Z/D2cmTNUXXDPKq6068vR162Ly9S2l48Ja2Zk54nPi9cWi6eFTsPF2L7sG0e5mdxY2E+9u72kpkQLGF2p/r0CnVfaLDW/nB++d4hH1sRRUqlmwgR4OPOP28fzxvcnEOjprI2NDvPiw7um4OmiLrLvT7rMT/9zhMoa+wBNc/1mtlzaQq2inmdJ/yWyUNwKM8Ksi6t70/c2Oc7fw1n7fqdcLmdLgpr1EurtYpdV05jc8lwe3P4gv975a+0z4uXkxZMzn+SN+W8Q4m4tBdnfuz/vLn4XPxe1D9Kpy6f42eafUVpdandM6TfTPVyNrswMnwmA3liGyfsYoP48Vwxa0Z1TE91g+aDl6HV6DDoDD45/UK65okM59G7S6XQ8/vjj/OUvfyEqKsrutaVLl3L+/HnefPNNfvGLX3DvvffywgsvcP78ee68805HTiuEEEIIIYQQoottPp2jbS8YEdyNM4EbxlvLMK09lt7MyL7Ftn/J8tHdE5yxDYLMjZzbLXOod93A67TtdUnrGh0zPsoXj7p+OrvO5WLpwEwri0XhP/tTWPzyLnbbZOWsGh/OlodmsySm8cX8URHefHDXFDzr5rXnfB73fBBrF6Bprt/MxuSN2vaSAUs65Gvp64b6DiXQVQ3UHc463GSmFcDSRoIwd0zvj8nQ+BKaoih8ef5LVn61km2p1tL+C6IW8NX1X7Fi0IpGM24Geg/k3UXv4uusZiHG5cXxsy0/o6ymTBsj/Wa6z6J+DXuK3DT0JtxMbt0wG9GdxgWN46NrP+Kjaz+yy6IRoiN0aqjPz8+Pe+65h1deeYU33niDX/3qV4SH9+1apkIIIYQQQgjR11gsClvrSjU5G/XMHBLQwh6da8GIYG1he0N8ZqNlofqa8zmlnM5Uyx6NifQhyr97FgjrgzM6dN3ed2FyyGStd8eetD0UVBY0GONk1DO9rsn75bJq4tMb7+3RHi9sPssfvzpFed37L9TbhX/9aBIvrh6Lj5tTs/uOjfTh3z+ejLuTAVCzeu7771Gqas3N9pvJLc/lUNYhAKI8oxjpN7LDvp6+zLY0UaW5kiNZR5ocuzg6GNtYipuTgVsnRTU5/j+n/8Mf9v6BkuoSAPxc/Hhhzgu8NO8lrXRZUwb7DuadRe/g7ewNwPHc49y35T7Ka8ql30w3mxUxC2eDNevNqDNy6/Bbu3FGojtF+0cz0l+ut6LjtTk4k52dzSOPPMKoUaPw8vLC3d2dIUOGcPfdd5OQkNAZcxRCCCGEEEII0Y3i0ovIKVHvNJ85OAA3p+7te+BiMrB0lFoiqKzazKbTWd06n66wLs42a6b58kqd5VLxJS4UXQBgTOAY/F39W9ijcxn0BpYNXAZArVLLxpSNjY6bNzxI2+6oTKvSqlr+tTdFe3zblCg2/Wo284YFNb3TFSb08+XfP56MW12AZtuZHH7+32McuZjfZL+ZTRc3oaBm/ywZsOSq6hvjqFnhs7TtpjKtAII8XZjUz097fNOECLzdTI2OrTHX8N7J97TH1w28jq9WfsWi/g2zLpoyB587wAABAABJREFUzG8Y7yx8By8ntR/U0Zyj/GLbLzidf1r6zXQjd5M708Oma48XD1hMsHv3Zo0KIfqeNgVnDhw4QHR0NC+88AKnT5+mtLSUiooKkpKSePfddxk7diwfffRRZ81V8+abbzJ69Gi8vLzw8vJi2rRpbNiwQXtdURSeeOIJwsLCcHV1Ze7cuZw6dcruGFVVVdx///0EBATg7u7OihUrSEtL6/S5CyGEEEIIcbWorrV09xREB9ly2trgfOHIti1O1ZhrOno6ANwwLkLb/rKPlzZTFEUraabTwXXdVNJse+p2bXte1LxumcOVWlPabNmoUFxM6vLH2mPpDfq7tMfXxzO0jJnbpkTx1A2j8HRpfAG/OZP6+/HenZO0+W1JyObn/z2qvd5cSbNlA5a1Z+pXrTmRc7QMlc0XN1NU1XQW1Z0z+gPg42birpkDmxy3PXW7FkBZ1G8RT896Gh8XnzbPbYT/CN5e9DaeJk9ALb127+Z7tdel30z3uHX4rejQ4Wp05ccxP+7u6Qgh+qBWB2eKi4v53ve+R35+PoqioCgK/v7+BAerf5grikJNTQ133XVXp2fQRERE8Mwzz3DkyBGOHDnCNddcw8qVK7UAzLPPPsuLL77Ia6+9xuHDhwkJCWHhwoWUlJRox3jwwQdZu3Ytn3zyCXv27KG0tJTrrrsOs7nvp8MLIYQQQgjRmcwWhTveO0TME9/x529OU15d291TEg7akmANzlwzonWZAYqi8MjOR5j80WSe2PeEVvKno0wZ4EeYt9qMfldiHrklTfeQ6O1OZxZzIVftQzG5vx8hdV93V+tJ/WbqDfMbxlDfoQDE5cZxsfhigzHeriatmXtRRQ0bTzqeafXp4Uva9m2Tmy551RpTB/rz3h2TcDaqSzQF5TV2r9XLKM3geO5xAIb4DmGQzyCHznu1cTY4s3zgcgCqzFXNZs8sGxXK+gdm8e0Ds5otIfhF4hfa9veGfs+h+UX7R/PWwrfwMHkAUFhVqL0m/Wa6x7SwaXyx4gvWrFijXWeEEKIjtTo4895775GRkYFOp+P666/n/Pnz5ObmkpmZSWZmJvfffz8A1dXVvPDCC502YYDly5ezbNkyhg4dytChQ3nyySfx8PDgwIEDKIrCyy+/zGOPPcaqVauIiYnh/fffp7y8XMvqKSoq4t133+WFF15gwYIFjBs3jg8//JD4+Hi2bNnSqXMXQgghhBCirzuQdJmd53KprrXw3t5klry8m30X8lreUfRIqfnlnMlSAytjI30I8mxdYOBM/hk2pGyg1lLLmsQ1XP/V9exM3dlh89Lrdawcp/Y0NVusmSV90TcnMrXt5WO6J2umsLKQYznHAOjn1Y8BXgO6ZR6NaU32zC02PUM+sQmstMfpjGJOpKlZF9FhXsSEezt0PIDpgwP45x0TcTJal2mu7DdjW7Ztaf+lDp/zanTjkBu17TWJa1AUpcmxI8O8CPdxbfL19NJ09mXsAyDcI5wpoVMcnt+owFG8ueBN3IzWgJD0m+leg30HE+kZ2d3TEEL0Ua0uWLl+/XoApk6dypo1a+zqmgYFBfH3v/+d0tJS/vWvf2lju4LZbOazzz6jrKyMadOmkZycTFZWFosWWet7Ojs7M2fOHPbt28c999xDbGwsNTU1dmPCwsKIiYlh3759LF68uNFzVVVVUVVlvRuruFhtxlhTU0NNTeek6gsheof6a4BcC4QQcj0QAj6PTbV7fCm/nNveOcgtkyJ4ZNFQPF2ujrr5feV6sPGkNehxzbCAVn89XyV+Zfc4pzyHX2z7BUv7L+X/xv8fvi6+Ds9t+ahg3tyh9kD54mgaP5gS0cIevY9a0kwt22bQ61gwvPU/g460/dJ2zIpaaWJ22Gxqa3tORtyiyEW8FPsSCgrrLqzjpyN/2qAXy9hwDwYGuJGUV86BpHwSswrp7+/ervN9dDBF275pQnirfh6tuR5M7e/DG7eO4WcfHafGrLBgRJDd93lDkrWc+/yI+b3+2tId+nn0Y3TAaOLy4kgsSOR41nFiAmLadaw1Z9do/X9WDlyJudaMGcersUT7RvPq3Ff5xY5fUFFbwezw2ShmpdNKRIru0Vf+RhBCNK61n+1W/6/o5MmT6HQ6fv7znzfZcO6Xv/wl//rXv8jOzuby5cv4+3dec8D4+HimTZtGZWUlHh4erF27lpEjR7Jvn3rXQn25tXrBwcFcvKimN2dlZeHk5ISvr2+DMVlZTac3P/300/zpT39q8Pz27dtxc2s6zVUIcfXYvHlzd09BCNFDyPVAXK2qzbD+hAHQ4WJQCHeDCyXq/x8+OZzGxhOprB5oIdq36buV+5refj343yk99UUXnHLPsH79mRb3MStmvipWgzMGDPQ39udCrRpE2ZCygV0Xd7HcdTnRpmiHG5pHuBtIK9NxMqOY9z5fT0gf+69ZSgmkF6r/dR/iaebgzu6p9vBp2afatnOaM+uzuu6mzNYYaBzIhdoLpJWm8dY3bxFlbFhqbJS7jqQ8AwDP/G83K/q1vS9WtRnWxKrXOJNewSUrnvXr41u9f2uuBw/HQEqJjhhLEuvXJwGQa87lTIn62Qs3hBO/O554Wn9eYTWoahBxxAHw9x1/5wa3G9p8DIti4dNi9TOhR49HigfrL3XsZ+Je13tJqk0iJj+mS2+CFl2rt/+NIIRoXHl5eavGtTo4k5+vNjgbPnx4k2NGjBihbRcUFHRqcGbYsGEcP36cwsJC1qxZwx133MHOndYU+Sv/wFcUpcU/+lsa8+ijj/LQQw9pj4uLi4mMjGTevHmd+rUKIXq+mpoaNm/ezMKFCzGZ2t4IVAjRd8j1QFztvonLpOqQumC4fGwEf10xko8Op/LcpkTKq80UVut4+4yB68eE8tiy4fi49d3PSV+4HhRV1PDQwR2AQqSvKz++cWargin7MvZRuqMUgNkRs3l+1vOsS17H87HPU1JTQplSxiflnzAvYh6PTnqUANeAds8x2+ciT204C0Ch9xB+vHBIu4/VE/11/RlALcN15zWjWDY+vMvnUG2u5qk1TwHg4+zD3dfdjVHfszLgzElmHj/wOAD5IfncO/neBmOmlFWz/rmd1JgVjhe58Ori2ZgMra72DsCXxzOoOHQSgOvGhHPjitZlXTh6PXg7/m3qYzE3j76ZZSOWtfkYQjWvdh6bvthEWW0Zpy2neXnhy7ib2pZFtTt9N8U71Woqs8JnccucWzpjqqIP6wt/IwghmlZfcaslrf5rqrq6Gp1Oh4tL0/WFbS8m1dXVrT10uzg5OTF48GAAJk6cyOHDh/n73//Ob37zG0DNjgkNDdXG5+TkaNk0ISEhVFdXU1BQYJc9k5OTw/Tp05s8p7OzM87Ozg2eN5lMciEVQgByPRBCWMn1QFytvo6zZqLfOCESZ2cnfjRzEAtGhvK7tfHsTlR7z3x5IpM9F/L5y8polo4KbepwfUJvvh7sPZWD2aJmOS0cGYKTk1Or9ttwyVp+aeXglTg5ObFq2CpmRc7irwf+yrbUbQBsT9tObE4sj0x6hBWDVrQri+b68RE8s/EsFkV9//16yQj0eseycXoKs0Vhw8lsAJwMepaODu+W99LBnIOU16p3gM6OmI2rc9N9OLrL4oGLefrw01SaK9l0aROPTn0UJ4P9+zXEx8SikSF8G5/J5bJqdp3PZ0lM264/n8Vay/x9f0q/Nv882nM9UBSF7y59pz1eOmhpr72m9AQmk4llA5fx2bnPqKitYGvaVm4cemPLO9r4MulLbft7w74nPw/Rbr35bwQhRNNa+7lu2y0iPZiiKFRVVTFgwABCQkLs0gKrq6vZuXOnFniZMGECJpPJbkxmZiYnT55sNjgjhBBCCCGEaFpuSZUWfAn3cWVyfz/ttUg/N/7z48k8+73RWs+ZvNIqfvbfo/zsw1hyS6oaPaboXptPZ2vbC0cGNzPSqrymnG2X1OCLl5MXsyJmaa8FugXy8ryXeW7Oc/i5qO+P4upifr/39/xs68/ILM1s9JjNCfJ0YdaQQADSCys4nJLf5mP0VIeS88mp+2zMGRaIt2v3LODtSN2hbc+LnNctc2iJu8mda6KuAdT31O703Y2Ou3mStbH3x4dSGx3TlAu5pRyqe38NDvJgQj/H+ya1xrmCcyQXJQMwPmg8Ie4hXXLevuzGIdZgzJrENW3aN7c8l11puwAIcg1iZvjMDp2bEEKIq0evDM787ne/Y/fu3aSkpBAfH89jjz3Gjh07+P73v49Op+PBBx/kqaeeYu3atZw8eZI777wTNzc3brvtNgC8vb256667ePjhh9m6dSvHjh3j9ttvZ9SoUSxYsKCbvzohhBBCCCF6p29OZGhZFivHhjXIXtDpdKyeGMmWh+awYIR1oX/DySwWvbST8zklXTpf0bzqWgs7z+YC4O1qYmL/1i1Eb720lYraCgAW91/cIHtBp9OxpP8Svlz5JcsGWEsz7U3fyw1f38DJvJNtnusqm1Jfa4+lt3n/nuqbOGuWxvIxYd0yB0VR2J66HQCT3sT0sJ57Q+PyQcu17XUX1jU6ZubgACJ81cyfXYm5pBdWtPr4nx62BnNumRTpcL+k1tqYslHbXjpgaZecs68b6T+S4X5q2f74vHjO5p9t9b5fXfgKs2IG4Poh1/e4En9CCCF6jzb/Bvn973+Pj4+Pw+N0Oh3vvvtuW08PQHZ2Nj/4wQ/IzMzE29ub0aNHs3HjRhYuXAjAI488QkVFBffddx8FBQVMmTKFTZs24enpqR3jpZdewmg0snr1aioqKpg/fz7//ve/MRgM7ZqTEEIIIYQQVzvbRfEbxjXdFyPYy4V3fjiBb+IyeeLrU+SXVVNQXsNLmxN5/fvju2KqohUOJedTUlULwLxhga3uzfHNhW+0bdvF8iv5uvjyt9l/Y0n/JfzlwF/IrcilrKaMl4++zD8X/bNNc104Mhg3JwPl1Wa+jc/kiRXRuJh69//taswWNsSrmUSuJgMLRgR1yzwS8hPIKc8BYEroFNxMbt0yj9aYGjoVfxd/LldeZmfaToqqivB29rYbo9fruHliJC9sPoeiwP8Op/KrhUNbPHZ1rYU1sWkAmAw6Vo2P6JSv4UqKorAhWS0TqNfpWdhvYZect6/T6XTcOORGnjz4JABfJH7Bo1MebXE/i2JhzTlrps0Ng2/otDkKIYTo+9ocnPnqq6+afb3+zpGWxgHtDs60tJ9Op+OJJ57giSeeaHKMi4sLr776Kq+++mq75iCEEEIIIYSwOp9TQnx6EQAx4V4MCfZsdrxOp2PFmDBmDPJn8cu7yCut5rtTWeSWVBHo2bDPo+h6m09b+wctaGVJs5zyHA5mHQQg3COcsYFjW9xnXtQ8JoRM4OZvbiatNI2DmQe5WHyRfl79Wj1XNycjS2JC+OJoOiWVtWxNyOHa0b27l9He83kUlNcAMH9EEG5O3XN3fn3WDPTckmb1jHojSwcs5cOED6mx1PBdynesHra6wbjvTYzgpS3nsCjw2ZFUHpg/BEMLfYq2JGRzuUztrbsoOgQ/99b1X3LUybyTpJeqge8pIVPwd/XvkvNeDZYNXMYLR16g0lzJN0nf8KsJv8LF2HSfZYDDWYdJK1WDdNNCpxHh2TVBOiGEEH1Tm8qaKYrSYf+EEEIIIYQQfYd91kzrF6v8PZxZPVHtAVFrUfjfkbb1gBCdQ1EUtiSo2RImg445QwNbtd+G5A1YFAsA1w28rtVln7ycvOwW0T8/93kbZwyrbN53faG02TcnrP13uqukGdj3m5kTMafb5tFattla3yZ92+iYUG9X5g1TM5EyiirZlZjb4nE/PnRJ2751UpSDs2y9DSkbtG0padaxvJy8WNR/EQAl1SVsubSlxX1ss2ZuHHpjMyOFEEKIlrX61pvk5OTOnIcQQgghhBCil7JYFL48pvbGMOjVjJi2uHVyFG/uvICiqAugP5szqEG/GtG1EjJLtF4cUwf64+nSukb0rS1p1piVg1fy6rFXqbHU8OX5L7l/3P0N+tU0Z9ogf4K9nMkurmLH2Rzyy6q7LLuho1XWmNl0Ss1c8nQ2tjo41tEySzM5k38GgGj/aILdW5dB1Z1G+I1goPdAkoqSOJpzlLSStEazG26ZHMXWM2oA8pNDl7RgTWNS88vZcz4PgEg/V6YP6prsFYti4bvk7wA1K+iaqGu65LxXkxuH3MjXF74G1MDLdQOva3JsYWWhFsDxdfbt8ZlkQggher5WB2f69Wt9SrkQQgghhBDi6nEoJV9byJ81JKDNZcki/dyYPSSQnedySSuoYFdiLnObWSgVnW9LQra2vbCVJc3OFZzjbIHaVHt0wOg2lSUD8HPxY0G/BWxI3kBhVSGbL27m2oHXtnp/g17HyrHhvL0riVqLwrq4DH44rX+b5tBT7DyXq/X7WRwT0m39c3ak7dC250bO7ZY5tJVOp2P5oOX8/ejfAViXtI57x9zbYNy8YYEEeTqTU1LF1oQcckoqCfJsvKTVZ0dSqS8AcvPEyC4LHh/NPkpOhRpAmhk+s0H/HOG4cUHjGOA9gOSiZI5kHyGlKIX+3v0bHftN0jfUWNRSgysGrWhT8FgIIYRoTJvKmgkhhBBCCCHElb60K2kW3q5j3DbFWibovwcvNTNSdIXNp63BmfkjWhecWZe0TttuS1DF1uqh1tJm/zv7vzbvb/v+682lzb45kaFtd2dJs+2Xek+/GVvXDrC+/75N+rbR0upGg56bJqoZNbUWhTWxjb9fas0W/ndE7TFi0Ou4qa4MY1fYmLJR217aX0qadQadTseNQ6zlyb44/0Wj4xRFsStptmroqk6fmxBCiL5PgjNCCCGEEEKIdqusMfNtvNobw93JwKKRIe06zvzhQQR7qRk3287kkFlU0WFzFG2TWVRBfHoRANFhXoT7uLa4j9li1vp7GHXGdvfGmBA8gYHeAwE4mnOUC4UX2rT/iFAvhod4AnDsUiHJeWXtmkdHiUsr5KFPjzP/hR28uOksRRU1Le5TXl3L1rp+P37uTl1WQutKJdUlHM4+DECYexhDfYd2yzzaI9QjlEkhkwBIKU7hZN7JRsettgm0fHr4UqNBnF2JuWQVVwIwb1gQwV7NN4xvTGJBIn85+BdeLn6Zl4+9TH5lfov71Fpq2ZSyCQAXg0uvyVzqjZYPWo5RrxaW+er8V9SYG35OT+Se4EKRej0aHzReu04JIYQQjpDgjBBCCCGEEKLdtibkUFKpll9aEhOKq1P7yi8ZDXpuqWuybbYofHo4tcPmKNqmPjAAsKCVWTOHsw+TU24tv+Tr4tuuc+t0Om4aepP2+LNzn7X5GN2dPVNrtrA+PpPvvbmPFa/t5Ytj6VzILeOVbeeZ+bdtvLI1kZLKpoM0m09nU1FjBmBpTAgmQ/f8t31vxl5qLepne27kXHS63tUHyrZ3yDdJ3zQ6pp+/OzMGq8GvlMvl7E+63GDMx4es16JbJrU+a8aiWNiRuoOfbPoJq75exdoLa8mz5PGfhP+wZM0S/n707xRWFja5/8HMgxRUFQAwJ3IObia3Vp9btI2fix/XRKr9fPIr8+3K+dVbk2jNmrlx6I0NXhdCCCHaQ4IzQgghhBBCiHZb2wElzerdMjmS+lYOnxxKpdZsceh4on3a029m3QWbkmaD2lfSrN7yQctxNqhZVF+f/5qK2rZlUa0cG059HOHLY+mNZkN0hqLyGt7aeYE5z+3gvv8e5cjFggZjSipreXHzOWY9u503dpynrK6vjK1vTmRq291Z0mxH6g5tuzdmbSzst1B7H21M3qj1CrlSfVAYaBAUzimuZNsZNegY7OXM3GGBLZ63rKaM/yb8l+Vrl3P/tvs5mHmwwZiK2gr+Gf9PlnyxhNePv05xdXGDMRuSN2jbUtKs89kGXGwDMQCl1aV8l/IdAJ4mTxb2W9ilcxNCCNF3SXBGCCGEEEII0S75ZdXsOGtduJzmYPmlUG9XrhmuBgOyiivZfjbX4TmKtimtqmXfeTV7INTbhegwrxb3qaitYPPFzQB4mDyYGzHXoTl4O3uzuP9iAEpqSrRF0dYK8XZhxqAAAC7ll3P0UsMgSUc6n1PK77+MZ+rTW3l6wxnSC63BpCFBHjy9ahSbfzWb1RMjMNRFHwvLa3h241lmPbudt3ddoKJazZQpKq9h5znrZ2pSf79OnXtTaiw17ErbBaiL0RNDJnbLPBzh6eSpBZUKqgrYl76v0XGLooPxdTMBsOFkFoXl1dprn8WmYbaowb3VEyMxNpPFlFaSxrOHn2XBZwt45tAzXCqx9s6K8ozikQmP8EvPX7J6yGpMevV8ZTVl/OPEP1jy+RL+ceIflFaXAlBtrmbrpa0AuJvcmRkxs53fBdFaU0OnEu6h3mCwL30fGaXWvk/rk9drQeJlA5fhamy51KMQQgjRGhKcEUIIIYQQQrTLurgMausWLq8fG64tPDvi+1Osd7H/9+BFh48n2mb3uVyq6zKWFowIblUpq+2XtlNeWw7Aov6LcDG2vSfHlVYPW61tf3a27aXNrrfJ4vriaMeXNlMUhZ3ncrnjvUMseHEnHx64pJUiA7hmeBAf3jWFTb+aza2ToxgS7Mmz3xvD1ofmsGp8uJYhll9WzVPrzzDr2e28+//s3Xd0FGUXwOHfpvfeSYeE3nuXKh3pRSkiiGBvYEWRT8UudhFFkCrSpAlI770n1IQkhISQ3pNt3x8LA5GSTbJJKPc5J4eZnXln3k12l2Tu3Ht3RvP3sXjUWsN7qmddP5O8p0rjyJUjZBVmAYYyddeDCfcbY0qbWVuY07+RPwCFGp3yetH9p7zizf1prtPr9RxIPMCLm1+k5/Ke/BHxB9nqbGV7c9/mfNfxO1b1W8XQ6kPxNPfkjaZvsKbfGgaFD8JCZehzkqXO4vuj39NtWTdmnZjFhpgNynE6BXZSMoBE+TFTmdGvWj8A9OhZfn65su3mTJqB4QMrfG5CCCEeXBKcEUIIIYQQQpTKzRe9HytjSbPr2oV7Kg3ot529SlxqrkmOK4yz8aaSZp2NLGl280Xvmy+Gl0U9j3pKA/rjycc5nXq6ROO71fHBxtLw5+7q4wkUaLTFjCiZD1ZHMOq3/Ww7eyO7y87KnFEtg9j8ant+G92UNmEetwS3gj3s+XJwAza83J4+9f2U8mvJ2QVMWx3BuytPKfv2ru9r0jmXxJa4Lcry/VjS7LrWVVrjam3of7QldosScPqvm3vJLD4Qh16vZ09UCrHXPn/ahnkQ4HZrz5efjv3EmPVj2By3GZ3eENS0NrdmQNgAlvZZyqyus2gf0B4zVdFLL74OvkxpOYVV/VbRr1o/zFWGXl0ZBRnMODyDN3e8qezbLbhbGb4DoiQeq/aY8rNafm45Wp2WyJRIIlIiAKjlXosabjUqc4pCCCEeMBKcEUIIIYQQQpRYdHIOR+PSAajh40hN3+LLXxnD3EzF8GvZM3o9LDoQW8wIYSoarU7pr+FgbUGL0OJLaiXnJbPn8h4AfO19aezd2CRzUalUDA4vffaMg7UFj9b2ASAjT81WE5bIyynQsGDfjddlFRdb3ulZkz1vdmJq3zqEejoUe4xqXg58M6wh619qR4+6Prds93e1pUGAi8nmXBJ6vV4JzlioLO7rklqWZpZ0CzEENwp1hfwb8+9t9wvzdqRxkCGIc+ZKFkfi0ll0U9bMkKa3Zs2odWrmn56vrHvZevFCwxfYOHAj77d6Xwku3o2/oz8ftP6Avx/7mz5V+9wSxHG2dqaFX4vin6gwCW97b9pWaQvAldwr7Lq8q0jWzICwAXcaKoQQQpSKBGeEEEIIIYQQJbb8yI2smf6NTJM1c92gJv5YXCvntPjAJdTXymyJ8nUoJo30XEPT9PbhnlhbmBc75p/of9DqDVkpPUN73nJxuSx6hvZUejusjlpNjjqnRONvzuZabsLSZptOJ1GgMbwm+zWswrbXH2Fs21CcbUte+ivc25EfHm/M2hfa0vWmTKVhzQKNKilXHs6nnyc+2/D9auzTGCcr0wReK4sxpc2gaPbMj1svsP5kIgBu9lZ0uU0W2f6E/WQUZADQ3r89/wz4h3H1xuFq41riOQY6BfJhmw9Z3nc53UO6o8Lws3+s6mP3bUm5+9XNAZgFkQtYG7UWAFsLW3qE9KisaQkhhHhASXBGCCGEEEIIUSJ6vZ4V14IzKhX0bWDa4IyXow1daxsuhiZnF7Ax4koxI4Qp/FukpJmXUWPKo6TZdQ5WDsrF0FxNLmuj15ZofNtqHng4GHp1bD6dRMa1wFNZrT2eoCwX1yTeWLX8nJg5sgn/vNSWX0c1YXy70DIfs7S2xm1VljsEdKi0eZhKXY+6BDkFAXAg8QAJ2Qm33a9nPV8crQ09YDZGXFF6Lw1oVOW2gcoNMRuU5ceqPYaledmDKKHOoXza7lNWPraSGR1m8EKjF8p8TFEybf3b4mnrCcCuy7vIUhtK4T0a/CgOVsVnxQkhhBAlUabfIvfu3WuqeQghhBBCiIeMXq9nx7mr/BtxBb1eX9nTESVwODZN6cXQuqoH3k6lbwCv1+s5mHiQjTEblZ4NAMObBSnL8/fFlH6ywih6vV4JgpmbqehQvfjgTFR6lNKLoaZbTaq6VDX5vAZVH6QsLzmzpESfFRbmZvSp7wdAoVbHquOXyzyf7AINW84YSr95OFjTLKT40m8lUcPHiU41vU0S8Cmtm4Mz7f3bV9o8TEWlUhUJHK6OWn3b/eysLOjTwO+Wx+9U0ux6iTRbC1vaVDFt6bcQ5xA6BnbEytzKpMcVxbMws+Cxao/d8riUNBNCCFEeyvQbX6tWrahduzZffPEFSUlJppqTEEIIIYR4wJ2Mz2DIz3sZ8et+xs49yJoTt7+TWdyblt1UIqpfw9JnzURlRDHh3wk8uf5JXtn6Cn+d/UvZ1qqqO8Huhgbcu86nEJ1cspJWomQuXM3mYooh4NYkyBUXu+IvCt98kbt31d7lMq/a7rWp414HgMjUSE6lnCrR+JtL7i09fKnM89kUeUUpadajrg/mZpVTeqy8XM29yvHk4wCEuYbh7+hfyTMyjZtfn39f+PuOQb5hzQKLrDcNdqWal+Mt++1L2EdmYSYAjwQ8go1F6QPU4t7TL6xfkfVqLtWo71m/kmYjhBDiQWZR1gOcPn2aSZMm8dZbb9GzZ0+efPJJevbsiZmZVEwTQgghhBBFpWQX8PmGsyw6EMvN18Z+33WRXvVuvWNZ3HsKNFpWXyvrZGtpTrc6tzYzL05mYSY/HfuJhZEL0eg1yuPzI+czKHwQKpUKMzMVw5sH8tHa0wAs3B/LWz1qmuZJPERyCjQkZxeQlqsmPbeQ9Gv/Kut5atJy1cSm3Ah+3a6/xn/p9DolOGOuMqd7SPdyew6Dqg/i5O6TAPx55k/qeNQxemxtPydq+DhyOjGLI7HpXLiaTVXP0pcmWnNTSbOedX1LfZyKlqvOJSU/hcyCTNIL0skoyFD+zSi8sXw5+0Z20YNQ0uy6Kg5VaOLdhINXDnIx8yLHk4/f9mJ7nSrO1PZz4tRlQ+BlaNPAW/YBWH9xvbL8aPCj5TNpUWkCHANo7tucfQn7AOgf1r/SekAJIYR4sJUpODNjxgx+//13jhw5glqtZuXKlaxcuRJvb29GjRrFk08+SXh4uKnmKoQQQggh7lNqrY4/9sTw9b9nyczX3LL9YEwa55OybnuHsri3bD1zlYw8Q++OrrW9sbc2/k8KrU7L8vPL+fbIt6Tmp96yPSojimNXj9HAqwEAAxsH8Pn6sxRqdSw5GMcrXcKxsSy+Sf3DJl+t5WJKDheTc4hKNvx7MTmXqOQckrMLSny8zjWLD84cvnKYhBxDoKKFXws8bD1KfB5jdQvuxmcHPiNbnc266HW81vQ1o5vUq1QqBjb2539rIgFYeugSk7rVKNU8svLVbD17FQAvR2uaBJu2pFlZFWgLiMuMIyYrhtjMWGIyY4jJNCwn5ZW80sWDFJwB6FutLwevHATg7/N/3zET4o3uNXjq94PUqeJEz3q3BuDUWjWbYjcBYGdhZ/KSZuLe8HzD5zl+9Tj+jv63LXMmhBBCmEKZgjPPP/88zz//PMePH+fXX39l4cKFJCcnk5iYyKeffsqnn35Ky5Yteeqppxg8eDD29vammrcQQgghhLhPbD97lQ9WR3A+KVt5zMHaguc7VkMPTF9nyIxYtD+Od3rVqqRZCmMtL2VJs8NXDjN9/3QiUyOVx6zNrXmqzlO427ozbe80AP46+5cSnHGzt6J7XR9WHr1MWq6a9acS6dug9GXUHgR6vZ6/j11mX3QqF5NziE7OISEjv8zHVanAxdaSx5sHEexR/N9tRUqahZZPSbPr7Czt6F21NwtPLyRfm8/qC6sZXnO40eP7NqjCx+tOo9XpWX4knle7Vi9VObJNkUkUKiXNfCu9pNmm2E3subxHCcAk5CSgp+z9u5ysnOgV2ova7rVNMMt7R5egLny07yPyNHmsu7iOSc0mYW1ufct+bcM8iZzWDTMVt82W2JOwh6xCQ5P4DoEdbnsMcf+r71mfPcP2GDI5VVIZRgghRPkoc1kzgHr16jFjxgw+//xzVq1axezZs/nnn3/QarXs2bOHPXv28OKLLzJ48GCefPJJWrdubYrTCiGEEEKIe1hMSg7TVkfyb+SVIo8PbOzPpG7V8XK0IS2nkC83GDIjlh6+xOvdqmNtIZkR96qMXDWbT99oht6mWvHZEok5iXx56EvWRa8r8vijwY/yauNX8XXwJU+Tx9eHviZLncX6i+uZ3GwyjlaGLKrhzQJZedRQamn+3tiHPjiz/lQiLy46atS+Hg7WhHjY4eNsi4utJa52lrjYWeFiZ4mrnRXO1/51tbPE0cbS6GBDgbaADRc3AIbMgY6BHUv7dIw2KHwQC08vBGDJ2SUMqzHM6DJDno7WtA/3ZPPpJBIy8tlzIYU2YSXP9Fl9U0mzHpVc0mx/wn5e2vKSUfu6WrsS6BSIj70PLtYuOFs742zljIuNCy7WLjhZOSmPO1k5YW72YH4G21va0zmwM6uiVpFVmMWWuC10C+52233v9l64uaRZ16CuJp+nuHc8qO8FIYQQ9w6TBGeus7S0pH///vTv35/ExETmzJnDnDlzOH36NNnZ2cyePZvZs2cTHh7OmDFjGDlyJN7exafMCyGEEEKI+0d2gYbvt5zn1x3RFGp1yuMNAlx4v09tGgS4KI+52lvRrY4Pfx8zZEZsjLgivWfuYatPXFZ+pn3q+2Fhfue7ifM1+cw5NYdfT/5KniZPeby6a3UmN5tMU5+mymO2Frb0DO3JojOLyNfmsy56HYOrDwagWYgb1bwcOJ+Uzf6LqZy7kkWY98Nb/u733ReLrLvYWRLiYU+Iuz3BHoYvw7IdjjaW5TKHrXFbyVIbMgc6B3XG1sK2XM5zszDXMBp6NeRI0hHOp5/n6NWjNPRqaPT4AY38lcDi0sOXShycycxXs/3mkmZBriUab2oLTi8osu5o5UiQYxCBToEEOQUpX4FOgUaXgHsY9KnWh1VRqwBDabM7BWfupFBbyJbYLYAh2NO6itx4KoQQQojSM2lw5mY+Pj5MnjyZyZMns2fPHmbPns3ixYvJysrizJkzvPHGG7z99tv06NGDZ555hm7dSvZLkRBCCCGEuPdk5Knp9/0uopJvNBf3crTmje41eKxBFcxuczfy0KYB/H3MkBmxaH+cBGfuYTeXNOvf6M4ZLPmafEauG1mkhJmLtQvPN3yeAWEDbns38oDwASw6swgwlDa7HpxRqVQ83jyQqasiAJi/L5b3+zxY5ZaMdT4pm71Rhl49oZ72LJvQChc7qwqfx+oLN0qa9QrtVWHnHRQ+iCNJRwD488yfJQrOdKrphZONBZn5GtadTOCDvrVLFLz6N+KKEpjsUdf3tp9lFeVKzhW2xm0FwNPWkyW9l+Bm4yYNy43QzKcZPvY+JOYksvvybpLzkkvUL2nP5T1KYLJDgJQ0E0IIIUTZVEjhzMLCQgoKCtBqtcovjHq9Ho1Gw6pVq+jZsycNGzZk7969FTEdIYQQQghRTj5aE6kEZqzMzZjwSFU2v/YI/Rv53/FiZotQd4Lc7QDYeT6Z2JTcCpuvMN6JSxkcjEkDIMzLgdp+d74b/4ejPyiBGXOVOY/XfJzV/VYzuPrgO5aJqeFWQ+lxEZkaSURKhLKtf0N/rC0Mf7osPXyJvEKtSZ7T/WbBvlhl+fHmQZUSmInNjGXbpW0AeNl50cynWYWdu0tQF5ytnQHYcHED6fnpRo+1sTSnTwND4DdfrWPdicQSnXvNTSXNet2mSXxFWnZ+GVq94T3QP6w/7rbuEpgxkpnKTOmRpNVrWRO1pkTjby5p9mjwoyadmxBCCCEePuUWnImNjWXatGlUrVqVjh07Mm/ePHJzczEzM6NXr14sXryYd955B39/f/R6PceOHeORRx5h37595TUlIYQQQghRjnacu8rig3EAOFhbsPbFNkzuVgMH67sna5uZqRjSNEBZ//PaMcS95eftF5TlJ1uH3PFi8ImrJ5gTMQcASzNL5veYzxvN3lAuqt/NgPAByvKyc8uUZWc7S3rXN1xYz8rXsPr45VI9h/tZvlrL0sOXALCyMGPAXTKXytPciLlK0/lhNYZVaE8GGwsb+lbtC0ChrpCVF1aWaPyARv7K8l/XvpfGyMhTs/2coaSZj5MNjQIrr6SZRqdh6dmlgCHQMCBsQDEjxH/1qdpHWV5xfgV6vd6ocQXaArbEGUqaOVg60MqvVbnMTwghhBAPD5MGZ/Lz81mwYAFdunQhNDSU999/n+joaPR6PSEhIfzvf/8jNjaWv//+m0GDBvHBBx8QHR3NvHnz8PDwoLCwkClTpphySkIIIYQQogLkFGh4Y+kJZf3NHjWo5mV8X5CBjfyVBsxLDsWhualXjah8sSm5rD1hyBzwcLC6Y0mzQm0hU3ZPQac3/PwmNphIbQ/jS5B1D+6u9C9ZE7WGXPWNLKrhzQOV5fk3ZZA8LNYcTyAjTw0YMjcqI2smJS+FFedXAGBnYaeUnqtIA8MHKstLzi4x+sI6GPpehXraA7A/OpW4VOOy9DZGXEGtNZynskua7YzfyZXcKwC0rdIWX4fKzeK5HwU7B1Pfsz4A59PPFym/eDe743eTrc4GoGNgR6zMK/49KIQQQogHi0mCM/v27eOZZ57B19eXESNGsHnzZnQ6HVZWVgwZMoSNGzdy/vx53nrrLXx9i/7yaGZmxvDhw/nyyy8BOHTokCmmJIQQQgghKtCn/5wmPt3Q9L1lqDvDmgYWM6IoLycbOtXwAuBKZgFbzlw1+RxF6f26MwrdtWvgo1oGY2N5+2yJX078wvn08wDUdKvJqNqjSnQeBysHpUF3tjqbDTEblG0NA1yo6WsopXY0Lp2T8RklfRr3tfn7YpTlx5uX7P1lKovOLKJAWwAYspwqo9F8iHOIUkotJjOGfYnGV15QqVRFsmeWGpk9cz0wCdCzkkuaLTm7RFkeFD6oEmdyf7s5e+bvC38bNebmzyMpaSaEEEIIUyhTcOazzz6jVq1atGrVil9++YWMjAz0ej21atXiq6++Ij4+noULF9KpU6dij9W0aVMA0tLSyjIlIYQQQghRwfZHpzJnj+HCsa2lOdMH1C3VneVDm90obbb4wMOXGXGvSs0pVMrV2VqaM6Jl0G33O5N6hlnHZwFgobJgWutpWJoZ33D9uv5h/ZXlm0ubqVSqIkGJX3dGl/jY96vIhEwOx6YDUMPHsVLKauWqc1l4eiFg+PmOqDmiwudw3aDqN4ISc0/NLdHY/o2qcL0i39LDl9Dp7p55k5GrZse1kmZ+zjY0DHAp0flM6XL2ZXZc2gGAj70Pbaq0qbS53O+6hXTDysyQ+bI2ai1qrfqu+99c0szR0pGWvi3LfY5CCCGEePCVKTgzefJkzpw5g16vx87OjjFjxrB7925OnDjBiy++iJubm9HHsrC4ey1yIYQQQghx78kr1DLpr2PK+uuPVifI3b5Ux2of7oWvsw0Am08nkZiRb5I5irL5Y08M+WpDmbIhTQNuW05LrVPz7q530eg1AIytN5bqbtVLdb76nvWp5lINgCNJR7iQfqPXTf9GVXC1MwR8/j52WcnWetAtuKmM2+PNAyul+fuK8yvIKDBkK3UL6Vap5bQ6BXbCz97Qg2hH/A7OpZ0zeqyvsy1tqnkAEJeax4GLqXfdf0NE4j1T0mzpuaVKv5+BYQMrtN/Pg8bJyokOgR0ASCtIY0f8jrvuvyt+FznqHMBQ0szSvOSBZyGEEEKI/ypzWbMmTZrw888/k5CQwKxZs2jRokWpjlO1alV0Oh1arbasUxJCCCGEEBXkq3/PcjHF0LehUaALo1oFl/pY5mYqBjUxZM/o9LDkWraGqDx5hVrm7LkIGH4+T7UJue1+c07NUfo2VHOpxtN1ny71OVUqVZEm50vPLVWW7awsGNEyGACtTs9vD0H2TE6BhuVH4gFD5lLfhrfv91OeNDoNcyNuZKiMrj26wudwM0szS0bUupG58/up30s0viSlzdbcVNKsRyWWNFPr1EommbnKnH5h/SptLg+KkpQ2W39xvbLcNbhruc1JCCGEEA+XMgVnjh07xr59+xg3bhwODg6mmpMQQgghhLgPHI1LZ9aOKACsLMz4dGB9zMt4V/ngJv5KyaHFB+OKLTkkytdfhy+RmlMIQM+6vgS42d2yT1R6FD8e/REAM5UZH7T6oMx3lfcK7aWURFt1YRWF2kJl26iWQVhbGP6MWbQ/lozcu5cjut+tOnaZ7AJDRlLfBn442VT8Hfv/xvxLfLYhQNTar3Wps6JMqX9Yf6XnzdrotSTmJBo99tHaPjhYGyo3rD2RSF7h7W8QTM8tZOe5ZACquNhWakmzrXFbSc4zzKVDQAe87LwqbS4PilZ+rfCwNWRRbbu0jbT825dYz9fkszVuKwCOVlLSTAghhBCmU6bgTN26dU01DyGEEEIIcR8p0BjKmV2PnbzUOYxqXmW/Wcff1Y62YZ4AXErLY9eF5DIfU5SOVqdXgm8AT7cLvc0+WqbsnkKhzhA8GVVrFHU9y/43gouNC52DOgOQXpDO5tjNyjZ3B2sGNTFkPuQUapm3L6bM57uXzb+ppNnwm3ruVBS9Xs/sU7OV9dF1Rlf4HG7HztKOIdWHAIbMnvmR840ea2tlTo+6PgBkF2hYf+r2gZ0Np66g0V0vaeZTKeXkrltyZomyPCh80F32FMayMLOgV2gvwPAaWhu99rb77YrfRa7GkCHaKbCTlDQTQgghhMmUuayZEEIIIYR4+Hy/+Txnr2QDULeKM0+3vfXCfWkNaxqgLC/aL6XNKsv6U4nEXCtZ16aaB3WqON+yz4LTCzh21dBzKMgpiIkNJprs/APDBirLf537q8i2sW1ClQyr33dfJF/9YJZGPn4pnRPxhj4vdas4U8/fpcLnsD9xPxEpEQDUdKtJc5/mFT6HOxlec7jS1H3J2SVkFWYZPXZg4xufM3cqbbb6ppJmPev5lXKWZRebGcuehD0A+Dv408KvdKXExa2MKW12c0mzR4MfLfc5CSGEEOLhYWHMTrGxscXvVAqBgRV/55cQQgghhCibiMuZ/LDV0KTdwkzFJwPqYWFuunt+OtX0xt3eipScQjZEJJKSXYC7g7XJji+Kp9fr+XnbBWX9dlkzcZlxfHP4G2V9aqup2FjYmGwOTXyaEOAYQFxWHPsS9hGXFUeAo+GCerCHPd3r+LD2RCJXswpYcSSeoc0evL8tFtyUNfN4JWTNAEWyZp6s82SlZo/8l4etB32q9eGvs3+Ro85hydkljKkzxqixTYNdCXSzIzY1l53nk7mcnoefi62yPS2nkF3nb5Q0q+9/a3CyotwcnBxUfRBmKrnH0lTCXMOo6VaTyNRIIlIiOJ92nmqu1ZTteZo8tl7aCoCTlRPNfe+d4KQQQggh7n9G/VYXEhJi8q/QUNPdXSmEEEIIISqGWqvj9b+OKaV+JnaoRi0/J5Oew8rCjIGN/a+dT8+yw/EmPb4o3r7oVI5dMmRs1PR1om2YR5HtOr2O9/e8T742H4BhNYbR2LuxSedgpjKjf1h/ZX35ueVFtj/drqqyPHNH1APXnygzX83fxy4D4GBtQe/6FZ+5cSb1DLvidwFQxaEKXYK6VPgcijOq1ihUGAJG8yPmo9Ya14NIpVLRv1EVAPR6WH6k6OfM+lOJaK+9pnrV8620oFShtpAV51YAhjJcfav2rZR5PMj6VrvxPf1v9szO+J3kafIA6BzUWemFJYQQQghhCkYFZ/R6fbl8CSGEEEKI+8vM7VGcupwJQHVvR57rUK2YEaUz+KbSZgsPxMrvjhVs5vYbvWbGtwu95cL0X2f/Yn/ifgD87P14qdFL5TKPvlX7Yq4yB2DF+RVodBplW4MAF5qFuAEQdTWHTaeTymUOlWXlkXhyrzWq79ewCvbWRhU9MKk5p+YoyyNqjcDCrOLnUJxg52A6BHQAICkviTXRa4weO6CRv7K89PClIp8za4qUNPM1wUxLZ1PsJtIKDI3quwR2wd3WvdLm8qDqEdJDeW2vjlpd5HOmSEmzIClpJoQQQgjTMuq369mzZxe/kxBCCCGEeKCdT8pixr/nADBTwacD62FlUT7ldap6OtAsxI390alEXc3hYEwaTYPdyuVcoqizV7LYfC3Q4edsc8uF6cScRL489KWy/n6r97GztCuXuXjaedLevz2b4zZzNe8qOy7toENgB2X7M+1D2R+dCsDP2y7QpZZ3ucyjoun1eubfVNJseCWUNEvMSWRd9DoAnK2d6VetX4XPwVhP1nmSzXGbAfj95O/0qdrHqNJfAW52RT5njsal0zDQlZTsAnZfSLm2jy11b9NvqaL8eeZPZXlQ9UGVNo8HmauNK+2qtFM+Z/Ym7KVNlTbkqnPZfmk7AC7WLjT1bVrJMxVCCCHEg8ao4MyoUaPKex5CCCGEEOIeptXpmfTXcQq1OgDGtQulfoBLuZ5zWLMA5cL7wv2xEpypIDdnzTzVNhTLm/oJ6fV6pu6ZSo46B4ABYQNo6deyXOczIHyAcuF96bmlRYIzj4R7EeblwLmkbA7GpHEoJo3GQa7lOp+KcDg2ndOJhub2jQJdqOlr2tKBxvgj4g80ekMGwdDqQ8stAGcKDbwa0NCrIUeSjnAh4wI743fSzr+dUWMHNvZXPmeWHr5Ew0BX1p+6opQ061nXr9JKmkWlR3HwykEAgp2CaeLdpFLm8TDoU62P8jmz8vxK2lRpw474HUpJs06BnaSkmRBCCCFMTjoJCiGEEEKIYs3eFc3h2HQAQjzseblzeLmfs3sdX5xsDPcSrT2RQEaecb0kROklZuSz8qih94aTjQVDbyovB4Z+DDvjdwLgZevFq01eLfc5tfZrjbedISNmR/wOEnMSlW1mZiqebnejl+XM7RfKfT4VYf6+GGV5ePOgCj9/ZmEmf501NKG3NrdmWI1hFT6Hknqy9pPK8uyTxld+6FHXF1tLQ+m8v49eJl+tZe1NJc16VWJJsyVnlyjLg8IHVVqQ6GHQrko7XKxdANgcu5nMwkw2XNygbH80WEqaCSGEEML0JDgjhBBCiAqXlJnPi4uO8NqSY+SrtZU9HVGMDacSmb7uNACqa+XMbK5dzCyLjIIM3t31LpO2TSK7MPuW7TaW5vRraGjYna/W8ffR+Fv2EaY1e1c0aq0hY2BEy6AifU72Jexj2t5pyvqUllNwtHIs9zmZm5nTL8xQUkun17Hy/Moi2/s2qIK3kzUAGyKuEHX11tdSZdDr9czcEc2PEWb8G2l8P5z03EJWHzcEB5xsLColOPDnmT/J1eQChr4/90Ofk/YB7QlxDgHg4JWDnLh6wqhxDtYWdKvjA0BmvoYlB+PYfSEZgEA3O2r7lS1rSa/Xs+j0IsZvHM+aqDVG98/K1+Sz8oLhtW5lZlWkab0wPUtzS3qE9ACgUFfIinMrlJJmrtauNPWRkmZCCCGEMD0JzgghhBCiQp29kkW/H3az8uhl/jp0id93X6zsKYm72BR5hWcXHEZzrcTPmNYhJikvFpcVxxNrn2DF+RWsu7iOX078ctv9hja70Wtj4f44oy9sipLLylez4FqfEytzM0a1Cla2HUw8yPObn6dAWwBAn6p9aB/QvsLm1q9aP1QYsgaWn1+OTq9TtllZmDGmteGivF4Pv+yIrrB53YlWp+eNpSf4bMM5TmeYMWHBUZ5bcJiU7IJixy49HE+hxvD8BjYOMEkgtCQKtYXMj5wPgAoVI2uPrNDzl5aZyozRtUcr67NPGZ89M6CRv7L80drTXPu4o2c93zJlq+j1er44+AUf7vuQ3Zd388aON3hu83NFsr/uZEPMBrIKDaXtuoV0w9m68vrePCz6VOujLH939DvytfkAdArqhIWZURXhhRBCCCFKxGS/YRw7dowdO3YQFRVFVlYWWu3d74JVqVT8+uuvpjq9EEIIIe4Duy8kM/6PQ2Tla5TH5u+LYVzbUMzNpFzLvWbrmSQmzDusZFL0a1iFt3rULPNxTyaf5NlNz5Kan6o8tvzcciY2mIi1uXWRfWv6OlHf35ljlzKISMjkZHwmdf3lImV5WLg/lqwCw3uzf6MqeDnaAHAk6QgTN01Uei884v8I77d8v0Ln5ufgRyu/Vuy6vIv47Hj2JuyllV8rZfuw5oF8u/k82QUalh6+xCtdwvF0tL7LEctPoUbHy38eZc3xhCKPrz6ewK7zybzfpzZ96t++j4ler/9PSbOAW/Ypb2ui1pCcZ8gc6RTYiSCnii+rVlq9Qnvx7ZFvSc5L5t+Yf4nNjCXQKbDYcS2ruuPrbENCRj55N2Vz9qxb+qwlrU7LtL3TWHpuaZHHt1/aTr+V/Xi1yasMCBtwx+DPn2f+VJYHhQ8q9TyE8Wq51aKaSzXOp59XPu9ASpoJIYQQovyUOThz5swZxowZw969e40eo9frJTgjhBBCPGSWH7nEpL+OKxf6VSrDXe5xqXlsO5tExxrelTxDcbMd567y9B+HKNQa7uDvU9+PzwfVL3MQbXPsZiZvn6zckaxChR49aQVpbLi4gd5Ve98yZmizQI5dMpQoWngglrr+dcs0B3GrQo2O33ZeBAzvzXHX+rgcTTrKMxufUS5Utq3Sli8e+QJL84pvjD0gfAC7Lu8CYOnZpUWCM042lgxvHsjM7VEUanTM2X2R1x6tXuFzzFdrmTj/MJtPG8qYWZipaOej5Ui6NWm5atJy1by46Cirjl3mf4/VxcfZpsj4fdGpRF3NAaB5iBvVvMq/bNzNdHpdkYyTJ+s8eZe97z1W5lY8XvNxZhyegR49cyPm8k6Ld4odZ26mon+jKny/5UbPomD30pc0U+vUvL3jbdZdXAcYPudG1BrBuuh1XM27SrY6m6l7pvJP9D+81+o9AhyLBuHOpJ7h2NVjAIS5hlHfs36p5iFKRqVS0bdqX7449IXymJuNG028m1TirIQQQgjxICtTWbP4+HjatWvH3r170ev16PV67O3t8ff3JzAw8I5fQUFBBAYWfweTEEIIIe5/er2ebzed4+XFx5TATIfqnnw9pIGyz9w9MXcYLa7TaHXkFmoqpKzX7vPJjJ1zUCmt1LOuL18OLntgZkHkAl7a8pISmGns3ZhvOn6jbF90etFtx/Wu74ed1Y2G3TkFmtvu9zDQ6rTkqnNN/jr4+9hlEjMNP5fONb2p6unAiasnmPDvBKX3SCu/VnzV4SuszK1Mem5jPeL/CG42hpJ6m+M2k5KXUmT7k62DsTQ3vEb/2BtT4a+T7AINo2fvVwIz1hZm/Ph4A/oG6Vj3fCt63tQ75t/IJLp8tY3FB2KL/CznXysrBzC8ecX/vbT90naiMwxl4Rp5NaKeZ70Kn0NZDQofhJ2FHQArzq8okqF3N/1vKm0GpS9pVqAt4JUtryiBGQuVBZ+0+4TXm77O8r7L6Vetn7LvvsR9DPh7APMi5qHV3cjYWXJ2SZHnU5bSaqJkeob2xEx14zJJ58DOUtJMCCGEEOWmTL9lfPjhh1y9ehWVSsXYsWN57bXXCA8PN9XchBBCCHGfU2t1vLP8JIsPximPPd48kKl9aqNSqfj0nzPEp+ex7exVYlJyCHK3r8TZ3ruOX0rn8V/2kVWgwcrCDBdbS1ztrHC2s8TVzhIXWytc7A2Pudha4mpvRcNAF6UsVUnsjUrhqTkHKbgWmHm0tjdfD22AhXnp7+nR6XV8cfAL5kbMVR7rEdKDaa2nYWlmSbhrOGfTznI8+TinUk5R2712kfEO1hb0rufH4oNxZBdomLc3hvHtq5Z6PverqIwoRqwZRaY6HUszS5ytnXGxdsHJygkXa5cb69aGdRdrF+p41MHH3ueux9Xr9czcfiNj4Jn2oZxKOcX4f8eTrc4GoLlvc2Z0mHFL2bmKZGluSd+qfZl9ajYanYY/Iv7gpcYvKdt9nW3pU78KSw9fIiNPzeIDcYxpE1Ihc0vPLWTU7AMci0sHwN7KnF9HN6VxgBNrz4O7gzXfD29E73qJvLPiJMnZBWTla5i89ASrjiXwcf+62FqZ889JQyk0N3srpUn97cRmGoI4xpTsKonZJ29kzYypM8akx64oztbODAgfwB8Rf1CgLWDh6YU82+DZYsdV9XSgYaALR2LTAehRipJmuepcXtj8AvsS9wFgZWbFl498qfRncrZ25oPWH9AtuBvv73mfhJwE8jR5fHLgE9ZfXM/U1lPxsfNhddRqAGwtbOkV2qvE8xCl52nnSSu/VuyM3wlA1+CulTwjIYQQQjzIyhSc+eeff1CpVIwcOZKZM2eaak5CCCGEeABk5auZOP8wO84lK4+90b0G49uFKncBD28eyGfrz6DXG+4YN0U/kwfRD1suKL1ACjU6krIKSMq6e2NxS3MVver58WTrYOr5uxh1ngMXUxnz+wGl50Lnml58O6wRlmUIzORr8nlr51tsjNmoPDau7jiea/iccnfykOpDmLZ3GgCLTy/mg9Yf3HKcp9qG8OehOPR6+GHrBYY2C8TZtuJLa1Wmz/bMJFOdDhjKJiXnJSu9Qe7ETGVGp8BOPFHzCRp6NbztHfj/RiZx9oohCNM4yBV7xySeWv+00oy8qU9Tvu34LTYWJQ/2mdqwGsOYFzkPtU7N/Mj5PF7zcTztPJXtT7cLZenhSwD8ujOaES2DyvT6NUZSVj4jf93P6UTD98vZ1pI5Y5rRIMAFtVpdZN9udXxoEerGtNWRyjx3nk/m0a+30zjIVckuHNTEH2sL89ueLyIlgmFrhqHT6+gZ2pOXG72Mt33Zy0IevnKYw0mHAQh1DqWtf9syH7OyjKg5goWRC9HoNSw6vYgnaz+JnaVdsePe6VmLd1acpF2YB7X9StbbKqMgg4mbJnL86nHAEFj5ruN3NPNtdsu+raq0Ynnf5Xx96GsWnTFkDB69epRBfw+iuW9zctSG0nY9QnrgaFWxpe0EvNz4ZVLyUqjjUYdmPrf+/IQQQgghTKVMf6lcvnwZgJEjR5pkMkIIIYR4MCRk5DHopz1KYMbK3IxvhzXkmfZVi1wcHto0AKtrF07/PBhH/k2NmIVBVr6azWcMZZLsrMyp4eOIj5MNNpZ3/zVOrdWz/Eg8fb7bxcAfd7PmeAKaa/1jbudQTBqjf9tPbqHhZ9ChuiffP94IK4vS/7qYmp/K2A1jlcCMucqcKS2n8EKjF4qUjekV2gsHSwcA1kavJaMg45ZjhXs70q9hFQAy8tT8sj2q1PO6H6m1avZe2QqAXmeBNt8XNM5YqO6eyaLT69gYs5FR/4xi6JqhrLqwCrXWEDC4lJbLG0uP88y8Q8r+vZuoGLdhHJmFmYChtNV3Hb/D1sK2fJ5YCfk6+DKk+hAA8rX5/Hz85yLbq/s40qG6IVgTn57H2hMJ5Tqf+PQ8hvy8VwnMeDhYs3h8CxoEuNxxjIudFV8Mrs/sJ5vid63nTG6htkgge1jTO2fErDy/Ep3e8F5eE7WG3it6M/P4TAq0dw/Y3snV3Kt8vO9jxm4Yqzw2uvboIu/R+42vgy/dQroBkF6QzorzK4wa1zjIlXUvtuXNEt4okJyXzJj1Y5TAjKOVI790/eW2gZnr7C3tebvF28x+dDaBjoafd6GukB3xO5R9BoUPKtE8hGmEu4bzZ+8/mdJyipSUE0IIIUS5KlPmjKurK0lJSbi4uJhoOkIIIYS430UmZPLk7ANK/wpnW0t+GdmEZiFut+zr7mBNz3q+LD8ST3qumr+PXWZwk4Bb9nuYbYy4ovR+GdTYn6l96yjb8tVa0nILSc9Vk5ZbSMa1huPRydn8degSabmGi/AHY9I4GJOGn7MNI1sFM6xpIM52N7JOjsalM/q3/eRcC8y0DfPgxyca3/HOfWPEZMYw8d+JxGYZyi/ZWdjxefvPb3s3vp2lHX2r9WV+5HwKtAWsOL+CUbVH3bLfy53DWXXsMmqtnl93RjOqVTCejpVXZqsi7YzfjQbD3fSarDrkXx6qbKvqZc2LXatQo4o5GQUZytel7EusOL9Cya6JSIngrZ1v8fmBL/CkA8ciaqIuvFFKsHZQLr9d+IT0gnQAGng24IfOPxiVcVCRxtUbx7Jzy8jV5LL07FJG1RpFgNONz42n21Vly5mrAPy8LYo+9f3K5QJr1NVsnpi1j8sZhs+6Ki62zBvbnBAP48ozdqjuxfqX2/HJP6eZt/dGr5m2YR4E3+EYer2e7Ze2F3ksT5PHt0e+Zdm5Zbze5HU6BnY06vkm5yXz28nf+PPMn0UCO9VcqtEztKdRz+FeNrr2aKU82NyIuQyuPrhceock5iQybsM4LmZeBAwN5Gd2mUl1t+pGjW/i04SlfZbyw9EfmBMxRwm81XKvRW2P2sWMFkIIIYQQ97My3Q7VpEkTAM6ePWuSyQghhBDi/rb97FUG/bRHCcwEuNmybGKr2wZmrhvRMkhZnrc3ptzneL9Zdeyysty7vl+RbTaW5vg621LT14lWVT3oXteX4c0DebtnLfa82Ynp/esS7u2g7H85I5/p607T4uNNvLPiBOeTsjlxKYMRv+5Tyqa1rubOLyObYGNZ+sDM0aSjPLH2CSUw42nrye/dfr9rmaTr2RAAi88sVi5Q3izAzY7HmxteL3lqLd9tPlfqOd5vFkasVJbD7dsyuMmN5uUXkgp4YV4Un/6dgadFLToHdWZA+ABebPQi6wes56M2H1HT7UYmQGpBCmcK/sI65GNsfJfg6JjE6Pa2ZLt9T1pBGgD1POrxY+cfsbe89/pAudm4KcE7jV7Dd0e/K7K9Ragb9f0NJakiEjLZdT7F5HOITMhk8M97lcBMqIc9S55paXRg5jpHG0v+91hdFo5rQZiXA442Frzc5c49PKMzo7mUbSiHVs+jHsNqDMNcZXivxmfH89LWlxi3cRzn087f8Rhp+Wl8efBLui/trvRlAUMZrjF1xvB7t9+xMrcq0fO4F1V3q05rv9aA4Xvzb+y/Jj9HbGYsI9eNVAIzPvY+zOk2x+jAzHU2Fja80uQV5nWfR023mthZ2PFiwxdNPl8hhBBCCHFvKVNw5oUXXjA0EJV+M0IIIcRD788DcYz5/QDZ1y7y1w9wYfnE1lT1dLjruIYBLtT2cwLg+KUMjl5rqC0gLadQKXXk52xDo0BXo8faWJoztFkg619qx7ynmtOphpeyLU+tZd7eWDp/uY2BP+0mK9/wM2sR6saskU3LFJjZcHEDT61/Ssm+qOZSjfk95lPT/e5lgkKcQ2ju2xyAuKw4dl/efdv9nu1QDTsrw/wW7I8lLjW31HO9X+Rr8jmQZCh1pNfaMKxuZz4dWJ+Vz7amUaCLst+/kUl0/Wo709edVt6HVuZWtPDqSh3eQx03AXVmHfR6Q1aFykyDpcsh8P+SNSmTSc03BDFqu9fmxy4/4mB19/duZRpZayQu1i4ArItex5nUM8o2lUrF0+2qKus/b79g0nMfi0tn6My9JGcbgho1fZ1YPL4lfi6lL/3Wsqo7G15ux9EpXe/6Pt9x6UbJqy5BXXir+Vss6b2E5j7Nlcf3Jexj4KqBfLzv4yIlAjMKMvjm8Dd0W9qN2admk681BJasza0ZVWsU6/qv4+XGL+NsXbJeK/ey0XVGK8uzT85Gr9eb7Njn084z6p9RJOQYSucFOgYyt9tcgp2DS33Mup51+bP3n+wetptWVVqZaKZCCCGEEOJeVabgTJcuXZg0aRJbtmxhwoQJtzS8FEIIIcSDT6/X8+WGM0xaehyNznDhq0stbxaNa4GHQ/Elp1QqFSNvyp6Zu+dieU31vrPuZKLyPe1V3w8zs5KXZlKpVLQJ8+DX0U3Z/Gp7RrUMUoIbAAXXSqY1C3bj11FNsbUqXWBGr9cz59QcXtv2GoW6QgCa+zZnbve5+Dr4GnWMYdWHKcuLTi+67T6ejtY81SYEMPTV+Wrjg5/Bvf3SdjR6w4V0dVYdOtU0ZFDVD3Bh6YRWfDWkPt5OhvdaoVbHT9su0PHzrSw+EMvH6yJp9+kWftt1kfzsIPLjn0Ab8yY17XorfX4A5UJ9Tbea/NzlZ5ysnCr4WZaMg5UD4+qOA0CPnm+OfFNke7c6PgS6Gcqx7TiXbLKgb75ayzPzDpGRZ/i7p0GAC4vGtTBJeT2VSoV5Me/xm4Mz7fzbARDmGsYvXX/h60e+poqDoS+TVq9lwekF9Frei4WnF/LD0R/otrQbv5z4hVyNIaBpZWbFEzWfYF3/dbzW9DXcbd3L/BzuNc19mitZYxEpEXcM+paURqfh1W2vKiUDw1zDmNN9jtGfdcUxNyt9gFwIIYQQQtw/jCq6O3fu3Dtuq1WrFq1atWLmzJmsWrWKgQMHUqNGDezsiq9NPXLkSONnKoQQQoh7TqFGxxtLj7PsSLzy2OhWwbzbq1axFxlv1qd+FT5ae5qMPDWrjyfwTs9auNnf/2V1yqpISbN6fnfZ0zihng5M7VuHV7pWZ8nBOGbvukh8eh7NQtz4bXRT7K1L149Bq9PyyYFPWHh6ofJYn6p9eL/l+1iaW95lZFHtA9rjY+9DYk4i2y9t51LWJfwd/W/Zb1y7UP7YG0N6rprlR+MZ374q1X0cSzX3+8GKc2uU5QDLVng52ijrKpWKfg396VrLh++3nGfWjmgKtTqSsgqYvPREkeNYW5gxsmUQ49tXxcNhGLnqXP6+8DfzI+dzMfMitdxr8XPnn++bzIkhNYbwR+Qfyuvl8JXDNPJuBIC5mYrx7UN5e/lJAD5bf5r5Y1uU+Zxz91wk4Vops8ZBrswZ0wyHUr5vSiq7MJtDVw4BUMWhCiHOIco2lUpFp6BOtPFvw5xTc5h1YhZ5mjzSC9L5aN9HRY5jYWbBgLABjKs7Dm977wqZe2VRqVQ8VfcpXtv2GgAzDs+gpV9LzFRlukeRVRdWEZURBUB11+r8+uiv9837RgghhBBC3DuM+kti9OjRRjWVTEhI4NtvvzXqxCqVSoIzQgghxH0sI0/NM38cYk+UoRSSSgXv9KylZDWUhK2VOYMa+zNrZzSFGh1/HozjmfZVix/4AEvKzGdvtOF7G+xuR50qpstkcLa1ZGzbUJ5sHUJcai6BbnalysoByFXnMnnHZLbGbVUem1B/AhPqTyhxE3YLMwsGhQ/i2yPfokfPkrNLeLnxy7fs52RjyYT2Vfl43Wn0evhs/RlmjWpSqvnf67ILs9mTsBMAncaBbtXa3HY/e2sLJnWrwZCmAfxvTSQbI64o26wszHi8eSAT2lfFy+lGYMfO0o6hNYYyuPpgLmVdws/Br1wappcXa3NrJtSfwHu73wMMF95/7/a78rob3CSAmdujiEnJZdf5FHaeS6ZNmEepz5eZr+aHrYYSaSoVfNivToUFZgD2JOxBozeUq2vn3+627y9rc2uervc0far24atDX7E2eq2yzUJlwWNhj/F03adNluFxP+gS1IWabjWJTI0kMjWSjTEbeTT40VIfr0BbwA/HflDW32r+lgRmhBBCCCFEqRh9y5Berzf5lxBCCCHuT5fSchn0024lMGNtYcaPjzcqVWDmusdb3ChtNm9vDFrdw/27wpoTCVz/dal3fb8SBzqMYW6mItjDvtSBmeS8ZJ5a/5QSmLFQWTCt9TQmNphY6vn2D+uvBAiWnVumNCv/r1GtgpVSXv9GXuFQTFqpznev2xK3BY3eUCZOk1mXzjXvflE9yN2eX0Y24Y+nmtGxhhdjWoew/fUOvNe7dpHAzM3MVGYEOgXeV4GZ6/pU7UOwUzAAh5MOsyP+RtkvS3MzXukSrqx/tv50mf4G+WV7FOm5hnJmjzWoQg2fii39tv3SdmX5ekmzO/Gx9+GTdp8wt/tcOgR0YEj1Iazqt4r3Wr73UAVmwPD6fqHRC8r6d0e+Q6PTlPp4i08vJjEnETD8HK5nawkhhBBCCFFSRv0FFh0dXd7zEEIIIcR94mR8Bk/+foCrWYaL5m72Vswa1aREzepvJ8TDnnbhnmw/e5VLaXlsPZNEp5oPdsmduylS0qx+2UuamVpURhQT/51IfLahpJ2DpQNfPvIlLf1alum4HrYedAnqwrrodaQXpLP+4nr6VO1zy342lua82Cmct5YbSnd9+s9pFj3dolyCWJVpbdQ6Zdle3Zh6VYy7Q79tmCdtwzzLa1r3DAszC55v+DyvbnsVgG8Of0ObKm2UslW96/nx49YLnE7M4tilDNafSqRbnZIHJ65mFfDrzuhr51TxcufwYkaYlk6vU/rN2FrY0tSnqVHjGno1pGHHhuU5tftCa7/WNPFuwsErB7mYeZGV51cyIHxAiY+TXZjNrBOzlPUXGr5wl72FEEIIIYS4O6MyZ4KCgsrlSwghhBD3ly2nkxj88x4lMBPiYc+yCa3KHJi5buRN2TNz98SY5Jj3o7jUXA7HpgNQ3duRcO97q5/KwcSDjFg7QgnMeNt5M6f7nDIHZq4bVmOYsrzo9KI77jeoiT8hHvYA7ItOZfu5ZJOc/16RUZDB7gRDA3Od2plHgpqVOsvpQdYlqAu13GsBcCbtDP9E/6NsMzNTMalbdWX9s/Vn0Gh1JT7H91vOk1uoBWB480AC3Yvvr2lKkSmRpOQbMhWb+zTH2ty6Qs9/v1OpVLzY6EVl/YdjP5CvyS/xceZGzCWtwJCl1yOkB9XdqhczQgghhBBCiDsrWydEIYQQQjw05u2N4ak5B5QLlI2DXFk6oRXB1y6Om0KHGl5UcbEFYNvZq8Sk5Jjs2PeTNScSlOXe9e+tEkRro9by9ManySzMBAzNsOf3mE+4q+kyCRp4NqC6q+Gi54nkE5xKPnXb/W5Xtkr3AJXD+zfmX3R6w/tNk1mPTjV9KnlG96b/Xnj/7uh3qHVqZb1DdS+aBhsCyBeu5rDsSHyJjh+Xmsv8fYZgsa2lOc91rGaCWZfMzSXN2vq3rfDzPwgaeDXgkYBHAEjKTWLxmcUlGp+an8qcU3MAQwnH5xo8Z+opCiGEEEKIh0yZgjMdO3akU6dOxMQYf2fr5cuXlXFCCCGEuPfpdHqmrzvNOytOcv26d8+6vswf2xw3eyuTnsvcTMXjLQKV9Xl7H87smZtLmvWqd2+UNNPr9cw6MYvJOyYrF75bV2nNnO5z8LY3bfk5lUrFkBpDlPVFZ+6cPdOzri+1/Qy9P07GZ7L2ZMId973frIu+UdJMl9WAtuGlb2b/oGvp25LmPs0BiMuKY/m55co2lUrFpG41lPWvN54lX601+thf/3sOtdbw4TemTTBejrfv3VOeStJvRtzZCw1fQIUh++yXE7+QVZhl9NhZJ2aRq8kFYED4AAKcAspljkIIIYQQ4uFRpuDM1q1b2bp1Kzk5xt/VmpeXp4wTQgghxL3vzWUn+GnbBWX96XahfDusITaW5uVyviFNArAyN/yK8ufBS+QVGn8R9UFw4Wo2py4bslLq+zubNDOpLL469BUzDs9Q1geEDeDbjt9ib1k+8+sZ0hNHS0M5t3XR60jPT7/tfmZmKl5/9EZpoS82nEVdirJV95rkvGQOJB4AQFfoTiPfOjjZWFbyrO5dKpWqSNP3n479RJ4mT1lvGuxGxxpeAFzOyGf+vlijjnv2ShbLjlwCwNnWkqfbVTXhrI2TnJfMyZSTAIS7huNjLxlUpRXmGkav0F6AoWzg9UyY4iRkJyglFm3MbRhfb3y5zVEIIYQQQjw8pKyZEEIIIe7oZHwGiw/GAWCmgml9a/NWj5rl2vfC3cGaXvUMpbwy8tSsOn65mBEPltXHbi5pdm9kzVzKusTsU7OV9RcavsB7Ld/D0qz8ggV2lnb0rdYXgAJtASsvrLzjvu3DPWke4gZAdHIOfx26VG7zqigbLm5AhyHIpM6sR6caps1OehDV86xHp0BDdv7VvKssiFxQZPtrXW8E8b7fcp7sAk2xx/x8/Rn01zIGJzxSFWfbig+Q7YzfqSxL1kzZTWwwEQszC8DQQyY5r/heVT8e+1HJGHy85uN42nmW6xyFEEIIIcTDocKDM9ezbGxsKr4cgBBCCCFKZtGBG3eXv92zFiNaBlfIeUe0DFKW/9gTg17/4PQRuRu9Xs/fx270w+hZ797oN7Ps3DJleXy98YyrNw6Vqvwb0w+uPlhZXnxmMTr97TNi/lu2asa/50pUtupe9M/FG03tNZn1lawPcXfPN3weM5XhT5xfT/5KRkGGsq2WnxN9GxgCnqk5hczaEXXXYx2OTWNDxBUAvBytGVVBn3//JSXNTMvf0Z9B4YMAyNPkMevErLvuH5UepQSHHa0cebLOk+U+RyGEEEII8XCo8ODMunWG2tn+/v4VfWohhBBClEBuoYaVRwxZK7aW5gxuUnH/dzcIcKFOFUMfkRPxGRyNS6+wc1em04lZXLhquJGlWbAbvs62lTwj0Og0rDxvuDBprjJnSPUhxYwwnRDnEFr4tgAMfUR2xe+6476Ng1zpXNOQXZKYmc/cPRcrYorlIiE7gSNJRwDQ5ntTxT6Eqp4OlTyr+0NVl6r0Du0NQFZhFr+f+r3I9le6hGNxLfPvl+1RpGQX3PY4er2eT/85ray/2DkMW6vyKeV4N2qdmj2X9wDgbO1MPY96FT6HB9HT9Z7G1sLw+br4zGLis+PvuO93R79TAsNj6ozB2dq5QuYohBBCCCEefBYl2XnMmDG3ffydd97BxcXlrmMLCgq4cOECBw4cQKVS0b59+5KcWgghhBAVbO2JRLKulf3pXd8Xxwrsd6FSqRjZIphJS48DhuyZhoGuFXb+yrLq2I0Sbr3r3xtZMzvjd5KUlwRAe//2FV7OZ2iNoexN2AvAojOLaOvf9o77vv5odTadvoJeDz9svcDQZoH3ZZ+W9RfXK8uazHp0rO5VIZlKD4qJDSayNnotap2a+ZHzGV5juPK6DXK3Z2izAObtjSWnUMsPWy/wbq9atxxjx7lk9kalAhDsbsfgJpXT/P3IlSNkq7MBaO3XGnOzig8QPYg8bD14ouYT/HLiFzQ6DT8c/YEP23x4y34nk0+yMWajMmZ4jeEVPVUhhBBCCPEAK1Fw5vfff7/lD0O9Xs/KlXeuAf7ffQHc3Nx48803S3JqIYQQQlSwRftvlDQb0jSwws/fu74fH66NJCNPzerjCbzTqxZu9lYVPo+Kotfrlf46ZiroXvfeCM4sPbtUWR4QPqDCz9/evz0+9j4k5iSy49IOLmVdwt/x9llc1X0ceaxBFZYfiSc9V80v26N49aY+I/eLdRfXKcvqzPp0kJJmJeLn4Mfg6oOZHzmfPE0ePx//mXdavKNsf6FjGH8dukS+Wscfe2IY0yaEKi43stR0Oj2frT+jrL/StTqW5pXTqlNKmpWf0XVGs/jMYjILM1l1YRVP1n6Saq7Viuwz4/AMZXl8vfHYWdpV9DSFEEIIIcQDrER/ZQQGBhb5AsOdrb6+vrdsu/krKCiI6tWr06FDB95++22OHz9OSEhIuTwhIYQQQpTduStZHIxJAyDc24FGgS4VPgdbqxul1Aq1OhYfiKvwOVSkY5cyiEvNA6B1NQ88HKwreUZwJecK2+MNF4e97bxp7de6wudgYWbB4HBD7xk9ev48++dd93+5cziW5oabiX7dGU1aTmG5z9GUYjJjiEiJAECbVwVrvGgR6l7Js7r/jKs7TilbtfTsUhKyE5RtXk42jG5l+FukUKtjxr9ni4xddzKRE/GGXjW1fJ3oVYmB0uvvPzOVWaW8/x5kTlZOPFX3KcDw2fLtkW+LbN+bsFfJ2vN38GdAWMUHp4UQQgghxIOtRMGZixcvEh0drXxdt2HDhiKP//crKiqKiIgINm3axLRp0/Dz8zP5ExFCCCGE6dwcCBnaNLDSSio93jxIWZ63NwaN9vYN4R8ERUqa1bs3fldaeWGl0muhX1i/Siup1C+sHxZmhoTv5eeWk6/Jv+O+ge52DL2W6ZVbqGXBTRlg94N/ov9RltWZ9Wld1QMbSyllVVLutu6MqDUCAI1ew4LTC4psn9C+Kk42htfUX4cucT4py7CvVscXG25kzbzerTpmZpXz+ReXFUd0huFvrnoe9XCxcamUeTzIhtUYhpetITNtc9xmjl09BhgyGWccupE182zDZ7E0v/9KJAohhBBCiHtbmfLz27VrR7t27bC3tzfVfIQQQghRyQo0WpYevgSAlbkZ/RpWqbS5BHvY0z7c0CsiPj2Pr/89V2lzKU86nZ7V10qaWZqreLS2TyXPCHR6HcvOLQNAhYp+1fpV2lw8bD3oGtQVgPSCdD4/+Pld9x/XNpTr8cQ5uy9SqLl/gnr/XLwRnNFk1pOSZmUwvMZwrMwMpRD/OvsXOeocZZuznSXj21cFQKeHz9cbsmf+OnSJqGTDfs2C3XgkvGJ7LN1MSpqVP1sLW8bXH6+szzg8A71ez6bYTZxMOQlAuGs4PUJ6VNYUhRBCCCHEA6xMwZmtW7eyZcsWgoKCit9ZCCGEEPeFjRFXSMtVA9Ctjg+uldzn5cXOYZhfu3P9uy3n2XI6qVLnUx4OXEzlSmYBAO3DPXG2q/w7tPcl7CM+Ox6AVn6t8HOo3GyecXXHYW1uKPW2+Mxi1kStueO+ge52dK3lDUBSVoES+LrXnUs7x/n08wBocoPQa1wkOFMG7rbu9KraC4BsdTbLzy0vsv3J1sF4OhpeU/+cSmRfVAozNt0IAE/qVr3SsgYBdlzaoSxLcKb89AvrR6CjIdvuQOIBdsTvKFLi7IWGL2CmqpyeQ0IIIYQQ4sEmv2UKIYQQoohF+28uaRZQiTMxaBToyuRuN5q6v/znUS6l5VbijExv1U3Bg971742SZkvPLVWW+4f1r8SZGFRzrcbbzd9W1qfumcqF9At33H9s21Bl+ded0ej1+nKdnymsi16nLGsy61PDx7FIo3pRciNqjlCW50XOQ6vTKut2Vha80PFGA/in5hwkIcNQMq9TDS+aBLtV3ET/I1edy4HEA4Ch31O4a3ilzeVBZ2lmyXMNn1PWJ22fRFRGFAANvRpKYEwIIYQQQpQbkwdnMjMziY+PJzY2ttgvIYQQQtxbYlNy2Xk+GYAgd7t7phH5uLahSiZEeq6aZxccoUCjLWbU/UGj1bH2RCIANpZmdK7pXckzgtT8VDbFbgLAzcaNDgEdKnlGBv3C+vFYtccAyNPk8crWV8hV3z5Q1yTIlXr+zgCcupzJ3qjUcp9fUlY+608l8sk/pxk6cw99v9/F4dg0o8bq9XqlpJler0KTWVeyZkygmms1Wvm1AiA+O54tcVuKbB/SNJBANzsAsgs0AKhU8Nqj1Smt1PxUtsVt49sj3zJh8wR+yPqBfYn7SnSMfQn7KNQVAtDWv22lZvA8DB4NfpQabjUAipS/e7HRi/K9F0IIIYQQ5cYkwZmNGzfSr18/PDw8cHV1JTAwkJCQkLt+hYaGFn/gO/j4449p2rQpjo6OeHl58dhjj3HmzJki++j1et5//338/PywtbXlkUce4dSpU0X2KSgo4Pnnn8fDwwN7e3v69OnDpUuXSj0vIYQQ4n7358EbWTNDmgZUWiPs/1KpVHw2qL5yEfVYXDofrYms5FmZxu4LKaTmGC7Cdqrhjb21RSXPCFZdWIVGZ7hQ3bdq33uqEfZbzd8izDUMgKiMKKbumXrbrBiVSsVTbUKU9V93Rpt0HgUaLYdj0/h1ZzTPLThM6+mbafbhJsb/cYgft15gb1Qqx+LSGfP7AaKuZhd7vIiUCOKyDO8/bU5V9FpHOkpwxiRG1hqpLP8R8UeRbVYWZrzSpWhWSt/6ftT0dTLq2GqtmlPJp1gQuYA3drxBj2U9aL+4Pc9tfo6Zx2eyL3Efl7WXeWX7K0SmGP+ZtSP+ppJmVSRzo7yZqcx4oeELRR5rU6UNjb0bV9KMhBBCCCHEw6DMwZkXXniBbt268ffff5Oamoperzf6q7S2bdvGs88+y969e9m4cSMajYauXbuSk3PjLqdPP/2UL7/8ku+++44DBw7g4+NDly5dyMrKUvZ56aWXWL58OYsWLWLnzp1kZ2fTq1cvtNoH405cIYQQoiQ0Wh1LDhkuDpubqRjYyL+SZ1SUs60lPzzeCCsLw68vc/bEsOrY/dFL5G7+PnZzSTPfSpyJgV6vL1LSrF9Yv0qcza1sLWz5sv2X2FvaA7A2ei1Lzi657b496vri62wDwKbTV4wKktxNfHoeU1edou/3u6jz3nr6/7CbaasjWH08gfj0vNuOSc9VM+b3A0oA7k7WRq9VljWZ9XG2taRhgEuZ5isMWvm1oqpzVQAOJx3mxNUTRbb3qe9HDR9HACzMVLzS5e5ZM8l5yXxx8AtGrB1BiwUtGLpmKB/v/5g1UWuUANt/5WnyeG7TcyTmJBY7X71ez/ZL2wGwMrOiuW/zYseIsvtvMObFRi9W4myEEEIIIcTDoEy3Zi5YsIDvvvsOABsbGx577DEaN26Mm5sbZmbl187mn3/+KbI+e/ZsvLy8OHToEO3atUOv1/P111/z9ttv07+/oUb6nDlz8Pb2ZsGCBYwfP56MjAx+/fVX/vjjDzp37gzAvHnzCAgI4N9//+XRRx8tt/kLIYQQ96ItZ64qTek71fDCy8mmkmd0qzpVnJnapzZvLjNcXH1j6XFq+jpRzcuhkmdWOgUaLetPGi7WOlhb8Ej1ys+UOJJ0hOgMQ5ZJY+/GhDiHFDOi4gU7B/NBqw94ddurAEzfP53a7rWp7VG7yH6W5maMahXM9HWn0eth9q6LTHusTqnOmZxdQJ9vd5JyhyCLraU59fydaRDoQsMAV8K9HZg4/zCnE7O4mJLL03MPMm9sc2wszW8Zq9PrbippZo46qzbt63piYS7tIU1BpVIxotYI3t/zPmDInvm0/afKdjMzFb+MbMKP2y7QpaY3ge52dzxWdmE2w9cMJyEn4bbbrc2tqeVei3oe9ajrWZdqTtV4fu3zxGnjSMpL4rlNzzGn+xwluHg7Z9POciX3CgBNfZpiZ3nn+QjTUalUfN7+c3469hPNfJopZc6EEEIIIYQoL2UKzvz8888ABAQEsHnzZqpWrWqSSZVURkYGAG5uhqad0dHRJCYm0rVrV2Ufa2tr2rdvz+7duxk/fjyHDh1CrVYX2cfPz486deqwe/fu2wZnCgoKKCgoUNYzMzMBUKvVqNXqcnluQoj7w/XPgIf1syAzT83zi45x8nKm0WNq+Try9ZD6uNtblePMREks3BejLA9q7Ffi13OeJo83d73JkaQjRo8Jdgrmkzaf4GPvY/SYAQ182HchmRXHEsgp1DJh3kH+Gt8cO6vKLwcGJfs82ByZRNa1Phddanpijg61Wleu8yvOX2f+UpYfC33snv1c61ClA8OqD2PhmYWodWpe2foKC7ovwMmqaDmqgQ19+WbTOXILtfx1KI4XOoTiYlfyMm1TVpwoEpgJ9bCnfoAzDfydaRDgTLiXwy3BlJ8fb8DAn/dxNbuQgzFpvPbnUb4cVPeWHhaHkw6TlJsEgDY7DHR2tAtzv2e/9/ejrgFdmXF4BmkFaWyI2cDz6c8X+dzxcbRkai/Dxfi7fd+/OPBFkcBMgEMAdT3qUse9DvU86hHmElakDKBareYJ+yeYq51LfE48Z9LO8NrW1/iy3ZdYmN3+M2tr7FZlubVva3kdVCBnC2cmN54MPLy/04ny87D/vSCEKEo+E4R4sBn73lbpy1BfzNXVlczMTH755RfGjBlT2sOUiV6vp2/fvqSlpbFjh6E28+7du2ndujXx8fH4+fkp+z799NPExMSwfv16FixYwJNPPlkk2ALQtWtXQkJClMDTzd5//32mTp16y+MLFizAzk7uaBNCPLwWnDdj39WS3+EdYK/nudpabG69kVxUsPQCeP+wOXpUuFjpea+RlpK2m1mXt45dBbtKfG5PM0/GOozF3uzOd5L/V4EWvjxhTmKeYZJNPXQ8Xk3H/da3ec5ZMw6nGN4742toqeVa+rKvppCny+PTzE9Ro8ZGZcNkp8lYqu6dfjP/pdFr+DX7V+K0hlJS1S2q87j945ipin4eLY02Y3ui4bFegVq6VCnZ9/lYiorfzho+qOws9Eyqp8XV2rixcdnwzSlzCnWGF+ejVXT0CCwagFuVu4p9hYaG8XnxQ9BmNuDDJlrs791v/X1pU94mthRsAaCNdRu62XYr0fgodRS/5fwGgBVWTHCcgKe5p1Fjr2qvMjN7Jnl6Q/m75lbN6WXb67bN5mdmzSRWGwvAK46v4GbuVqJ5CiGEEEIIISpXbm4uw4cPJyMjAyenO/ezLNMtptcjQA0bNizLYcrkueee4/jx4+zcufOWbf/9Y0ev19/2DyBj93nzzTd55ZVXlPXMzEwCAgLo0KED7u7upZi9EOJBoVar2bhxI126dMHS8uG6mrbjXDL79hwGDI2V/V2KL4WVlFVIdoGGuBwVK5K9+GVEI6wtpHxPZfphaxR6zgPwRKuq9OpUrUTjjycfZ8/GPQBYmFng71B8v5qUvBSy1Flc1V1lleUqfur4U4nK99RvkUP/n/aSU6jlQLIZfVvXYUiTyumTk5WvISYll+iUHKKSsjh+JoqAwADMzO4eeYzMvATocLG15MWhnbGs5DJWS84tQX3A8Ptd37C+9G3St1LnY4xmOc0Y/s9w0gvSOaM5Q3JIMqNrjS6yT+3UXLp8vRO9Hg6k2fHJk22V3kXFSc9V879vdwGGrJkPHqtH3zv0BspV5xKbFat8peanghd0dM1ly5lkALZqoFDlRjXPG6X4Tl88DYBeZ4EmuxaNAl0Z1LdZCb8TojjN85qzc+VO1Do1R3VHmd5lutGfOXmaPH5a+5Oy/lLjlxhafWix467/fjC823Bqp9Zm4paJaHQa9hXuo3Wd1jxR44ki+6cXpDNl2RTAkFn4RK8nbndYIcR96GH+e0EIcSv5TBDiwXa94lZxyhScCQ4OJjIykuzssjVXLa3nn3+ev//+m+3bt+Pvf+NijI+PoURBYmIivr43/nhOSkrC29tb2aewsJC0tDRcXV2L7NOqVavbns/a2hpr61tvk7S0tJQPUiEE8PB9HmQXaHj370hl/f3etRnePLDYcWevZDHopz1k5KnZE5XK60tP8t3wRpiXNFVDmIROp+evI/EAqFQwtHlQiV7HhdpCPtj3ATq9IRvg2QbPMrbu2GLHXcq6xMh1I7mad5WTKSd5fefrfN/p+yIlge6mup8L0wfU4/mFhjJqH6w5TYNAN+pUcTZ67iWRV6jlYkoO0cmGr4vX/03JITn7v31IzCAx3uhjd6/ri52NkakY5WjFhRXK8qDqg+6Lz7MAlwCmt53OhH8noEfP98e+p4F3A5r6NFX2qebtTJea3myIuMKVrAI2nk7msYZVjDr+9PURXL328+1Uw4ue9T25mH2R2MxYYrJiiMk0fMVmxnI17+odj2N1U/LDwTTD139psmuCzppONb3vi+/9/cbH0odeob1Yfn452epsVses5vGajxs19qujX3Ep+xIAjbwa8XjtWzO07sbS0pKW/i2Z2moqb+9823DMw18R6BxIp8BOyn774/Yrn6Xt/dvL60CIB9DD9veCEOLu5DNBiAeTse/rMt2e2b9/fwA2bdpUlsOUmF6v57nnnmPZsmVs3ryZkJCijWpDQkLw8fFh48aNymOFhYVs27ZNCbw0btwYS0vLIvskJCRw8uTJOwZnhBBCFPXJutPEpxtKtLSq6s6wZgFGjQv3duS30U2xvdYYe93JRN5ZcZIyVNp8oCVl5VOoKb8+JLsuJBOXavg5tg3zxN+1ZKU6fz7+M1EZUQDUcq/F6NqjjRrn7+jPT11+wtHKEYA9CXt4a+dbaHVao8/du74fo1oGAVCo0TFh/iEyck1bt1mv1/Pqn8eoOeUfus/YwcT5h/ls/RmWHLrEwZi02wRmSsbKwoyR157D3aTkpVCgLSh2v9I6lXKKyFRDsLWOex2qu1Uvt3OZWusqrRlffzwAWr2WSdsnkZyXXGSfsW1DleVZO6OM+rzZeiaJpYcNF+QdrS2oUnUDLRa2oP/f/Xlp60t8degrlp1bxqErh+4amDGK3gx1amsAOlT3KtuxxB2NqDVCWZ4XMc+oz5tjV48xL2IeANbm1kxtNbVEgZmb9anah2fqPwOAHj1vbH+DU8mnlO3bL21Xltv5tyvVOYQQQgghhBD3hzJlzrz66qv88ccffP311wwdOpQaNWqYal539eyzz7JgwQJWrlyJo6MjiYmJADg7O2Nra4tKpeKll17io48+IiwsjLCwMD766CPs7OwYPny4su9TTz3Fq6++iru7O25ubrz22mvUrVuXzp07V8jzEEKI+9neqBT+2GtoIG9rac70/vWKLR15s8ZBrvz4RCPGzjmIRqdn4f5YPByseLXr/XNBuLzp9Xq+2niW77acx8PBmp9HNKZhoGvxA0to0YE4ZXlYU+MCbNdFpkTy64lfAbBQWfBBqw/u2OT6dsJdw/m+0/c8veFp8rX5/HPxH5ytnXm7+dtGv57e6lmTo5cyOBaXTlxqHq/9dYyZIxqX6PV4N8sOxysX6G/H09GaEHd7QjzsCfawJ8DFmgsnD9GmdWssLIr/XgS62eFqb3XXfeacmsPXh77G3sqeL9p/QXPf5iV+HsVZdnaZsjwgfIDJj1/enqn3DEeTjrI3YS/JeclM2j6JmV1mKq/HpsGu1K3izIn4DE7GZ7IvOpUWoXcuS5uVr+atZSeU9UHtMlkSteiO+7vZuBHkFESgY6DhX6dAfOx9MFfdKG2n1er4cN1pDl5MBcDbyZrPBtbHxtKc4T+dRVtgh6+zDTV9Hcv67RB3EOYaRkvfluxJ2MOl7EtsjdtKp6BOd9y/QFvAlF1T0GMI5j3b4FmCnYPLNIeJ9ScSmxnL2ui15GvzeW7zc8zvMR9vO292XTb07XKwdKChd+WVjhZCCCGEEEKUvzIFZ5ydnfnnn3/o06cPrVu3Ztq0aQwbNqxImbDy8OOPPwLwyCOPFHl89uzZjB49GoBJkyaRl5fHxIkTSUtLo3nz5mzYsAFHxxt/7H711VdYWFgwePBg8vLy6NSpE7///jvm5tKZWggh7iavUMvkpceV9UndqhPoXrJsC4BHqnvxxeD6vLjoKADfbj6Pm70VT7YOufvAh4BOp+eD1RH8vvsiAElZBQz7ZS8zhjbk0do+JjtPSnYBG04ZbnJwt7eiU01vo8eqdWqm7J6CVm+483xcvXGlyrZo6NWQLx75ghc2v4BWr2XxmcW42bgxscFEo8ZbW5jz/fCG9PxmJxl5ajZGXGHm9ijGt69a4rn8V2pOIf9bE6Gs967vR7iXAyGe9gS7G4IxDtZFf51Sq9WsjYF6/s5lLlGg1+v54dgP/HTM0OsioyCDZ/59hqmtptKnap8yHftmuepc1kSvAcDWwpbuId1NduyKYm5mzvS20xm8ajBJeUkcSDzA90e/58VGLwKGXoRj24Yonze/7oy+a3Bm+rrTXM7IB6BFNXt2ps1QtnUM6EgN9xoEOQYpgZjrGWDFmT2sFoN+2kNEQiYJ+fDF6jzGtgmlsMAQAHykupfJAovi9kbWHsmeBEOPrLkRc+8anPn52I3MwDrudYpk3pSWSqViWutpJOYkcjjpMMl5yTy76VleafwKGQUZALT0a4mlmZQ4EUIIIYQQ4kFWprJmoaGhdO/enYyMDNLS0nj++efx9PTEx8eH0NDQu35VrVr6CyZ6vf62X9cDM2D4o+f9998nISGB/Px8tm3bRp06dYocx8bGhm+//ZaUlBRyc3NZtWoVAQElu2NYCCEeRl9uPENMSi4ATYJcGdUyuNTH6tugCu/3rqWsT10VwYojxvfqeBBptDomLT2uBGauy1freGbeIWbvijbZuZYfiUetNdwRPrCxv9FN0gF+P/k7p1MNjcyruVRjXN1xpZ5HO/92TGs9TVn/8diPLIhcYPR4f1c7vh7SQFn/dP0ZDsfepqlHCX28NpK0a2XSetb15dthDXm+Uxi96vlRp4rzLYEZU9Lr9Xx64FMlMHOdRqfh7Z1v8+OxH01WCnBDzAZy1DkAdA/pjr2lvUmOW9Hcbd35/JHPlWyVWSdmsTt+t7K9R11ffJxsAPg38goXk3Nue5w9F1KYvy8WADsrc6pX30NCTgIAzX2b83WHr5lQfwI9QntQ26O20YEZAHtrC34b3VSZx5HYdF5bckzZ3rGGlDQrb639WlPV2fC3yOGkw0XKit0sIiWC307+BoCFmQUftC5ZZuDdWJlbMaPDDIKcDCUNz6ef59VtryrbpaSZEEIIIYQQD74yBWcuXrzIxYsXSUpKAgwXEXQ6HUlJScq2u30JIYS4/xyOTePXnYbggJWFGZ8MrIeZWdnu8h7dOoQXOlZT1l9bcowtZ5LKdMz7VaFGxwuLjvDXIcNd9GYqmN6/Lo818ANArzcEsD5YFYFWV7YL83q9oZzcdUNKUNLsQvoFfjz247U5mjGt9TQszct2l3fvqr2Z1HSSsj59/3TWRa8zenyHGl4828FwwVWr0/PioiNk5pe+/8yeCyksOXSj38iUm4KI5U2r0/L+nveZFzlPeez1Jq8zpPoQZf2Hoz8wZfcU1Lqy99hZenapsjwg7P4raXazhl4Nebnxy8r6WzvfIiUvBQBLczNGtQoGDO+l2wU6cws1RTIDRz9iycroxQBYmVkxpcWUMme2+Djb8NvopthbGYJIeWpD9pmVhRmtq905m0eYhkql4olaTyjrcyPm3rKPWqdmyq4bmYFP13uaMNcwk87DxcaFHzr9gIu1CwB5mjxlW5sqbUx6LiGEEEIIIcS9p0y3fo0aNcpU8xBCCHEfKNBomfTXca7HBF7uHE5VTweTHPvlLuEk5xSyYF8sGp2eCfMOMX9scxoHuZnk+PeDvEItz8w7xLazhsbiluYqvhnakO51fRnSNIAANzu+3XwegN92RXM5PY+vhzbAxrJ05TgPxqRx4aohc6BZiBuhRv4stTotU3bdCAqMqj2KOh51ihllnBG1RpCan8qsE7PQo+etnW/hbOVMqyqtjBr/Uudw9lxI4XCsof/MuytO8vWQBiW+mF6g0fL2ihv9RiZ1r4H3tUyH8qbWqnlz55usv7geMAS/3m/5Pv3C+qHX6/F38OeLQ18AsOL8Cq7kXOHLR77Ewap078Xzaec5evUoYMiAqutR1yTPozKNqDWCPQl72BW/i5T8FN7d9S7fd/oelUrF8GaBfLPpHHlqLX8evMQrXarjbHcjsPjFhrPEpl7LDAx25nDu98oF+vH1xxPoFGiSOdbyc+K74Y14as4B5TO1Rag7dlbll40lbugV2otvDn9DWkEaGy5u4OXGL+Njf6Nk5G8nfuNM2hnA0BtrbJ2x5TKPQKdAZnSYwdgNY5XP1DrudfCw9SiX8wkhhBBCCCHuHWX662/27NmmmocQQoj7wHebz3M+KRuAulWcGdfWdL1hVCoV0/rWIT23kLUnEslX63hy9gGWPNOK6j4PfnPsrHw1T/1+kP3XGoXbWJrx84gmtA/3BAzfn1e7Vsff1Za3lp9Eq9Pzz6lEhv2yl1kjm+DuYF3icy7aH6csD2tmfNbM/Mj5HE82ZBYEOwUzsb5xvWGM9ULDF0jLT2PpuaVodBpe2voSs7rOop5nvWLHWpqbMWNoQ3rM2EFWgYaVRy/TLsyTAY39SzSHn7ZGEXUtcNUgwIXHm5nmgnxx8jX5vLL1FXbE7wDAQmXB9HbTeTT4UcDwOhhdZzQ+Dj68veNtCnWF7EnYw8h/RvJDpx+KXFw21rLzy5TlgeEDH4h+J2YqM/7X+n8M/HsgKfkp7IjfwfzI+TxR6wmc7SwZ1MSfuXtiyFNrWbA/lgmPGDKuDsWk8du1bBprCzPaNz7HzIiTAIQ6h/Jk7SdNOs8ONbyY2qc27640lNXqXc/XpMcXd2ZjYcPg6oP5+fjPaPQaFpxewCuNXwEMAcufjhvKCZqrzPmg9Qdlzgy8m0bejfhf6/8xecdkALqFdCu3cwkhhBBCCCHuHWUqayaEEOLhcTI+gx+2XgAMGR2fDaqHhblp/xsxN1Px1ZAGtKlmuGM4M1/DyN/2EXftLvYHVVpOIY/P2qcEZhysLZg7prkSmLnZkKaBRcohHYlNp/+Pu4m6ml2ic2bkqVlz4jIATjYWdK9j3EXh2MxYvj3yLQAqVHzQ+gNsLEybUaJSqXi3xbt0DuwMGEr9TNw0kQvpF4waH+Bmx4f9b2R/vLvyJNF36C1yO1FXs/l+iyFDydxMxcf965a5dJ8xctQ5TPh3ghKYsTa3ZkbHGUpg5mbdgrsx69FZOFs7A3Au7RyPr3mcM6lnSnTOQm0hqy6sAgwlu3qF9irjs7h3eNh68GGbD5X1Lw99SWRKJABPtg7hegxqzu6LqLU68tVaJv11jOttfJ7u4M78cz8r46e0nFIuF+hHtAxm7phmzBjagIElDCKKshlaYyiWZoaf6V9n/yJXnWvIDNw9BY1OA8Do2qOp7V673OfSI7QHsx+dzf9a/48naj5R/AAhhBBCCCHEfU+CM0IIIYql1uqY9NdxpcfJxEeqUcPHqVzOZW1hzk8jGlPf33DR+UpmAaN+209uoaZczlfZkjLzGTJzD8cvZQDgamfJgnHNaRZy53Ju7cM9+fOZlng7GbJlYlJyGfDjbg5eC+4UJyNPzdzdF8lX6wDo17CKUaXRdHod7+1+j3xtPgDDagyjoVdDo85ZUuZm5kxvN51mPs0Mcy7IYPzG8WQUZBg1vk99PwZdu9CdW6jlhYVHKNToih2n1+t5e/lJCrWGfce2DaGmb/m81m+WUZDBuA3jOHjlIAB2Fnb82PnHuzYFb+jVkHnd5+HvYHieSXlJjPpnFLvidxl1zhx1Dn+d/Yv0gnQAOgd1VoI9D4rWVVozqpahDK9ap2bS9knkqnMJ8bCnc01vABIz81l7IoFvN59TyvzV93cmVrWAHLVhfUDYABp7Ny63ebYL96RvgyoPRNbS/cTD1oOeoT0ByCrMYsX5FcyLnMeJZENJw2CnYCY0mFBh82ni04S+1fpibla6UpVCCCGEEEKI+4vJi1pfuXKFkydPkppquEDk5uZGnTp18Pb2NvWphBBCVJCZ26OISMgEoLq3I892qFau53OwtmD2k80Y+NNuoq7mEJWcw4xN53ize81yPW9Fi0vN5Ylf9xGTYsgM8nK0Zt7Y5oR7F1/GrbafM8sntmbM7wc4nZhFWq6a4bP28dXgBvSs50tOgYbo5BwupuRwMTmH6ORcLqbkEJ2cQ2pOYZFjDTWyZNdfZ/9SggdVHKrwYqMXS/iMS8ba3JoZHWYwZv0YIlMjuZJ7ha8OfcX7rd43avz7fWpzKCaNqOQcTsRn8MWGM7zZ4+6voaWH49kTZWge7+9qy4udTNsA/HaS85IZt2Ec59MN2TpOVk781Pkn6noW3/sl2DmY+T3n8/ym5zmefJwcdQ7PbnqWKS2n0D+sP3maPGIzY4nNiiUmM4aYzBhiMw3LKfkpRY41MHxguTy/yvZioxfZn7ifyNRILmZe5JMDnzC11VSeahPCxogrAHy+4QyX0w1BR0tzFYPaZfLJkU0AuNm48XLjlytt/qJ8jag1ghXnVwDw28nflGClChXTWk/D2rzkJSOFEEIIIYQQwhgmCc7o9XpmzpzJd999R0RExG33qVWrFs8//zzjxo2TuwKFEOI+cu5KFjP+PQeAmQo+G1QPK4vyT7x0s7di1sgmdPt6B4VaHbN2RPNYgyoVksVQEc4nZfPErH0kZhouCPu72jJ/bHOC3O2NPoafiy1/PtOSifMOs/N8MoUaHc8uOMzUVdYkZRUYdYxmwW5GfU8TshP44uAXyvr7rd7HztLO6LmWloOVA992/Ja+K/uSo85h6bml9K7a26gsBntrC74Z1pB+P+xCrdXz8/YoWlfzoN1tysUBpOYU8uGaG7/HTHusTrk3Z7+cfZlxG8YRmxULgLuNOzO7ziTcNdzoY7jZuDHr0Vm8ueNNNsVuQqvX8t7u9/juyHdczbtq1DGquVSjiXeTUj2He52luSWftvuUwasHk6fJY9m5ZbTya0XXkK7UqeLEyfhM4lLzlP3Ht/dnztlXlfXXm77+wGUUiRvCXcNp4duCvQl7uZJ7RXn88ZqP08CrQeVNTAghhBBCCPHAK/PVtbS0NNq2bcvEiROJiIhAr9ff9isiIoIJEybQrl070tPTTTB1IYQQ5U2r0/P6X8eVEk9Pt6tKPX+XCjt/qKeDkqWj1el5c9kJdNdKq93PNp++wpCf9yiBmaqe9ix5pmWJAjPXOdlYMvvJpkV6VdwtMOPtZE2LUDeGNQvgnZ41+fGJRsWeQ6/XM3XPVHI1hgyfAWEDaOHbosRzLS1ve29eaPiCsv7Bng9Qa9VGja1TxZnJ3Woo66/8eYzk7Nt/fz5aG0laruG4Pev50qG6VxlmXbwDiQcYuW6kEpjxtfdlTvc5JQrMXGdrYcsX7b8o0qviboEZD1sPGnk1ol+1frzc+GV+6vzTA33zTLBzMG81f0tZn7p7Kgk5CYxtE1pkvxo+jmic/yExJxGAlr4t6RnSs0LnKireyFoji6xXcajC8w2fr6TZCCGEEEIIIR4WZbodVK/X07dvX3bv3g2Au7s7gwcPpnnz5vj4+KDX67ly5Qr79+/nzz//JDk5md27d9O3b1+2bdtmkicghBAPKq1Oz9YzSdT0dcLPxbZS5vDrziiOxqUDEOppz0udy7/E038980gofx+L58LVHI7GpTN/fywjWgRV+DxMIS2nkKmrTrHi6GXlsVq+Tsx9qhkeDrcvnaPX69l1eRdBjkEEOAXcdh9LczM+G1iPEA97Zmw6h5ONBcHu9oR42BPsce1fd3uCPexKlQmy4vwKdl029DHxsvPi1SavFjPC9IZUH8LqqNWcSD5BVEYUs0/N5ul6Txs1dkzrELafS2b72askZxfw+pJj/Da6aZFgxJ4LKfx16BIAjtYWvNerVrk8D4Dswmy+PPQlS84uUR4Ldgrml66/4GPvU+rjmpuZM7nZZPwd/ZlxeAY25jYEOgUS5BREoOO1f6+t21uWPBB4v+tbtS+743ez7uI6stRZTN4+mZ87/4qPkw2JmfmYm6l49lFb3jkwHzCU1Xu3xbsPdNBKGLSu0ppQ51CiMqIAmNpqaoVkBgohhBBCCCEebmUKzixYsICdO3eiUqkYPnw4P/zwA46Ot9bJHzlyJNOnT+fZZ5/ljz/+YOfOnSxcuJBhw4aV5fRCCPFA+3htJLN2RmNvZc7Cp1tUWMaKRqtjQ8QVZu+K5sDFNABUKvh0QD2jmsabmrWFOR/1q8uQmXsB+HTdabrW8sbbyabC51IWa08kMGXlSZKzb/R7aR/uyTfDGuJsa3nHcb+c+IVvj3yLlZkVP3X5iaY+TW+7n0ql4tkO1Zj4SFWTXEzW6XXsuLSDeZHz2JuwV3n8vZbv4WhVfE8cUzM3M2dKyykMXT0UrV7Lz8d+pltwNwKdiu+XY2am4otB9ek+YzvJ2YVsOXOV2bsuMqZNCAAFGi1vLz+h7D+pew28yun1tePSDqbumVqkfFIjr0Z88cgXeNh6mOQcj9d8nOE1hktQ4T9UKhXvtnyX48nHic+O5+jVo8yO+IVvhg3j283nGNDYj/kXXkenN2QKjq83/o4BUfFgMVOZMb3tdGYcmUGnwE40921e2VMSQgghhBBCPATKVNZswYIFALRv354//vjjtoGZ6xwcHJgzZw7t27dHr9czb968spxaCCEeaAkZeczdEwNATqGW0bMPcOFqdrmeMyNXzc/bLtD+s61MnH9YCcyAIfOgSbBbuZ7/bpqHujO4iaFsV1aBhg9W3b6/2b0oKSufZ/44xMT5h5XAjJONBZ8Pqs/vTza9a2AmoyCD307+BkChrpDnNz9PZErkXc9X1gvyOeoc5kfOp8+KPjy3+bkigZk+VfvQzr9dmY5fFjXcajCi1gjA8P2Ytncaer1xZe48Ha35YnADZX36utOcjM8A4MetF4hKzgGgQYALjzcrPuBTUhkFGby9820mbpqoBGZsLWx5q/lbzO4222SBmeskMHN7jlaOfNLuE8xVhkDzzOMzMbON4o+nmpNjvZ1TKacAqOpcldG1R1fiTEVFq+lek586/8Sg8EGVPRUhhBBCCCHEQ6JMwZnDhw+jUql47rnnjB7z/POG+s1Hjhwpy6mFEOKB9vO2KKXPCxgalY/8dT8JGXl3GVU6F65m8+6Kk7T4eBMfrztNfPqNc4R5OTC9f13e7lHT5OctqTe718TN3gqANScS2Hz6SjEjKpder2fpoUt0+XI7/5xKVB7vWsubf19pz8DG/sVeQJ8fOZ8cdY6ynqPO4Zl/nyEmM8bk872UdYlPD3xK5yWdmb5/epFzBDgG8EazN5jaaqrJz1tSE+pPwNfeF4C9CXtZE73G6LHtwz0Zey1bplCr44VFRzgZn8EPWy4AYG6m4uP+dTEzM21gY2PMRvqu6MvfF/5WHmvp25LlfZczrMYwzFRlbgEoSqC+Z30mNpgIGDLE3tjxBmfTzvLtkW+Vfaa0nIKl+Z0Dp0IIIYQQQgghRFmVqaxZamoqACEhIUaPub7v9bFCCCGKSsrMZ8F+Q4NwW0tzgtztOJ2YRXx6HiN/3c+f41viei1IUVp6vZ7t55KZvSuarWdubRresYYXT7YOpk01j3vmDnxXeyve6VmTV/48BsC7K07R4hX3UvVQKW/x6Xm8tewE287e+N6621sxtW9tetb1Nep7mlWYxbwIQ5aphcqCcLdwIlIiSM1PZfzG8cztPhcvu7I1rNfr9Ry8cpD5kfPZErdFKed0XXPf5oyoOYK2/m3vmQCCnaUdbzd/m+c2G24M+ezAZ7St0hZna2ejxr/erTp7o1M4GZ9J1NUcBvy4WwmEjm0bQk1fJ5PNNSUvhU93fcrGmI3KY46Wjrze9HUeq/bYPfPeehg9Vecp9ibs5UDiAa7kXmH4muEUaAsAGBA2gEbejSp5hkIIIYQQQgghHnRlutLi7Gy4EHL58uVi9rzh+r5OTqa7+CGEEA+SmdujKNQYLhaPaBnE3KeaEehmaEx8LimbMXMOkFuoKdWx9Xo9q45dpstX2xn12/4igRk7K3NGtgxi86vt+W10U9qGed5zF4/7NaxCq6rugCEAMuPfc5U8o6J0Oj3z98Xw6FfbiwRm+jbwY+Mr7elVz8/o7+nC0wvJUmcB0Ltqb2Z2mUmYaxgA8dnxjN84noyCjFLPdWvcVgavHsyY9WPYFLtJCcxYmVkxIGwAS/ssZVbXWbQPaH/PBGauax/Qni5BXQBIzU/ly0NfGj3W2sKcb4Y2xM7KUNaq4Np7zd/Vlpc6hZtkfnq9nqOFRxm4ZmCRwMwjAY+w4rEV9Avrd8+9tx425mbmfNzmYyWodz0w42bjxsuNX67MqQkhhBBCCCGEeEiU6XbjOnXqsG3bNmbPnk3Pnj2NGvPbb78pY4UQDwaNVkdartqofVUqQwaBXJi8veTsAubtM5STsrYwY2zbELwcbfjjqWYM+HEPydkFHIlNZ8K8w/wysglWFsZfNE/MyOedFSf4NzKpyONVXGwZ3SqYwU0D7tr/pDhanZa0grTid7zGzcatxBf9VSoV/3usDt1m7KBQo2PWzmj6NqhCLb/KDfhn5atZdyKRhQdiORKbrjzu7WTNh4/VpXMt7xIdL0edw9yIuYChUfXYumNxtnbmp84/MXLdSOKz4zmffp7nNz/Pz11+xtbC1uhjp+anMn3fdNZdXFfkcU9bT4bWGMrA8IG42VRefyFjvdHsDXZf3k2OOodl55bRO7Q3TXyaGDU21NOBqX1q8/pfx5XHpj1WB9trAZvSylXnsil2E8vOLuNg7kHlcVdrV95s/ibdgrvJZ989xNvemw9afcCLW15UHpvcdLLRWVhCCCGEEEIIIURZlCk4M3DgQLZu3cry5ct5//33ee+99+540UGv1zN16lSWL1+OSqVi0CBptinEg+DC1WyG/LyX5OwCo8eEezvw3fBGhHs7luPM7k+/7IgiX224k39480C8HG0ACHK3Z+6YZgyZuYesfA3bzl7ltSXH+HpIg2L7Y+j1ehYfiOPDtZFk5d/IuGka7MpTbULoXNMbC/OyZUYkZCcw6p9RJOQkGD0mwDGAT9t9Sh2PkgXrQz0deK5DNb7ceBatTs9by0+wdEIrzE3cJ6Q4aq2OHeeusuxwPBsjrigZGNcNaRLAWz1rlirgtej0IiUrpmdITwKdDA3qvey8mNllJiPWjSA1P5UjSUd4bdtrfN3hayzN7n4evV7PPxf/4eN9HxcJotV2r82IWiPoGtT1vuqx4WXnxYuNXuSjfR8B8MHeD/ir919YmRtX8m9gY39OxGcwd08Mo1oG0aF66UrEaXVa9iXuY/WF1fwb+y95mqJ9oboHd+eN5m/cFwGvh1HHwI48Xe9pZh6fSa/QXnQP6V7ZUxJCCCGEEEII8ZBQ6fV6fWkHq9Vq6tevz+nTp1GpVNSqVYvRo0fTvHlzvL29UalUJCYmsm/fPubMmcOpU6fQ6/XUrFmTY8eOYWFx7/UJKInMzEycnZ1JTk7G3d29sqcjRIXT6vQM/Gl3kUwBYznaWPDziMa0quph+olVArVazdq1a+nRoweWlqW7wJ2aU0ibTzaTW6jFysKMHZM64O1kU2Sf/dGpjPh1nxIIGN0qmPd617pjYDwuNZc3l51g5/lk5TFPR2um9a1Dtzo+pZrnf+n1eib8O4Fdl3eVeKythS2ftP2EDoEdSjSuQKOlx4wdXLiaA8AHfWszsmVwic9fUnq9nhPxGSw7HM+qY5dJySm8ZZ9qXg6817sWbcM8S3WOXHUu3ZZ2I60gDRUqVj62khDnor3dIlMiGbN+DNnqbAB6h/bmf23+d8dMpKTcJKbtncbWuK3KY87WzkxuOpleob3u22wOrU7LiHUjOJF8AoDnGjzH+PrjS3SMvEItNpZmJf4enEk9w+qo1ayNWktSXtIt213NXHmn9Tt0De1aouOKypGvycfa3Pq+fS+Ie5cpfj8QQjwY5PNACHEz+UwQ4sF2PW6QkZFx1/YuZYqOWFpasm7dOjp27Eh0dDQRERFMmjTpjvvr9XpCQ0NZt27dfR+YEULA7F3RSmDG28maev4uxY65cDWbqKs5ZOVrGPXbfj4dWI9+Df3Ld6L3id92RpNbqAVgaNOAWwIzAM1C3Ph+eCPGzzuEVqfn990Xcbe34vlOYUX20+n0zN1zkU/Xn1GOCYZsgXd71sLZznS//P194W8lMONm40Z9z/rFjonLiuN8+nnyNHm8tPUl3mj2BsNqDDP6nNYW5nzUry5DZu4F4NN/zvBobZ/bfs9M4VJaLiuPXmbZ4UtKQOhmbvZW9K7nS79G/tT3dy7TBd4lZ5comS3dQrrdEpgBqOlek286fsMzG5+hUFfIqqhVuNi48HqT14ucW6/Xs+L8Cj478JnSvwagS1AX3mr+Fh6293dw1NzMnPdavseQ1UPQ6rXMPD6TbiHdCHIKMvoYJSlllpSbxNqotayKWsXZtLO3bHe0dOTRkEfpHtidywcu0yGgZEFHUXlsLMrns0MIIYQQQgghhLiTMkdIgoKCOH78OO+//z6//vor6enpt93PxcWFsWPHMmXKFBwcHMp6WiFEJYtJyeHzDWcAQx+Z74c3oklw8WV7cgo0vLDwCJtOJ6HW6nl58TEupebxXMdqD/Udyxm5an7ffREAS3MVz7Svesd9O9fyZnr/ukq/jC82nsXV3oonWhguSF+4ms3kv45zMOZG6So/Zxs+6l+XR0pZuulOruZe5ZMDnyjrH7T6gPYB7YsdV6gt5J1d77Aueh06vY6P9n1EfFY8rzR5xeg+NM1D3RnSJIDFB+PILtAwddUpfni8camfy+1odXreXHacPw9eumWblYUZXWp6069hFdpX98SyjKXhwHD3/uyTs5X1p+s+fcd9m/o05dP2n/LK1lfQ6XX8EfEHbjZujK07FoDL2Zd5f/f77EnYo4xxs3HjnRbv0CWoS5nneq+o7ladkbVGMvvUbAp1hUzbO41fuvxi0s8TvV7PZwc/Y37kfHT6ouXrLFQWtPFvQ+/Q3rQPaI+1uTVqtZoElfEl/oQQQgghhBBCCPHwMUn6ir29PZ999hkffvghhw4d4uTJk6SmpgLg5uZGnTp1aNy4MVZWxtWBF0Lc23Q6PZOXHld6o4xqGWxUYAbA3tpQzuz9VaeYtzcWMAQXLqXl8b9+dUxygft+9NuuaLILDP1gBjYOwM/l7g3eBzUJIC23kI/Wngbg3ZUncbK1JD4tj6/+PUvhTf1PnmgRyORuNXC0MW2qtF6vZ9reaWQVGjIyeob2NCowA2BlbsX0ttOp4lCFWSdmATAnYg6Xcy7zUZuPjL6L/c0eNfg38gopOYWsPZHIpsgrdKrpXbondBu/7Yy+JTDTLMSN/g2r0L2ub6n6ydzN0nNLSclPAQzZLdVcq911/06BnXiv5Xu8t/s9AGYcnoGztTManYavD31NriZX2bd3aG8mNZ2Ei42LSed8L3im/jOsv7ieyzmX2Zewj9VRq+ldtbfJjr/i/Ar+iPijyGP1POrRq2ovugV3w9XG1WTnEkIIIYQQQgghxMPBpLXFrKysaNmyJS1btjTlYYUQ95gF+2PZG2UIwPq72vL6o9VLNN7C3IxpfesQ4GrHx+sMwYXFB+O4nJHHD483MnkQ4V6Xma/mt13RAFiYqZj4yJ2zZm72dLuqpOQU8vO2KPR6eGHhkSLbg9zt+GRAPVqElk9PrPUX17MlbgtgyMh4o+kbJRpvpjLjxUYv4ufgx4d7P0Sr17IxZiNXc6/yTcdvjLrg7WJnxTu9avLy4mMATFl5ipZV3bGzKvt/b2cSs/hs/Y3ssBc6hjGwsT8BbnZlPvbtFGgL+O3Eb8r6+HrG9U7pH9aftPw0vj78NQAf7PmgyHZvO2+mtJxCO/92JpvrvcbO0o63W7zNs5ueBeCzA5/RtkpbkwSi4rPji2SHPVn7SfqH9SfYObjMxxZCCCGEEEIIIcTD6+G8RV0IUWrx6Xl8vDZSWf9kQD3srUt+IVylUjG+fVW+G94QKwvDR9GOc8kM+mkPCRl5Jpvv/WDu7otk5RuyZvo3qlKii/9vdKvB4CZFe/aYqWBc2xD+ebFduQVmUvNT+Xj/x8r6283fLvWF8EHhg/i247fYWhiyhY5ePcqIdSOIzYw1avxjDarQuprhecan5/Hlhlt7gZRUoUbHK38epVBryEAa2yaEl7uEl1tgBmDFuRVKY/mOAR2p7mZ80HNMnTGMqjXqlscHhQ9iRd8VD3Rg5rp2/u3oGtQVgLSCNL449EWZj6nT63hn5zvkqA19hh6r9hivNHlFAjNCCCGEEEIIIYQoMwnOCCGMptfreWvZCXKuNZgf1iyA1tXK1lC8Vz0/5o9tjsu1BvWnE7Po9/1uIhMyyzzf+0F2gYZZOw1ZM+ZmKp7tcPcyVv+lUqn4qF9detXzBSDMy4G/JrTi7Z61StTovKSm759Oar4he6pLUBe6Bnct0/Ha+rfl926/42nrCUBMZgxPrH2Co0lHix2rUqn432N1lSDfrJ3RrDwaX6b5fLv5HKcuG16DYV4OvNq1ZNlhJaXWqpl1cpayPr6+cVkz16lUKl5p8goDwwcC4O/gz69df2VKyyk4WD08fd4mN5uMg6Xh+a44v4LFpxeX6XjzIuZx8MpBAPzs/ZjcdHKZ5yiEEEIIIYQQQggBJShrtn37dpOfvF27B/9OXiEeJEsPx7Pt7FUAfJxseLNHTZMct2mwG0sntOLJ2QeITc0lMTOfQT/t4YfHG9Eu3NMk57hX/bEnhvRcNQB96/sR5G5f4mNYmJvx3fBGvN0zD29HG8zMTNcI/Xa2xG5hXfQ6AJysnHir+VsmOW4t91rM7zGfiZsmcj79PGkFaYzdMJbpbafTOajzXceGeNgz6dHq/G+NIavr9SXH8XOxpamRvZBudjg2je+3nAcMZea+GtIAG8vyC3QBrLywksScRMCQAVLLvVaJj2GmMuO9lu/xTL1n8LD1wNysfOd8L/Ky82Jys8m8u+tdAD7a/xF+Dn609W9b4mNdSL/AjMMzlPX/tfnfQxXoEkIIIYQQQgghRPkyOjjzyCOPoFKZ7oKfSqVCo9GY7HhCiPKVlJnPB6tOKesf9quDkwl7w1T1dGDZxFaMnXOQo3HpZBdoGPP7AT4ZUI8Bjf2LP8B9KLdQwy87ogBDT5NnO5Ysa+a/fJ1tTTGtu8oszGTa3mnK+hvN3sDDtmzZUzfzdfBlTvc5vLLlFfYl7qNAW8ArW19hcrPJPF7z8buOfapNCOeTsll0II5CrY6n5x5k+cTWBHsYH/DKK9Ty6p/H0OkN6y92CqNOFeeyPKViqXVqZp24KWvGyF4zd+Jt713WKd3XHqv2GFEZUcw+ORudXsdr215jbve5JSoTp9apeXPHmxTqCgEYUWsETX2alteUhRBCCCGEEEII8RAqcVkzvV5vsi8hxP1Br9fzzoqTZF7ri9KvYRU61TT9BWAPB2sWjmtB11qGY2t0eiYtPc6Fq9kmP9e9YMG+WFJzDBd/e9fzo6rnvX9X/ucHPudqniF7qm2VtvQK7WXyczhZOfFj5x/pU7UPAHr0TN8/nRNXT9x1nEqlYtpjdWhzrdReWq6aJ38/QNq177Expq+LJDrZ0F+kQYALEx6pWspnYbw1UWuIzzaUYWvt15p6nvXK/ZwPupcavUSXoC4A5GpyeXbTsyTlJhk9fubxmUSmGrKwQp1DeaHhC+UyTyGEEEIIIYQQQjy8StzF29bWlr59+9KlSxfMzKRljRAPgzUnEtgQcQUADwcrpvQqecklY9lamfPjE415Z8UJFu6PQ6vT88m608wc2aTczlkZ8tVaftp2I2vmuTJmzVSE3fG7WX5+OQD2lvZMaTnFpBmVN7M0t+R/rf+Hu407s0/NBuCLQ18w+9HZdz2npbkZPzzRiAE/7OZcUjbRyTmMn3eIP55qhrXF3ct87Th3lTl7YgCwsTTjy8H1sTAv3//nNDoNvxz/RVl/pv4z5Xq+h4WZyoyP2nzElZwrHE8+zpXcKzy36Tl+7/Y7dpZ2dx174uoJ5WdiobLgo7YfYWNhUxHTFkIIIYQQQgghxEPE6OCMo6MjWVlZ5OXlsXjxYrZu3crw4cMZMWIE9evXL885CiEqUUp2Ae+tvFHO7IO+dXC1tyrXc5qbqXi3Vy02RSaRlFXAhogr7I9OpVlIyfuH3KsW7o8lObsAgO51fAj3dqzkGd1djjqHqXumKuuvNnkVH3ufcj2nSqXi+UbPszluMzGZMRy6coitcVvpENjhruOcbCz5bXRT+v2wm+TsAvZHp/LG0hN8Obj+HQM7GblqXl9yXFl/q0dNQisgk2ld9Dpis2IBaO7TnAZeDcr9nA8LGwsbZnScwRNrnyA+O57I1EgmbZ/EjA4z7tiPJ0+Tx1s730Kr1wIwvv54arvXrshpCyGEEEIIIYQQ4iFh9C3BV65cYeHChfTo0QNzc3MSExP56quvaNSoEfXr1+fzzz/n8uXL5TlXIUQlmLoqgpRrZaG61/GhR13fCjmvnZUFr3QJV9Y/Whv5wJRDNGTNXFDWn+sQVomzMc7Xh77mco7hM765T3MGhg2skPNamlnyUqOXlPWvDn+FRld8v7IANzt+HdUEG0vDf3PLj8Tz9b/n7rj/e3+fJDEzH4C2YR480TyobBM3glanZebxmcr6+Ppl6zUjbuVh68H3nb7H0dIQ/Nx2aRufHfzsjvt/fehrLmZeBKCuR13G1h1bEdMUQgghhBBCCCHEQ8jo4IyNjQ1Dhgxh9erVxMfH89VXX9GwYUP0ej0nTpxg8uTJBAUF0aVLF/744w9ycnLKc95CiAqwMeIKfx8zXJB3sbNkat+KvYN8YGN/wr0N2QtH49JZeyKxQs9fXpYcusSVTEPWTNda3tTyc6rkGd3dwcSDLDqzCABbC1vea/VeuZUzu51OgZ1o4NkAgOiMaJadW2bUuPoBLnw9pCHXpzpj0zmWHrp0y35rjiew4qjhde5oY8GnA+thZlb+z29jzEYlENDYu7E0nC8nVV2q8mWHL7FQGZKF50fOZ37k/Fv223N5DwtOLwDA2tyaD9t8iIVZiau/CiGEEEIIIYQQQhilVMX0PT09efHFFzl48CCnTp1i8uTJ+Pv7o9Vq2bRpE6NHj8bb25sRI0awfv36B+ZudyEeJhl5at5efqMB+5RetfByrNi+CxbmZrzRvYay/un60xRqdBU6B1Mr0Gj5cct5Zf2FTvd21kyeJo/3dr+nrL/Q8AUCHAMqdA4qler/7d13eBRl28bha3fTSUih9y69SlFQAaVIEQEFpKugAqIUBbGDFTuWFxU+6SjygqIUgSBNXlC69N5LgJBK6iaZ748lS0I6ZDeF33kcOdjdmZ25J6yPYa48z62Xmr5kfz5191RFWbP3CwAP1yut1zvXtj+f8Mse/X3iqv355YhYvbHkxuf83UfrqYyvZy5UnbnEpER9v+d7+3N6zTjWPWXu0Vv3vmV//vG2j7X+7Hr784j4CL35vzftz8fcPUZVfKs4sUIAAAAAAHCnue1Ox7Vr19aHH36o06dPa+3atXryySfl4+Oj6OhozZ8/X507d1a5cuX0yiuv5Ea9AJzk/eUHdDnSNrujbc0S6tG4XJ7U0bZmSd1btZgk6fTVaM37+3Se1HE7DMPQvvPhemfpAbWavFYXwm3LZz1Yq6TqlfPN4+oyN3X3VHtPlEYlGqlvrb55Ukejko3UvlJ7SdLV2KuatX9Wtt875L4qGnBPRUmSNdHQc3N36PiVazIMQxN+2avQaKskqXP90nq0Udlcrz2lo6FH9fmOz9VhcQcdC7OFdA1LNFSL0i0cel5IPWr00DP1n5EkJRlJGr9xvPZftfXTmvzPZF2KviRJalGmRZ59zgEAAAAAwJ0jV9fraNOmjdq0aaOpU6dqyZIlmjt3rgIDAxUUFKSvv/5aH330UW6eDnnsaPBFjQ38WJdisnez3GQyqX5Ac33TeYw8XB3bUB63Z+vJEC3cblv+ycfdRR/0rJ/hMlaR8ZGaunuqDlw9kK1jm0wmNSvdTM/Uf0Zulqw/ByaTSa91rq1HvtkkSfpq7VE9dnd5+Xq6ZvNq8s6FsBgt2X1ev+48r6OXr6Xa5moxaXS7/D1r5nDIYc05MEeS5GZ206RWkzJspO4Mo5qM0roz65RgJGj2/tnqdVcvlfQqmeX7TCaTJj5SV2dDYrThyBWFx1j11MxteqJ5Ba09dFmSVNzbXe91z/hzfjuCY4K1/MRyLTuxTIdCDqXaZjaZNbLxSKcuE3cnG9l4pM5GntXKUysVkxCjF/58QU/Xe1pLTyyVJPm4+ui9Vu/JbLrt310BAAAAAADIlEMWUzeZTDKbzTKZTNxwKqT2BJ3SoOVDlOhyWcrBvdp/wo6o/fy9WtL7WxXz8nFcgbhliUmGJi3db3/+SqdaGS7zFBIbomGBw3Qw5GCOzrHj0g7tuLRDX7T5Qr7uWc8cqV/eV90bldWS3RcUFm3V1PXH9Gqn2lm+Ly9ExiZozb9B+nXnef198qpuXtXRzWJWuzolNeS+qmpQ3i9PaswOwzD08baPlWTYlpEb1nCYqvpWzdOaKhWtpN41e+vHQz8qJiFGU3dP1cSWE7P1XheLWd/0a6xe323RoaBInQmJ1scrD9u3f/x4fQUUyb3QONoarbVn12rZ8WXacnGL/ftor8fkovvK3acBdQaoRRlmzTiL2WTWe/e9p6CoIO2+sltXYq7oo203fnHk1RavqnSR0nlYIQAAAAAAuFPkajizYcMGzZ07V4sWLVJkZKQk2w2+MmXKaODAgbl5KuShv88c1rOBz8pwCbml94eZ/lWHBU9o/iPfq1aJ8rlcHW7Xoh1ntf9ChCSpTpmi6tu8Yrr7XYq6pGcDn9WJ8BO3dJ5tQds06I9Bmtpuqsp5Z71k2ssda2rFviDFJyRp5v9OaeA9lVTe3+uWzp3bEhKTtP7IFc0+Ytb4besVl05fnGaV/dWzSXl1rl+mQMz6WXtmrbYGbZUklfcur8F1B+dxRTbDGg7T78d/1zXrNf167FcNqD1A1f2rZ+u9Ph6umvFkM3X/z//sS/ZJUt/mFfRgrVK3XVtiUqK2XdqmpceXas3pNYpOiE6zT/3i9dW1alc9XOVhBXgE3PY5kXPuFnd9+eCX6r+8v85dO2d/vX2l9upatWseVgYAAAAAAO4ktx3OHDx4UHPnztX8+fN17pztJodhGPLy8lKPHj00aNAgPfTQQzKbWSKkMFhz7F+N2ThCcrHdvDcnFNe09tNUv0z6N/BT+unf9Zqy5w3JEqt4yxn1/r2/prT5Wg9Wa+DospFNkbFWfbLqiP35W4/UkcWcdvbb2cizemb1Mzp/7bwkqaRXSX3f7ntV8q2U5TkOXD2gF9e+qJDYEJ0IP6H+y/vrP+3+o7rF6mb6vvL+XnqqZWV9v/GE4hOS9NnqI/qiT6OcXWAusvWRidAvu85p6b8XFHwtXrY2XjeCmSrFi6hH43Lq0bicKgTkjyApO+IT4/Xp9k/tz19u+nK2lqBzBn8Pfw2pP0Rf7vxSSUaSPt/xuaa2m5rt95f189SMJ5up9/dbFB2fqAoBnnq9S53bqulI6BEtO75My08u1+Xoy2m2l/Mupy5Vu6hr1a40mc8nAjwCNLXdVA1YMUAR8REK8AjQG/e8wWxfAAAAAADgNLcUzly+fFk//fST5s6dq127dkmy3ag0m81q27atBg0apJ49e6pIkSK5Wizy1pIDf+vNv0dJFttvg7sklNWPj/yg2iWzN/tlSNOOquRXWmM3vCDDJVSGS4hGbRiql8M/1OAmDzmydGTTf9YdV/A124yCTvVK656qxdLsczzsuJ5d/awux9huQpf3Lq/pHaarvE/2PgcNSzTUvM7zNGLNCJ2KOKWrsVf11Mqn9MkDn6h1hdaZvndE2+r6eftZhUVb9euu8xpyXxXVK5f1smi56XxYjJbsOq9fd53XsZv6yEiSv5erHmlYVj0al1OjCn4F8mbvvIPz7DMKmpdurgcrPpjHFaU2oPYA/Xz4ZwVFBemv83/pn4v/5GhpsHrlfLXwuXu1fO9F9WteUd7uOf9f4ZXoK1pxckW6fWQkW++SDpU76JFqj6hxycb0MMmHqvhW0ZxOc7T0+FJ1q9aNmUwAAAAAACBjSYlSbLgUEypFh9j+jAlJ8TzF45Ar2Tpktu9IxcbGasmSJZo7d64CAwOVmJgo43ozhXr16mngwIHq37+/ypYte2sXh3xt7q61+mj3eJksthv37omVtKjHDFUOyLoZd0rtqjfUz0V/VP9lz8pqOStZYvTJnpd0Kuwlvf0gS98lMwxDx69cU2lfz1u6cXwrTl+N0oxNJyXZ+qK81jltT5cDVw9oWOAwhcaFSpKq+VbTtA7TstWUPaUKPhU0r/M8vbj2Re28vFMxCTF6cd2Leq35a+pTq0+G7/P1dNULD9bQu8sOSJI+/OOg5g1p4fAAJDLWqj/2BumXXef0z8mQtH1kXMx6sGYJlU+4oDFPtFMRT/dcO/ep8FMK8AxQUbeiuXbMzATHBGvanmmSbP05xjcbn+8CJg8XD73Q+AW9vul1SdJn2z/Tgq4LchSA1Cvnm+NgL1t9ZMrfp0eqPqLWFVrL3ZJ7nwM4RjW/ahp99+i8LgMAAAAAADiLYUhxkTcFK6FZhy4xYZKMrI5uE5e9/bJ917dkyZKKioq6Xr+h0qVLq2/fvho4cKAaNWqU3cOgAJr6zzJNPfCmTOYESZJXUg391usHlfbxv6Xj1S5ZXqt6L1CPRcMUbtorkylRi85+rDNLzmt6t/F3/BJ4SUmGxi/eo0U7zqmIm0W9mlbQky0rq3Jxx85E+2DFQcUn2m42D7m/SppluHZe2qnn/3xe16y22SJ1itXRd+2+k7/HrX0OfN19Na3DNL2x6Q2tPLVSSUaS3vvnPZ2/dl6j7x6d4Y32gfdU0uzNp3QmJFr/O3ZV649cUduaOQuHsmvf+XB9t+G4Ag9cSrePTPPKAerRpJw61y8jLxdpxYrzcnPJnc+vYRj6ZPsnmntgrjwsHuparasG1B6gan7VcuX4Gfl619eKstrG+p41eqpmQE2Hnu9Wda3aVXMPzNWhkEM6GHJQK06ucFi/kONhxzVj3wwFng5UTEJMmu0NijdQ12pd9XDlh2/5vwcAAAAAAADkUHx05jNYYsJsz28OXZISHFxY9u4PZjucuXbtmkwmkzw8PNStWzd16NBBFotFe/bs0Z49e26pxEGDBt3S++A8kzf+rHknPpTJnChJKmrU17I+0+Tv5X1bxy3hXVRr+s/S4/8dr9PWPyVJW8Pn65EFF7S416fycM0f/S2czTAMvbv8gBbtsC0pFRWfqFmbT2n2llN6qFZJPd2qiu6tVizXZzJsPh6sVfsvSZJK+Ljr+bapG6xvPr9Zo9aNUmxirCSpSckm+uahb+Tj5nNb53W3uOujBz5SWe+ymrFvhiRp5v6ZuhB1Qe/f9366Mw/cXMwa/3BNjfzRtqTi5BWH9ECNEun2xrkdxy5f02Pfbk4TylS93kem+019ZKxWa66e/7s932nugbmSpNjEWC06skiLjizSvWXu1YA6A3Rfuftyfamsg1cP6tejv0qSvF29NbLRyFw9fm4ym8wae/dYPRv4rCTpq51fqX2l9rk+WyUoKkgDVgywh5LJynmXU9eqXdW1aldV9q2cq+cEAAAAAAC4oyTE35i9kmHQks7sloRYx9fm4St5Bkie/pLX9T89A2567H/jsae/FCdpcta/wJvj9ZJiY2O1cOFCLVy48FYuxc5kMhHO5HOvB87Qb+enyGSyTcMqbmqmZU98qyLuuXPz08PVTb8/8bmeW/qZ/g6bI0k6Y12nh+YP1i+Pf6dS3s7tJZIfTF1/XDP/d0qSZDGb5GoxKdaaJMOQ1hy8rDUHL6tWaR891aqyHm1UTh6ults+Z2KSoXeWHrA/H9exZqql1P48/afGbRwna5ItfGhVtpW+aPuFPF08b/vcku0m+5i7x6icdzm9/8/7SjKStOrUKl2JvqIv234pPw+/NO/pUr+Mplc4qX/PhunwpUgt2nFWfZpVzJV6kn22+rA9mAko4qZHGpRRjybl1bC8r8OX+fr50M+auvtGk3svFy9FJ9h6PW25uEVbLm5R5aKV1a92Pz1a7VF5uXpldKhsMwxDH237SMb16ZnDGg5TMc+0PYfyk3vL3qtW5Vrpf+f/p4tRF/XjwR/1VL2ncvUc3/37nT2Y8XH1UccqHfVIVVsfmfy23BsAAAAAAECeSu7LkjJAybQ/y/XAJT7S8bW5eV8PUPxTBC1ZhC4evpLlFlpOWCOytVuOjmzc3GgBhdaLy7/WuuBpSr73WM7ygH5/4ku5ueRu/xOz2azpj47Te+vLacHJj2UyJyrCtE+dfn5Cc7tOU91SFXL1fPnZj/+c0SerDtufT+5ZX+3rlNJPW89qzpZTuhhuS4IPBUXqlcV7NfmPQ+rfopIG3ltJpYp63PJ5f952VoeCbANgvXJF9XiT8vZtS48v1Zv/e1OJhm3mVPtK7TX5/slys+T+zKbeNXurdJHSennDy4pJiNHOyzs18I+BmvrQVFUomvpzYDKZ9Hrn2ur9/RZJ0merj+iRhmXl5ZY7n89/z4bpj31Bkmwzida/3EZFnNT7Z+WplXr/n/ftz8c1HaeeNXpqybElmn9wvs5ds82qOhVxSh/884G+3vm1etboqb61+6qcd7lbPm/g6UDtuLRDklTRp6L61ep3exfiJGOajNHm85tlyND0PdPVo3qPdAO9W3Eq/JSWHFsiyTaTaHnP5SxbBgAAAAAACj/DkOIictaTJTrEFsxkty/LrbK4pw5QPP3SCVpuDl38JZf81xs423cb161b58g6kI8MWTJZW8Pn259Xc39Yi3pNlovl9mdpZOSNNv1U2a+MPtr5imSJkdXlnPotG6Sljy1URb8SDjtvfvHH3ot6Y8le+/NXO9VSr6a2QGJ4m2oaen8VrdofpBmbTmrnmTBJUmi0Vd+sO6bvNhxXlwZlNOqhGqpaImfLzUXEWvXZ6huB0Ftd68p8fXmwBYcWpAoJulXrpkktJ8nF7LiQ4oHyD2jWw7P0/J/PKzgmWKciTmnwysFa0HWBSnql7ivTvEqA2tcppcADl3Q5Mk7/99dJvfhQjVypI2VI9uKD1Z0WzGy+sFmv/vWqffbKkHpDNKiubYbhgDoD1LdWX204t0HzD87X1qCtkqRIa6RmH5ituQfn6sEKD2pYw2E57hMTlxinz3d8bn/+ctOX5WpxzaWrcqyaATX1aPVHteTYEkVaI/X9nu/1SvNXcuXY3+z+xh5MPln3SYIZAAAAAABQ8MRH52ypsOTXHN2XxWTJZNZKJrNbXL2kQrKaSbbvOLZu3dqRdSCfGPzL+9oZucD+vKH345rT402Zzbnb2yI9Axq1VfmiMzRq3UgluVxVkkuwnvh1pNYOmFuoe9BsPhasUQt2K+l6qPzcA1X1XOvUTd9dLWZ1bVBWXRuU1a4zoZr5v1NasfeiEpIMJSQZ+m33Ba3ef0kTu9VR76YVsr3c0td/HtXVqHhJUpcGZdS8SoAk6ceDP+rDrR/a9+tbq68mNJ+Q6z1O0lOnWB3N7zxfI9aM0PHw47oSc0Vj1o3RjIdnpOknMqFTLa09dFmJSYa+33BcfZtXVAmf20vB/3csWJuOBUuSKgR45vpyaRnZe2WvRq8brYTr/+N7rMZjGtVkVKp9LGaLHqz4oB6s+KAOhxzWvIPztPzEclmTrEoykrTmzBptOLdBY+8eq/61+2f7czBn/xydv3ZeknRPmXvUpkKbXL02RxvZaKRWnlyp2MRYLTi8QP1q91MFn9ubdXfg6gGtOrVKkhTgEaCBdQbmRqkAAAAAAAC3xt6XJbOgJeTGUmHJ25zSl8UvG0uF+aXe5l600IQst8o5vw6OHDl4+Zwq+pbItd4u2TVkyUepgpmW/oP1fbeXnVpDm6r1NNtzlgb+0VeyXFOk+YAG/Pq2FvX+MOs3F0B7zoXpmTnbFZ9o623S6+7ymtCpliQpOCZYRd2KpllCrHFFfzWu6K/XOtfW3L9P6cd/zig02qoYa6JeWbxXfx0N1vs96svXM/OZDyeDozRr8ylJkruLWa9eP+/CwwtTBTND6w/Vi41fdGp/jbLeZTXj4Rnqu6yvLkRd0J7gPXrv7/f0Tst3UtVRrYS3+javoHl/n1FUfKI+DzyiD3vWv+XzGoahj1PMmnmpfU25uTg+kDoRdkIj/hyhmIQYSdJDFR/SG/e8ken3vGZATb3b6l2NbjJa/z3yX/18+GcFxwTLmmTVR9s+0paLW/Ruq3cV4BGQ6bmvRF/R9L3TJdn6/4xvNr7A9VIpVaSUBtYZqOl7pyshKUFf7PhCn7f5POs3ZuKrnV/ZHz/b4Nlc6esDAAAAAACgpEQpJiybPVlCru8bIsVfc3xtbt6ZLBWWTk8WT3/bvmbHrbhUmBHO5CPxCQnqtmCUzidulJHkIk+jkioWqaWmpRur010t1KBURYfNYHnu90+1NXye/fn9AU9r6iNjHHKurDQqU1mvNPlAk3ePlsmUpMMxyzRxbW1NfHBQntTjKMevXNOTM7cpKt62bFK72qXswcLH2z7W3ANz5WJ2Ue2A2mpQooEaFG+gBiUaqJx3OZlMJpX29dC4jrX0fNvqenfZQf209Ywkadmei9p1Jkxf9W2kuytlfGP+/eUHZU20Tdd59oGqKu/vpcVHFuvdv9+17/NM/Wf0QuMX8uRmfYBHgL588EsNXDFQsYmxWnJsiWoF1FL/2v1T7Tfqobv0687ziopP1E9bz6hj3VJqU7NkBkfN3Kr9l/Tv2TBJUq3SPurWsOztXkaWgqKC9GzgswqLs523Welm+uiBj7K9fFwxz2Ia1nCYnq73tL7a+ZVmH5gtSdp4bqMe+/0xfXj/h7qnzD0Zvv/LnV/aQ6Fed/VSDf/cWRrO2Z6u97QWH12skNgQBZ4O1IoTK9S5audbOta2oG3634X/SZLKFimrXnf1ys1SAQAAAABAYZDclyWjWSsZhS6x4Y6vzd6XJYNZK+mFLp7+kkvhXb0oPyKcyScSEhP1yIIXdSHxL0mSyZygWB3XkdjjOnJquX48JSmxqAIs1VXTv57ur3C3OtdspmJePrd97hFLv9Dm0Nn25/f6D8qzYCbZgEZttfPiMAVenipJWnT6CzU6cJe618n4JnNBcjE8RoN+2KqQ60uKNa8SoG/6NZbFbNLH2z7WvIO2oCwhKUF7g/dqb/BezZetD1CAR4A9qGlQooHqFa+nD3vW1/01imvC4j2KiE3Q+bAY9f7+b41pV0PD21SXxZw6XNl0NFhrDl6SJJX0cdew1tW05NgSTdoyyb7P0/WezrNgJlmtgFp6t9W7GrdxnCTpk22fqLpfdbUo08K+Twkfd73csaYmLT0gSXr5v//qj1EP5Hh5s8QkQ5+m6L8zrmNNe/8dRwmNDdWzgc/qUrTt76J2QG191farNMu3ZYebxU0vN3tZ95S9R69vel0hsSEKjgnWs6uf1VP1ntLIxiPlak49m2p/8H79dvw3SZKPm4+eb/T87V9UHvF289bLTV/Wa5tekyS9+/e7alCigcr7lM/RcQzD0JSdU+zPn2/8fJrZawAAAAAAoBAxDMkanbOeLMnPr/eqdRizS4oeLNnsyeIZILmxAkhBQDiTD9iCmdH2YMYwzLIkBijJJTj1jpYIhWintoTu1JbQOfroX7Pck8qpY4XH9M6DT8rFkvPpY6NWfKO/QmbYnzf37a9p3cbd1vXklk87PqdHFhzSGetamcwJeuvvcapd4mfVLOH42QyOFBoVr0E/bNX5MNtshdpliur/BjeVu4tZn+/43B7MmGRSpaKVdCriVKr3h8SGaP259Vp/br19v+r+1dXrrl5a+kIXvbRwn7afDr0eNhzR/45d1Rd9Gqm0r4ckKSExSe8s228/3isP19Lac3/orf+9ZW9EP7jOYI1uMjpfLG/1cJWHdSjkkH7Y94MSjUS9vOFl/dTlp1Q33Z9sWVl/HQ3W2kOXFXwtXi/991/NerJZjsKVX3ae07HLtumhd1fy14O1bm32TXZFW6P1/J/P62T4SUlSRZ+KmtpuqrzdvG/ruPeVu0+Luy3W65te1+YLm2XI0Ix9M7QtaJs+euAjey8WwzD00baP7O8b3nB4gW94/0i1R7T5wmYtO7FM16zX9Mpfr2jWw7PShFKZWX92vfZc2SNJqu5XXV2qdHFQtQAAAAAAINclxOdsqbDk/RLjHFyYSfLwzcZSYTeFLu4+d3xflsKMcCaPJSUlqfvPL+lcwnpJtmBmcLU3NO7+XjoRckkrjmzV3+d36kTEAUXqhGS+0cDJZEpSvOWsll6YolWzf9X4pq+oT4P7s33usX98q7VXvrc/b+zTRz90n5Br13a7zGazFj72sdrO76MYy3EZljANXDpC6/r/7PR+PLklOj5BT8/epqPXQ4CKAV6a/XQz+bi76MudX2rW/ln2fSe1nKQeNXooPC5c+4L3aU/wHu25YvuKiI+w72fI0NHQo/rgnw9U3W+hxnUbr78P1NA3a48qyZC2nLiqTl9u1CePN1S7OqX009YzOnLJdv6GFfzk4fevXvvfG/ZgZkDtAXqp6Uv5IphJ9kLjF3Q49LA2nd+ksLgwjVo3SnM7zbX3ATGZTPrk8QZ6+Mu/dCUyThuPXNEPm07qmQeqZuv4cQmJmrLmqP35+I41HXr91kSrRq8brb3BeyVJJTxL6Pv236u4Z/FcOX5xz+L6tt23mrN/jr7c+aUSDNsMrF5Le+nNe95Ul6pdtPLUSu26vEuSVLloZT1R64lcOXdee73F69p9ebfOXTunPVf26Nvd3+rFJi9m672JSYn6ateNXjMvNH5BFtZMBQAAAADA+RITbMt/ZStouR62RIdI1ijH1+bmcz1A8c96qbDkxx6+9GVBGibDMIy8LqKgioiIkK+vr4KDg1WsWLEcvz8pKUk9fh6nE/GrJUmGYVK/Kq/qtdZ9090/ITFRG0/t15rj2/Rv8L+6EHNYCS4XUu1T2txKUzq8rrqlKmR67vGrpumPoK/tzxt4P6a5Pd5yWE+b23Hoyjn1XvqEDIttPcZKrg9pWb8peVvULYhPSNIzc7Zrw5ErkqTi3u5aPPxeVSpWRN/s+kbf77kRlL1979t6/K7H0z2OYRg6HXHaHtb8e+VfHQo5lGqfhyo+pIdKDdX7Sy4pKOJGoNe/RUWt2HtRodFWSdIrj8Xpu4OTlGQkSZL61Oyj11u8nq+CmWQR8RHqv7y/fSZRh0od9GnrT1PVuulosAbO+EeGIblaTPpleCvVL++b5bFnbDqpd5bZlkVrU7OEZj3VPMf1Wa1WrVixQp07d5ara8YzNRKTEjXhrwlaeWqlJNtyYrMenqW7/O/K8TmzY1/wPo3fOF5nI8/aX+tWrZu2Bm1VUFSQJOk/D/1HD5R/wCHnzwt7r+zVoD8GKcFIkEkm/V+H/1PzMln/nS49vtS+LFqDEg00r9O8fPnfAvK/7I4HAAo/xgMAyRgPAKR0R40JhnEjZIkJzV5PlphQ5/RlcfHI2VJhXgGShx99WZCl5NwgPDxcRYsWzXA/wpnbcDvhTFJSkh5bOEHH4v6QZAtmelcar7faDsjRcWZsX62v//0sVUhjJLmphX8vff7wi/L1SLu+4GuBP+j381/KZLL91dfxelQ/PfZOvgxmki058Lfe+Ge4TOYESVKHUiP02cPD87iq7Au+FqcR83dq68kQSZKPh4t+fvZe1SlbVN/++62m7p5q3/eNFm+oT60+OTr+v1f+1eR/Jmvf1X3219zMbupz10AdOtRMaw+GpXnPvfXP62DiVCVeXxvz8bse15v3vCmzKf9+Dk6EnVC/Ff0Udf23IEY1GaWh9Yem2ufDPw7q+w0nJElVihfRshfuUxH3jCcJXotL0AMfr7P3/1n2wn2qVy7rQOdm2fnBKiI+QhM2TtBf521LGHpYPDStwzQ1Ltk4x+fLiShrlN77+z0tO7EszbZWZVvp23bfFroQ4oe9P9h7x5T0LKlF3RZlumybNdGqR5Y8ovPXzkuSZnScoWalmzmjVBRCd9Q/tABkivEAQDLGAwApFcgxIbkvS7o9WUJShy43z2hxWl+WgJsa3PulE7SkeOzq6di6cMfKbjjDsmZ5ICkpSb0XvZ4qmHmswks5DmYk6emmHdSvYRu9tma6Ai/OkSzRMpnjtTV8vh6Yv1KDar6gMS172IOXt/+cnSqYqenZNd8HM5LUvc492nVxtH4596kkaVXQd2q8u5YGNGqbx5Vl7d+zYRo2b4cuhttmsLi7mPV/g5qqTtmimr5neqpgZkLzCTkOZiSpYYmGmt9lvn4//rum7Jiiq7FXFZ8Ur7mHflBJr6V6ou1A/fJXScUn2P7ePf0O6mDiPHsw06N6j3wfzEhSVb+qmnz/ZL249kUZMvTVzq90l/9dqWZ9vNS+prYcv6o958J1MjhKb/++X5/2apjhMX/466Q9mHmkYdlbCmay41joMY1aN0pnIs9Ikiwmiz5r85nDgxlJKuJaRB/e/6Falm2p9/5+T9EJ0fYaxjUbV+iCGUl6qt5T2nJxi/65+I8ux1zWW5vf0ldtv8rwWv975L/2YKZl2ZYEMwAAAACAwishLhtLhYWmDWAS4x1cmMkWqOSkJ4unP31ZUGARzjhZUlKS+i1+W4djbvwGe7dyozTpocG3fEwPVzd93ul5nQrppVGrPtLxuECZTIaSXK5q1vGJWnT0Z717/+vafHavFp/9zB7MVHfvpIWPv5/vg5lkkx4arH0/H9SR2OUymZL00c7XVK/UT2pUpnJel5ah/24/q9eX7FN8gm3ZsFJF3fXtgLvVpKK/Zu6bmaq/xbim49S/dv9bPpfZZFb36t3VrmI7TdszTXMPzlVCUoIuR1/W8ujP1KBZfYWc6azT4ZfkVma+PZjpVq2bJracmO+DmWRtKrTRyMYj9fWur2XI0CsbX9H8LvNV1dfWX8bNxayvnmisLl/9paj4RC3acU731yiuRxuVS3OskKh4Tf/LNsvGYjZpbHvHLC0WeDpQr296XTEJMZIkP3c/fdL6E91T5h6HnC8jj1R7RA1LNNSEvyZob/BePdvgWVXzq+bUGpzFbDLrg/s+0OO/P67QuFCtP7tePx/+Od3eOtHW6FTLCma3Rw0AAAAAAHkqMUGKDctm0JJidosz+rK4F00naMkidKEvC+4whDNONujXd7U/eon9eafSL+iD9kNy5diVA0rqt76fadXRXXp70/uKMh+WJF0zH9ToTYMkGfZgpopbey3uPbnABDPJ5vd8Rw/OO6lI8wHJck1D/hihP/v+V36eRfK6tFSsiUl6b9kBzd5y2v5a00r+mjqgiUr6eGjO/jn6fMfn9m1j7x6rQXUH5cq5vd28NbbpWPWs0VOfbv9UG85tkCQdDtsrU9F9KuprUYJhWx6uc5XOeqflOwUmmEn2TP1ndCjkkAJPB+qa9ZpGrR2lH7v8KB83H0lS5eJF9G73ehq78F9J0hu/7lOTiv6qEJB6mb+p647pWpzte9GnWQVVKZ67n6PEpET9Z/d/NH3vdPtrtQJqaUrbKSrnnTYscoaKRStqfuf5ioiPkK+7Y2YJ5RclvUrq3VbvauTakZKkT7Z9oialmqTp7zPv4DyFxNqWHOxQqYPqFqvr9FoBAAAAAHewpCQpLjxnPVmiQ23vcTQXz5z1ZEnex1JAlmsD8hDhjBMN/uV9/Xttkf15+5Ij9HHHZ3P9PB1rNFb7agv16aZFmnf0PzJcQmQyJdm3V3R9UL/0/qTABTOSbZbQgh7f6JHFvZTkclXxlrPqtWisVvX/Nt9cz5XIOD0/f6e2ngqxvzbgnop6q2tdubmYNf/gfH2y/RP7tlFNRumpek/leh2VfSvrm4e+0abzm/TR1o90KuKUDBn2YObhyg/r/fvel6UA/kaCyWTSe63e06mIUzoaelSnIk5pwl8T9FXbr+zX07NJeW08ckVLdl9QZFyCXlywSwufu1euFtvn5EJYjOb8bQvP3F3MevHBGrlaY0R8hF7Z+Io2nd9kf61zlc6a2HKiPF3ydk1Tk8lU6IOZZK0rtFa/Wv3046EfFZ8Ur1c2vqIfu/xo/zsIiw3TzH0zJdmWeRvZeGRelgsAAAAAKMgMQ4qPyjhMuR60WKKCdf/Fk3I5Pck28yUmVDKSsjz8bTG7pO3J4uV/43FGoQt9WQCHIZxxkiFLJmtn5AL787bFn9XnnRzX0N5sNmv8A731bLMuGrPyS20LWyST2aoKLm30W5/P5WIpeDfkk1X0K6FPH5iiMZuGyGSOV1DSZg1b9pmmdRuXa+eIiotTtDVeJbx9cvS+m/vLuFnMerd7XfVpVlGS9POhnzV562T7/iMajUjT0D633VfuPrXo1kI/HvpR3/37na5Zr6lDpQ764P4P5GIuuEOAl6uXvmr7lZ5Y/oTC48K18dxGfbXrK425e4x9n3e719POM2E6ExKtXWfCNGXNEY3rWEuS9OWao/bl5p5sWVmlfT3SnMOaZFVMQoyKumXcuCs9R0OPavS60an6y4y9e6wG1hlYKPu75Hdjm47V9kvbdST0iI6FHdOn2z7Vm/e+KUmasW+GrlmvSZK6V++uKr5V8rJUAAAAAEB+YY3NeU+WmNBs9WUxSwqQpFtaXcx00wyWbPRk8QqQ3LzpywLkMwX3zmwBkJSUpO+3r9S8A3MVYdpnf/3+gKf1VZcXnFKDn2cRzezxmk6FDNWBK2fVuebdTjmvo7Wv0UiDgyZozol3JElbQufozTUl9G67J2/72OtO7NWodSOVZAmTW1wjNfDpqjaVmqpxRX/VLlNUbi7pz9BZuP2s3ripv8x3A+5Wowp+2ha0TfMOzNPas2vt+z/X4DkNb+i4gC4lV4urBtcdrB41euhc5DnVDqhdKEKC8j7l9WnrTzUscJgSjUTN2DdDJb1K2nv3+Hi46qu+jfX4t5uVkGRo6vrjalW9uEr6eOi/O85e38dFw9uk7bty4doFPRf4nE5FnFKZImXUoEQDNSjeQA1KNFDtYrXlbnFPt6Y/z/ypt/5+K1V/mU9bf6oWZVo46LuArLhb3PXJA5+oz7I+ik2M1cIjC9WybEvVK15PPx76UZLkZnbTsIbD8rhSAAAAAECuS0y4EaRkGLSknd0ia7TDSzPcfWTKSU8WT3/Jw0/KJ6vHALg9hDMOcDU6Uh9snKc/z/+iRJcgKcU98Jb+gzX1kTEZv9lBKgeUVOWAkk4/ryONu7+XDgQf1faInyRJv577QiU2++vFlo/e8jE3ntyvF9c/J7lEyiTJ6rFTO6w7tXVvBcVvaCVzdEPVLxegRhX81LiinxpX9FdJH3e9u+yA5tzUX2ZK37raHrxOHyybr0Mhh1KdZ2j9oXq+0fO3XOetKupWVHWK1XH6eR3pnjL36JXmr+iDfz6QJE3eOlm+7r7qWrWrJKlRBT+91KGmPlp5SIYhjfl5t2qVLqokW/slPfdAVfl5uaU6ZlBUkJ5e9bTOXzsvSboYdVEXoy5q1alVkiQXs4tqB9RW/eL1baFNiQYq4VZCq2NWa+Omjfbj1A6orSltp6isd1lHfxuQhap+VfVK81c0acskSdJbm99Ss9LNFJcYJ0nqW6uvShcpnZclAgAAAAAyk9yXJTpEignLRk+W6/s5oy+Lq1eK5cH80glaUocuVlcf/bF+izp16SZXV3qzAHcqwplctPviKX246f90IHK1ZIlJ9d01JQSoa8VB+qD9kLwrsBD64dEJ6vHzVZ2IXy2TKUnTDk9UsSK+6t+wTY6Ptfn0IY1c95xkibS9YJgkk+0OvsXzrDzLLVCSdYX2ht6rnVuay9hkax7v5WZRdHyi/Ti9mvuqYuVd6r/qVXuT8WQlPEtoSP0h6lerX6GYuZJf9K3VV1djrur7Pd9Lkt7c9KZ83Xx1f/n7JdkCmE3Hruh/x67qUkScLkVckSQV93bXU61SL2N1KepSqmDG391fsYmx9pkwkpSQlKC9wXu1N3ivfeaFp4tnqn26VO2it+99O8/7y+CGx2o8ps0XNivwdKAi4iP055k/JUnert4OX14QAAAAAHCdYUjx1zIOUzIKXWLDnNCXxTWdWSt+mfdk8QyQXNMulZ4pq1WGiduywJ2OUSAX/Hrgb/148hddTtoukylJStHOxTOxhnrV6KsX7nlUHq5uGR8Et8RsNmtx74/V8ccIXU76WyZzgibvGKdint/r4buaZPs4/5w5qmF/PiPDYvttCteE8lrY/f/079Wtmrl3jk5fO2Y7n2uE3EuuklvxP2UNbyxraCtFx9l+297d64KaNtivNZEblLA3IdXx6xWrpwF1BqhDpQ5ytfAbEY7wfKPnFRobqoVHFirBSNDY9WM1vcN0NSrZSGazSZ/3bqSHp2xUaLTV/p4XHqyuIu43hsEr0Vc0ZPUQnY20LXlWqWglzeg4QwEeAToWdkx7ruyxfQXv0cnwk6nOnxzMWEwWvdT0JQ2oPYAALp8xmUx6+963tTd4r4KiguyvD647WH4efnlXGAAAAAAUVNbYLHqyhKReKiz59SRr1se+HSazbfmvdJcHC5C8/FPMdEkRurgVoS8LAKchnMkFn+8fJ4unxT52G4ZFZV3u1Qt3P6VHajfP2+LuAC4Wi37v/Y3a//SkIk37JEusxm0aqWJec9SsfPUs37/93DE9EzhEhkuY7XgJZbWoxyxVDSil6sV6qGeN7tp+abvmHZindWfXyZAhkzlBbv7b5Oa/TYqpIRdLkhLcjmtPipmyFpNF7Sq104DaA9SwRENu1DuYyWTSay1eU3h8uFadWqXYxFiN+HOEZj88WzX8a6hUUQ992quhhszeLkkq7++pvs0r2t8fHBOsIauH6HSEbXm68t7l9X8d/k8lvWzLAdYKqKVaAbXUu2ZvSVJ4XLj2Be/TnmBbYLP3yl7JKn384MdqWb6lk68e2eXr7qvJ90/W06ueVpKRpACPAA2qMyivywIAAACAvJVozdlSYcn7pVhBwmHcfTNYKiyd0CV5P3df+rIAyPcIZ3JTorca+XbW6w88pVolyud1NXeUIu7uWtp7mjou6K84y0nJEqmhq57Rwm7zVbNExv0+dl88pSGrhspwCZUkuSSU1sJHbcFMMpPJpGalm6lZ6WY6G3lWPx36Sb8c/UVR1ijbDp5HlXKeTFG3onr8rsfpYZEHLGaLPrjvA4XHhevvi38rMj5SwwKHaU7nOSrnXU4P1S6lD3rU16r9QXqpw11yc7H9oBYSG6Khq4baZ8OU8y6nGR1nZPr35+vuq1blWqlVuVaSJKvVqhUrVqhZqWaOv1DclrtL3a3J90/WkmNLNLT+UHm5euV1SQAAAACQO5KSbMt/xYSmncGSbtCS3JclwvG1uXplPmslvdDFw0+ycPsSQOHE6JYLXBPKqWvZvhp//xPy9eAmX14p5uWjRd3/T91/7a9ElyAluQSr7+9DtKzXfJUtGpBm/31BZzR4xVNKcrkqSbIklNRP3WaqRvEyGZ6jgk8FjW82XiMajtBvx3/T/IPz7UtgVfWtqv61+6tr1a7c7M1DbhY3TWk7RUNXDdW+q/t0Oeayngt8TrMfnq1insXUr0VF9WtxY8ZMaGyohq4equPhxyVJZYqU0Q8df1AZ74w/Byj4OlXppE5VOuV1GQAAAACQvuS+LOkuFZZZ6BImyXBsbfa+LClnrfhnHrR4+ue8LwsAFHKEM7lgTZ8fVaJEibwuA5IqB5TUrM7TNWjFQBkuYbK6nFOPxc9odd+5qYKzg5fPqf/yJ5XkEixJMieU0Pyus7I948nbzVv9a/fXEzWf0LZL22QxWdS0VFOWLssnirgW0dR2UzXoj0E6FXFKpyNOa/ia4ZrRcYa83bzt+4XHhevZwGd1NPSoJKmUVyn90OEHlfMul1elAwAAAAAKG2tMJrNWQtPvyeKsvixpZrBk0ZPF05++LACQSwhncoGZNSzzlUZlKuurtt/qhfVDJEu0os1H9MjPw7S63//Jw9VNh69cUN+lTyrJ5YokyZxQTHO7zFDdUhVyfC6L2aJ7ytyT25eAXODv4a9p7adp4B8DdSn6kg6GHNSodaM0td1UuVvcFREfoWcDn9WhkEOSpBKeJfRDxx9UoWjOPwcAAAAAgDtAojWbS4XdNLvFWX1ZvDIKWm6awZK8n3tR+rIAQB4inEGh1KZqPU2KmaK3to6UyRyvUO1Sj4Vj9VXH1/XE74OV6HJJkmRKCNCsTjPVoHTlvC0YDlHGu4ymtZ+mQSsHKTwuXFuDtmrCxgma2HKihq8ZrgNXD0iSinkU0w8df1ClopXyuGIAAAAAgMMlJUqx4WmDlkxDl1ApPtLxtbkWuR6m+GWvJ4unP31ZAKCAYuRGodWz7r26Gv2Bvtz3ikzmRJ1L2KCev2+VXGy/sWJK8NeMjj+ocdkqeVwpHKmqX1VNfWiqhq4eqpiEGK05s0Zbg7YqIt7W7DDAI0A/dPxBVXz5HAAAAABAgWIYUlxkDnuyhDqnL4vFLW1PljQzWNIJXVzcHVsXACDfIJxBofZMs466GhOmeSfel8lkSJbrwUyin6a3/z81LV89jyuEMzQo0UBT2kzR82ufV0JSgj2Y8Xf31/91+D9V86uWxxUCAAAAwB0uPjqLpcLC0p/dkpTg2LpMlvTDlaxmt7h60ZcFAJCpAhnObNy4UZ988ol27Nihixcv6tdff1X37t3t2w3D0KRJkzRt2jSFhoaqRYsW+s9//qO6deva94mLi9PLL7+sn376STExMXrooYc0depUlS+fvYbwKDgmPNBHV6JDtPrSVEmSKdFX3z40TS0q3pXHlcGZWpZrqQ/v+1DjN46XIUO+7r6a3mG6avjXyOvSAAAAAKDwSIi/MXsluz1ZYkKkhFjH1+bhm3VPFi//1LNb6MsCAHCQAhnOREVFqWHDhnrqqaf02GOPpdn+8ccf6/PPP9esWbN011136b333lP79u11+PBh+fj4SJJGjx6tpUuXasGCBSpWrJheeuklde3aVTt27JDFYnH2JcHBPnt4uD75q4S2nN+mV+97Rs2YMXNHerjKw/Jx89HGcxvVp1YfVfWtmtclAQAAAED+lNyXJSc9WWKc1JfFzfvGbJbs9GTxDLAFM/RlAQDkIwXy/0qdOnVSp06d0t1mGIamTJmi119/XT179pQkzZ49W6VKldKPP/6o5557TuHh4frhhx80d+5ctWvXTpI0b948VahQQWvWrFHHjh3TPXZcXJzi4uLszyMibEsjWa1WWa3W3LxEOMDoex7VaD0qSfx93cGal2yu5iWbS8rdz0HysfhsAWA8AJCM8QBAsjwdD5L7ssSGyhQdIsWGSTEhMkWHSrG2QMWUYqaL/XFsuEwO7stiWNztAYthn63iJyPVnykee/jdel+WJENKYjxG/sDPCEDhlt3/tgtkOJOZkydPKigoSB06dLC/5u7urtatW2vz5s167rnntGPHDlmt1lT7lC1bVvXq1dPmzZszDGc+/PBDTZo0Kc3r69atk5eXV+5fDIACJzAwMK9LAJBPMB4ASMZ4ACDZbY0HhiGLES/XhGtyS4iSW+I1uSVck+v1P90So2zPE66l2BYlt4QomZWYexeRjiSZZXUponiLt+JdvBVv8ZbVxVvxliKKd0l+nLztxmuJJrf0+7LEXv8KTX4h7PoXULjwMwJQOEVHR2drv0IXzgQFBUmSSpUqler1UqVK6fTp0/Z93Nzc5O/vn2af5Pen59VXX9XYsWPtzyMiIlShQgW1bdtWxYoVy61LAFAAWa1WBQYGqn379nJ1dc3rcgDkIcYDAMkYDwAkSzMeJN7oy5LerBVTTIgUE2Z7LTbMvpyYKTEuq1PdNuP67JQ0M1k8/CTPABle/pLHTbNd3H1kNpnkIcnD4RUCBR8/IwCFW/KKW1kpdOFMMtNNv3lhGEaa126W1T7u7u5yd087ddbV1ZWBFIAkxgMANzAeAEjGeAAUYkmJ10OUzHuyWKJD1DrolDxPvG4LYOKvOb42N297uJKtniye/pKnn0xmWx/ezO+gAMgN/IwAFE7Z/e+60IUzpUuXlmSbHVOmTBn765cvX7bPpildurTi4+MVGhqaavbM5cuX1bJlS+cWDAAAAAAA8pZhSHERqcOV6NAsQxfFhmfr8GZJfpIUcwu1WdxvhCte18OWVEFLOqGLp7/k4nYLJwMAAM5S6MKZKlWqqHTp0goMDFTjxo0lSfHx8dqwYYM++ugjSdLdd98tV1dXBQYGqnfv3pKkixcvat++ffr444/zrHYAAAAAAHAbDEOyRqcOUOyPQ296fNM2w9F9WSwyFQmQKc2sFf/MgxY3etwCAFAYFchw5tq1azp27Jj9+cmTJ7V7924FBASoYsWKGj16tD744APVqFFDNWrU0AcffCAvLy/169dPkuTr66shQ4bopZdeUrFixRQQEKCXX35Z9evXV7t27fLqsgAAAAAAQLKE+Mxnrdgfh6Xez+F9WUySh282lgq7EbpYXYtqxZqN6tylC0sYAQAASQU0nNm+fbvatm1rfz527FhJ0uDBgzVr1iyNHz9eMTExGjFihEJDQ9WiRQutXr1aPj4+9vd88cUXcnFxUe/evRUTE6OHHnpIs2bNksVicfr1AAAAAABQaCUm2Jb/ylbQcj1siQ6RrFGOr83N53qA4p/1UmHJjz18JXMO7x1YrVIWfXABAMCdpUCGM23atJFhGBluN5lMmjhxoiZOnJjhPh4eHvr666/19ddfO6BCAAAAAAAKGcO4EbLEhGavJ0tMaLb7stwWF4+cLRXmFSB5+NGXBQAA5JkCGc4AAAAAAIBblNyXJd2eLCGZhC5hDu/LIrNLBmGKv1I1u785dHH1dGxdAAAAuYxwBgAAAACAgiohLhtLhYWmDWAS4x1cmEny9Mt2Txb7fu4+LP8FAADuCIQzAAAAAADktcQEKTbsphksmYUu1wMXZ/RlcS+aTtCSRehyK31ZAAAA7iCEMwAAAAAA5JakJCkuPGc9WaJDbe9xNBfPnPVkSd7H4ur42gAAAO4whDMAAAAAANzMMKT4qMxnraQXusSESkaSY2tL7suSk54snv70ZQEAAMhHCGcAAAAAAIWbNTbnPVliQh3fl8Vkljz8ctaTxStAcvOmLwsAAEABRzgDAAAAACgYEhNuBCnZ7ckSEyJZox1fm3vRnC0V5ulvC2bMZsfXBgAAgHyHcAYAAAAA4FzJfVmiQ6SYsGwELdf/jItwfG2uXimWB/NLJ2hJJ3Tx9KMvCwAAAHKEcAYAAAAAcGsMQ4q/lkGYEpZx6BIb5oS+LK7pzFrxyzpocfVwbF0AAACACGcAAAAAANL1viyZ9WQJSb1UWPLrSVbH1pXclyXDpcL8U8xgSRG6uBWhLwsAAADyLcIZAAAAAChMEq05WCosxX4JMY6vzd03g6XCbu7JkiJ0cfelLwsAAAAKHcIZAAAAAMiPkpJsy3/FhKYzgyWj0CXMiX1ZMpm1kl7o4uEnWfgnKAAAACARzgAAAACAYxmGFBeZwVJhoZkHLTIcW5u9L0vKWSv+mQctnv70ZQEAAABuE+EMAAAAAGSXNSaD5cGSH9/oyeISfVUdwy7J5d9o5/RlSTODJYueLJ7+9GUBAAAA8gjhDAAAAIA7T6I1i1krGcxuyUFfFpOkW5pf4u57PVDJoieLp/+N/dyL0pcFAAAAKEAIZwAAAAAUXEmJUmx4DnqyXJ/dEh/p8NIM1yKKkYc8A8rK5JWNniye/vRlAQAAAO4Q/NQPAAAAIO8l92XJqifLzaGLM/qyWNzS9mRJM4MlbeiSYJgVuGKFOnfuLFdXV8fWCAAAAKBAIZwBAAAAkLvio7NYKiws/aAlKcGxdZks6YcrXgGSp1/GQYur1631ZbE6uM8MAAAAgAKLcAYAAABA+hLib8xeyW5PlpgQKSHW8bV5+GY6a8UWuvinnt1CXxYAAAAA+QThDAAAAFDYJfdlyUlPlhjn9GWRm/eN2SzZ6cnieX2Wi9ni+NoAAAAAwEEIZwAAAICCwjCkuIibwpSwrEOX2HA5vi+Le+azVtINXfwlF3fH1gUAAAAA+RDhDAAAAOBshiFZY3K2VFjya87qy5LurJWMZrcESK6et9aXBQAAAADuQIQzAAAAwO2w92XJKGi5aamw5NktiXGOr83DLxtLhfml3uZelJAFAAAAAByMcAYAAACQbH1ZYsKy2ZPl+nJiMSFS/DXH1+bmfaPXSnZ6sngFSB6+9GUBAAAAgHyKcAYAAACFS3JflsxmraQXusSGO742e1+WDGatpBe6ePpLLm6Orw0AAAAA4DSEMwAAAMifDEOyRme/J0vK141Ex9ZmdknRg+Wmniw3ByspQxc3L8fWBQAAAAAoEAhnAAAA4HgJ8TlbKix5P4f3ZTHZlv/Kcqkw/9RBi7sPfVkAAAAAALeMcAYAAADZl5hgW/4rW0HL9bAlOkSyRjm+Njcfycs/41kr6YUu9GUBAAAAAOQBwhkAAIA7kWHcCFmy25MlJtQ5fVlcPDKftZLeYw8/+rIAAAAAAAoMwhkAAICCLLkvS7qzVjILXcKc1JclvVkrWcxucfV0bF0AAAAAAOQxwhkAAID8IiEuG0uFhd70OERKjHdwYSbJ0y9nPVk8/enLAgAAAABABghnAAAAcpnJSJSigiVrZPaWCkue3eKMvizuRdMJWrIIXejLAgAAAABAriKcAQAAyEhSkhQXnqOeLC7RIeoWFyHtdnBtLp4568mSvI/F1cGFAQAAAACArBDOAACAws8wpPioHPZkuf66kZSjU+V4Ea/kviw56cni6U9fFgAAAAAACjDCGQAAULBYY3PekyUm1PF9WUxmycNPhqefQmNN8itTReYixVPPWkkZwCQ/d/OmLwsAAAAAAHcYwhkAAJA3EhNuBCnZ7ckSEyJZox1fm3vRTJYKS6cni6e/5OEnmc1KsFr114oV6ty5s8yuLCEGAAAAAADSIpwBAAC3J7kvS3SIFBOWjaDl+p9xEY6vzdUrxUwVv2z0ZLm+H31ZAAAAAACAAxHOAAAAG8OQ4q9lEKaEZRy6xIbluC9Ljpld05m14pd10OLq4di6AAAAAAAAbgHhDAAAhZE1NmdLhSXvl2R1bF3X+7JkvFSYv9L0ZPEMkNyK0JcFAAAAAAAUGoQzAADkZ4nWHCwVlmK/hBjH1+bum8FSYTf3ZEkRurj7Smaz42sDAAAAAADIxwhnAABwhqQk2/JfMaGp+65kGrqEObEvSyazVtILXTz8JAs/RgAAAAAAANwK7qoAAJATyX1Z0mtwf3PocnPQIsOxtdn7sqScteKfQdCSYjt9WQAAAAAAAJyKcAYAcOeyxuS8J4uz+rKkmcGSRU8WT3/6sgAAAAAAABQQhDMAgIIv0ZrFrJUMZrc4qy+LV3aWCksRurgXpS8LAAAAAABAIUY4AwDIP5ISpdjwHPRkuT67JT7S8bW5FrkepvhlryeLpz99WQAAAAAAAJAu7hgBAHKfYUhxkVn3ZLk5dHFGXxaLW9qeLCmDlYxCFxd3x9YFAAAAAACAOwbhDAAgc/HRWSwVFpZ+0JKU4Ni6TJb0w5WsZre4etGXBQAAAAAAAHmKcAYA7hQJ8Tdmr2S6VFhY6m0JsY6vzcM3Zz1ZPP3pywIAAAAAAIACi3AGAAqa5L4sOenJEuOkvixu3jdms2SnJ4vn9VkuZovjawMAAAAAAADyCcIZAMgrhiHFRWQ+ayW90CU2XI7vy+Ke+ayVdEMX+rIAAAAAAAAA2UE4AwC3yzAka7Q84q9Kl/ZJ8RE3zWAJvelxitec1Zcl3VkrN72eMnRx9aQvCwAAAAAAAOAghDMAkJK9L0s2lwq7vp9rYpw6StJ+RxVmut6XJaulwvxSb3MvSsgCAAAAAAAA5DOEMwAKp6RE2xJh2QpaUsxkib/m+NrcvG/0WkkTtGQQunj40pcFAAAAAAAAKCQIZwDkb8l9WTKZtZJu6BIb7vja7H1ZApTk6aegsDiVrlJb5iLFMg5aPP0lFzfH1wYAAAAAAAAg3yKcAeAc1/uyZDprJWXQkvJ1I9GxtZldbvRgyW5PFk9/yc3LfohEq1XbVqxQ586dZXZ1dWy9AAAAAAAAAAo0whkAOZcQl36D+zShS1jqoCUxzsGFXe/LkmlPFv+bgpcAyd2HviwAAAAAAAAAnIZwBriTJSZIsWGZzFq5OXQJsz22Rjm+Njcfycs/41kr6YUu9GUBAAAAAAAAUAAQzgCFgWHYeqzkpCdLTKhz+rK4eGQ+ayW9xx5+9GUBAAAAAAAAUGgRzgD5SXJfljTLg2UVuoQ5qS9LerNWspjd4urp2LoAAAAAAAAAoIAhnAEcJSEuGz1ZQtMuKZYY7+DCTJKnXxZLhd0UwHj605cFAAAAAAAAAHIJ4QyQleS+LNnqyZJidosz+rK4F81G0HLTkmL0ZQEAAAAAAACAPEU4gztHUpIUF5758mBpQpdQ23sczcUz81krGc1usbg6vjYAAAAAAAAAQK4inEHBYxhSfFQOe7Jcf91IcmxtyX1ZctKTxdOfviwAAAAAAAAAcAchnEHessbmvCdLTKjj+7KYzJKHX+bLg6U3u8XNm74sAAAAAAAAAIBMEc4gdyQm3AhSstuTJSZEskY7vjb3ojcFKln0ZPH0twUzZrPjawMAAAAAAAAA3HEIZ5Bacl+WjGatpBu6hEpxEY6vzdUrZ0uFeQZInn70ZQEAAAAAAAAA5CuEM4WVYUjx1zJYKiws46AlNswJfVlc05m14pdJ0JLcl8XDsXUBAAAAAAAAAOAEd3w4M3XqVH3yySe6ePGi6tatqylTpuj+++/P67JSs8bmbKmw5P2SrI6tK7kvS4ZLhWUwu8WtCH1ZAAAAAAAAAAB3rDs6nPn55581evRoTZ06Va1atdL333+vTp066cCBA6pYsWLunzDRmvmslYxmtyTE5H4tN3P3tc1eybInS4rQxd2XviwAAAAAAAAAAOTQHR3OfP755xoyZIiGDh0qSZoyZYpWrVqlb7/9Vh9++GG2j2M6sU46l5hF6BLmxL4sOejJ4hVgm/1iuaM/CgAAAAAAAAAAOM0de0c+Pj5eO3bs0IQJE1K93qFDB23evDnd98TFxSkuLs7+PCLCFra4/PK05J67y3QZZld7kGJ4+ksetrDF8AqQPPxtr13/MpIb33v6Sy630JclyXD8EmhAIWe1WlP9CeDOxXgAIBnjAYBkjAcAUmJMAAq37P63fceGM8HBwUpMTFSpUqVSvV6qVCkFBQWl+54PP/xQkyZNytF5DJkU7+KteIu3rC5FFG/xVvz1P60uPoq3FFG8i7es1/9M3jfR7J5+X5bY619hyS+EX/8CkB8EBgbmdQkA8gnGAwDJGA8AJGM8AJASYwJQOEVHR2drvzs2nElmuikAMQwjzWvJXn31VY0dO9b+PCIiQhUqVJC1+Uglliovw9PPvnSYkbxsmLuPzCazPCTdwpwWAAWE1WpVYGCg2rdvL1dX17wuB0AeYjwAkIzxAEAyxgMAKTEmAIVb8opbWbljw5nixYvLYrGkmSVz+fLlNLNpkrm7u8vd3T3thgdekqVYMUeUCaCAcXV15QcrAJIYDwDcwHgAIBnjAYCUGBOAwim7/12bHVxHvuXm5qa77747zfTBwMBAtWzZMo+qAgAAAAAAAAAAhd0dO3NGksaOHauBAweqadOmuvfeezVt2jSdOXNGw4YNy+vSAAAAAAAAAABAIXVHhzN9+vTR1atX9c477+jixYuqV6+eVqxYoUqVKuV1aQAAAAAAAAAAoJC6o8MZSRoxYoRGjBiR12UAAAAAAAAAAIA7xB3bcwYAAAAAAAAAACAvEM4AAAAAAAAAAAA4EeEMAAAAAAAAAACAExHOAAAAAAAAAAAAOBHhDAAAAAAAAAAAgBMRzgAAAAAAAAAAADgR4QwAAAAAAAAAAIATEc4AAAAAAAAAAAA4EeEMAAAAAAAAAACAExHOAAAAAAAAAAAAOBHhDAAAAAAAAAAAgBMRzgAAAAAAAAAAADgR4QwAAAAAAAAAAIATEc4AAAAAAAAAAAA4EeEMAAAAAAAAAACAExHOAAAAAAAAAAAAOBHhDAAAAAAAAAAAgBO55HUBBZlhGJKkyMhIubq65nE1APKS1WpVdHS0IiIiGA+AOxzjAYBkjAcAkjEeAEiJMQEo3CIiIiTdyA8yQjhzG65evSpJqlKlSh5XAgAAAAAAAAAA8ovIyEj5+vpmuJ1w5jYEBARIks6cOZPpNxmFX7NmzbRt27a8LgN5KCIiQhUqVNDZs2dVtGjRvC4HeYjxAIwHSMZ4AMYDJGM8AOMBUmJMAGMCkjEeFE6GYSgyMlJly5bNdD/CmdtgNtta9vj6+jKQ3uEsFgufAUiSihYtymfhDsd4gGSMB2A8QDLGAzAeIBnjASTGBNzAmADGg8IrO5M5zE6oAyj0nn/++bwuAUA+wXgAIBnjAYBkjAcAUmJMAJCM8eDOZjKy6kqDDEVERMjX11fh4eEknMAdjvEAQDLGAwDJGA8AJGM8AJASYwIAiZkzt8Xd3V1vv/223N3d87oUAHmM8QBAMsYDAMkYDwAkYzwAkBJjAgCJmTMAAAAAAAAAAABOxcwZAAAAAAAAAAAAJyKcAQAAAAAAAAAAcCLCGQAAAAAAAAAAACcinAEAAAAAAAAAAHCiOz6c2bhxox555BGVLVtWJpNJS5YsSbX90qVLevLJJ1W2bFl5eXnp4Ycf1tGjR9M9lmEY6tSpU7rH2blzp9q3by8/Pz8VK1ZMzz77rK5du+agqwJwK3JjPGjTpo1MJlOqryeeeCLVPu+//75atmwpLy8v+fn5OfiqANwKZ40H3bp1U8WKFeXh4aEyZcpo4MCBunDhgqMvD0AOOGs8qFy5cpp9JkyY4OjLA5ADzhgP1q9fn2Z78te2bduccZkAssFZPx9wPxEo3O74cCYqKkoNGzbUN998k2abYRjq3r27Tpw4od9++027du1SpUqV1K5dO0VFRaXZf8qUKTKZTGlev3Dhgtq1a6fq1avrn3/+0cqVK7V//349+eSTjrgkALcot8aDZ555RhcvXrR/ff/996m2x8fHq1evXho+fLhDrwfArXPWeNC2bVstXLhQhw8f1uLFi3X8+HE9/vjjDr02ADnjrPFAkt55551U+7zxxhsOuy4AOeeM8aBly5aptl28eFFDhw5V5cqV1bRpU4dfI4DsccZ4wP1E4A5gwE6S8euvv9qfHz582JBk7Nu3z/5aQkKCERAQYEyfPj3Ve3fv3m2UL1/euHjxYprjfP/990bJkiWNxMRE+2u7du0yJBlHjx512PUAuHW3Oh60bt3aGDVqVLbOMXPmTMPX1zeXKgbgKM4YD5L99ttvhslkMuLj42+3bAAO4MjxoFKlSsYXX3yRyxUDcBRn/XwQHx9vlCxZ0njnnXdyo2wADuCo8YD7iUDhd8fPnMlMXFycJMnDw8P+msVikZubmzZt2mR/LTo6Wn379tU333yj0qVLp3scNzc3mc03vt2enp6SlOo4APKv7I4HkjR//nwVL15cdevW1csvv6zIyEin1grAsRw1HoSEhGj+/Plq2bKlXF1dHVM8gFyV2+PBRx99pGLFiqlRo0Z6//33FR8f79gLAJBrHPXzwe+//67g4GB+Ux4oQHJrPOB+IlD4Ec5kolatWqpUqZJeffVVhYaGKj4+XpMnT1ZQUJAuXrxo32/MmDFq2bKlHn300XSP8+CDDyooKEiffPKJ4uPjFRoaqtdee02SUh0HQP6V3fGgf//++umnn7R+/Xq9+eabWrx4sXr27JmHlQPIbbk9HrzyyisqUqSIihUrpjNnzui3335z5uUAuA25OR6MGjVKCxYs0Lp16zRy5EhNmTJFI0aMcPYlAbhFjvr3wg8//KCOHTuqQoUKzrgMALkgt8YD7icChZ9LXheQn7m6umrx4sUaMmSIAgICZLFY1K5dO3Xq1Mm+z++//661a9dq165dGR6nbt26mj17tsaOHatXX31VFotFL774okqVKiWLxeKMSwFwm7IzHki29WKT1atXTzVq1FDTpk21c+dONWnSxNllA3CA3B4Pxo0bpyFDhuj06dOaNGmSBg0apGXLlqXbxw5A/pKb48GYMWPs+zRo0ED+/v56/PHH7bNpAORvjvj3wrlz57Rq1SotXLjQKdcAIHfk1njA/USg8GPmTBbuvvtu7d69W2FhYbp48aJWrlypq1evqkqVKpKktWvX6vjx4/Lz85OLi4tcXGx512OPPaY2bdrYj9OvXz8FBQXp/Pnzunr1qiZOnKgrV67YjwMg/8tqPEhPkyZN5OrqqqNHjzqxUgCOlpvjQfHixXXXXXepffv2WrBggVasWKG///7b0ZcAIJc46ueDe+65R5J07NixXK8ZgGPk9ngwc+ZMFStWTN26dXNk2QAcILfGA+4nAoUb4Uw2+fr6qkSJEjp69Ki2b99uX8JswoQJ2rNnj3bv3m3/kqQvvvhCM2fOTHOcUqVKydvbWz///LM8PDzUvn17Z14GgFyQ0XiQnv3798tqtapMmTJOrBCAs+T2eGAYhqQb61QDKDhyezxInpnPzxBAwZMb44FhGJo5c6YGDRpELzqgAMutnw+4nwgUTnf8smbXrl1L9dtoJ0+e1O7duxUQEKCKFSvqv//9r0qUKKGKFStq7969GjVqlLp3764OHTpIkkqXLq3SpUunOW7FihVTpdjffPONWrZsKW9vbwUGBmrcuHGaPHmy/Pz8HH6NALLndseD48ePa/78+ercubOKFy+uAwcO6KWXXlLjxo3VqlUr+3HPnDmjkJAQnTlzRomJifZQt3r16vL29nbqNQNInzPGg61bt2rr1q2677775O/vrxMnTuitt95StWrVdO+99+bJdQNIyxnjwZYtW/T333+rbdu28vX11bZt2zRmzBh169ZNFStWzJPrBpCWs/69INlW6Th58qSGDBni1GsEkD3OGg+4nwgUcsYdbt26dYakNF+DBw82DMMwvvzyS6N8+fKGq6urUbFiReONN94w4uLiMj2mJOPXX39N9drAgQONgIAAw83NzWjQoIExZ84cB10RgFt1u+PBmTNnjAceeMD+33q1atWMF1980bh69Wqq8wwePDjd86xbt86JVwsgM84YD/bs2WO0bdvWCAgIMNzd3Y3KlSsbw4YNM86dO+fsywWQCWeMBzt27DBatGhh+Pr6Gh4eHkbNmjWNt99+24iKinL25QLIhLP+vWAYhtG3b1+jZcuWzro0ADnkrPGA+4lA4WYyjOvrZwAAAAAAAAAAAMDh6DkDAAAAAAAAAADgRIQzAAAAAAAAAAAATkQ4AwAAAAAAAAAA4ESEMwAAAAAAAAAAAE5EOAMAAAAAAAAAAOBEhDMAAAAAAAAAAABORDgDAAAAAAAAAADgRIQzAAAAAAAAAAAATkQ4AwAAAECzZs2SyWSSyWTSqVOn8rocFHBPPvmk/fOU8ut2P1sTJ05M97jr16/PlboBAAAAZyGcAQAAAAqwU6dOpXuzOqdfAAAAAADnIZwBAAAAgBQqV64sk8mkJ598Mq9LKfDKli2rvXv32r/KlSuXZp+Us2GyMmLECPuxZsyY4YiSAQAAAKdwyesCAAAAANy6cuXKae/evRlu79ixoy5cuKCyZctq1apVGe5Xr149wgjkOldXV9WrVy/XjleyZEmVLFlSkhQcHJxrxwUAAACcjXAGAAAAKMCyuvnt6uqarf0AAAAAAM7DsmYAAAAAAAAAAABORDgDAAAAQLNmzbL3/Th16lSa7W3atJHJZFKbNm0kSceOHdOwYcNUtWpVeXp6qnLlyhoyZIhOnz6d6n379u3TU089papVq8rDw0MVKlTQ8OHDdfny5WzVFRgYqAEDBqhKlSry9PRU0aJF1bBhQ40fP14XL17M9L0XLlzQhAkT1KRJE/n6+srNzU2lS5dW/fr11bdvX82aNUsRERFprjH5GmbPnm3/niR/JV9/stDQUM2cOVMDBgxQnTp15O3tbT9Px44dNW3aNMXHx2dY46lTp+zHnjVrliTpl19+UYcOHVSyZEkVKVJEDRs21Ndffy2r1Wp/n2EY+vHHH9WmTRuVLFlSXl5eatKkib777jsZhpHh+ZLPNXHiREnSmjVr1K1bN5UpU0YeHh6qWrWqRo4cqXPnzmX6vc0NyZ+5SZMmpakv5Vd6n0cAAACgoGNZMwAAAAA5smbNGvXs2VORkZH2106fPq0ZM2Zo2bJl2rBhg2rVqqWffvpJTz31lOLi4uz7nTt3Tt99953++OMPbd68WWXLlk33HFFRURo4cKB+/fXXVK/HxsZqz5492rNnj7799lv99NNP6tq1a5r3//XXX+ratWuq8EWSLl26pEuXLmnfvn1asGCBihcvnu77s6tx48ZpAqnk86xevVqrV6/Wd999pxUrVqh06dJZHm/EiBH69ttvU722Z88evfjii1q/fr0WLlyohIQEDRgwQIsWLUq1365duzR8+HDt3LlT06ZNy/JckyZNsoc0yU6ePKn//Oc/mjt3rpYuXaoHHnggy+MAAAAAyDnCGQAAAADZduHCBfXu3Vt+fn764IMP1Lx5c8XHx2vx4sX68ssvdfnyZQ0dOlRffPGFBg0apBo1auill15SgwYNFBUVpRkzZmju3Lk6ffq0xo4dqwULFqQ5R2Jioh555BGtW7dOJpNJTzzxhHr27KkqVarIarVq69at+uyzz3TmzBk99thj2rx5s+6++277++Pi4vTEE08oIiJCPj4+Gj58uNq2bauSJUvKarXq9OnT2rJlixYvXpzqvDNnzlRUVJQ6duyoCxcu6NFHH9V7772Xap8iRYqkqbVFixbq2rWrGjdurFKlSik+Pl4nT57UvHnztHLlSu3atUtPPPGE1q9fn+n39rvvvtM///yjzp07a+jQoapUqZLOnj2rDz/8UP/8849++eUXzZw5U3v27NGiRYvUr18/9evXT2XKlNHRo0c1ceJEHTp0SNOnT1fPnj318MMPZ3iu5cuXa/v27apZs6bGjx+vBg0aKDw8XP/97381ffp0RUREqGvXrtq7d68qVaqUad23qnv37mratKmmTp1qD6T27t2bZr9y5co55PwAAABAnjIAAAAAFFqVKlUyJBmVKlXKdL+ZM2cakgxJxsmTJ9Nsb926tX17jRo1jMuXL6fZZ9y4cfZ9SpQoYbRq1cqIiopKs1+vXr0MSYaLi0u6x/n0008NSYarq6uxYsWKdOsNCQkx6tata0gy7rvvvlTb/vzzT3sdS5cuzfCarVarER4enub15O/Z4MGDM3xvsiNHjmS6fcaMGfZa1qxZk2b7yZMn7dslGaNHj06zT1RUlFG5cmVDklG8eHHDZDIZU6ZMSbPfxYsXDR8fH0OS0a1bt3TrSXmuJk2aGJGRkWn2mTNnjn2fxx9/PNPry8jgwYOz9bkzDMN4++237efLiXXr1tnft27duluqEwAAAMgr9JwBAAAAkCNfffWVSpQokeb1ESNG2B8HBwdr+vTp8vLySrPf8OHDJUkJCQnasmVLqm1Wq1WfffaZJGnkyJHq1KlTujX4+/vrk08+kSRt2rRJx44ds28LCgqyP85sWS4XFxcVLVo0w+3ZUaNGjUy3P/XUU2rcuLEkacmSJZnuW6FCBX388cdpXvfy8tLgwYMl2b6vLVq00KhRo9LsV7p0afXo0UOSbVm3rEybNk3e3t5pXh84cKD9+75kyZIse/sAAAAAyDnCGQAAAADZ5ufnp44dO6a7rXLlyvawo0GDBqpdu3a6+zVs2ND++MSJE6m2bd261R4G9O7dO9NaUgYvKUOeMmXK2B/PnDkz02PkJsMwFBQUpCNHjmjfvn32r+S+Ov/++2+m7+/Zs6dcXV3T3dagQQP74z59+mR4jOTvbWhoqMLCwjLcr379+qmWgrvZ008/LckWoGW1HBsAAACAnKPnDAAAAIBsq1GjhkwmU4bbfX19FRERobvuuivDffz8/OyPIyMjU23bvn27/fG9996b7bpSzpa57777VLVqVZ04cUKjR4/W/Pnz1aNHD7Vu3VpNmzaVm5tbto+bHcuXL9e3336rjRs3prmelIKDgzM9Tna/Zzn53qZ8nlKzZs0yraV58+b2x/v27ct0XwAAAAA5RzgDAAAAINvSW6YsJbPZnOV+yftIUmJiYqptly9fvqW6oqOj7Y9dXV21dOlSPf744zp48KC2bdumbdu2SZI8PT3VunVrDRw4UH369JHFYrml80m2mTLPPPOMfvjhh2ztHxMTk+n27H7PbvV7m1LJkiUzraVUqVL2xyEhIZnuCwAAACDnCGcAAAAA5BspA4X169erWLFi2XrfzWFDnTp1tHfvXi1dulRLly7Vhg0bdPz4ccXExGjlypVauXKlPv/8c61YsSLLoCIjM2bMsAczjRo10ujRo9WiRQuVK1dOXl5e9uBn0KBBmjt3rgzDuKXzOEJms58AAAAAOB7hDAAAAIB8I2UY4+bmpnr16t3ysSwWi7p3767u3btLki5evKg//vhDU6dO1Y4dO7Rjxw4999xz+vXXX2/p+NOnT5ckVatWTZs3b5anp2e6+4WGht7S8R3p0qVL2d4eEBDg6HIAAACAO445610AAAAAwDkaN25sf7x69epcPXaZMmX09NNPa8uWLWrSpIkkadmyZWmWG8vurJL9+/dLkh599NEMgxnDMLRz587bqNoxkpd5y8722wnIsoNZPAAAALgTEc4AAAAAyDfuu+8++0yN7777ThEREbl+DldXV7Vu3VqSlJCQoLCwsFTbPTw8JElxcXGZHichIUFS6n43N/v999914cKF26jWMfbu3atdu3ZluH3GjBmSbLOP2rRp49Bakr/fUtbfcwAAAKCwIJwBAAAAkG94eHjo5ZdfliQFBQXpiSeeUFRUVIb7R0ZG6ptvvkn12l9//aVjx45l+J74+Hht2LBBkuTt7a0SJUqk2l6mTBlJ0vHjxzOttUaNGpKkpUuXprt02fHjxzVixIhMj5GXnn322XS/tz/++KNWrFghSerevbv9++EoKY+f1fccAAAAKCzoOQMAAAAgXxk/frz+/PNP/fnnn/rjjz9Up04dDRs2TPfee6/8/PwUGRmpw4cPa/369VqyZIk8PDw0cuRI+/v//PNPvfvuu7r//vvVpUsXNWjQQCVKlFBMTIyOHDmi7777zr7U2NChQ+XikvqfRS1bttS6deu0bds2TZ48WZ06dVKRIkUkSZ6enipXrpwkadCgQRo3bpzOnz+vli1bavz48apbt65iY2O1du1aTZkyRXFxcWrSpEm+W9qsadOm2r59u5o2bapXXnlF9evXV3h4uBYtWqTvv/9ekuTj46NPP/3U4bW0bNnS/njMmDF6/fXXVaZMGftyZ5UrV07zdwQAAAAUdPyECwAAACBfsVgsWrp0qYYNG6Y5c+bozJkzeu211zLcv2TJkmleS0pK0oYNG+wzZNLTs2dPffjhh2leHz58uL799luFhITo1Vdf1auvvmrf1rp1a61fv16SNGrUKAUGBmr16tU6dOiQnn766VTH8fT01Jw5c7R8+fJ8F8506dJFXbp00aRJk/TUU0+l2V60aFH9/vvvqly5ssNrqV69unr37q2FCxdq9erVaXoNnTx50il1AAAAAM7EsmYAAAAA8h1PT0/Nnj1b27dv1/Dhw1W3bl35+vrKxcVFfn5+atSokYYMGaJFixbp4MGDqd47fvx4rVixQmPGjNE999yjihUrysPDQx4eHqpcubL69Omj5cuXa/Hixan6nSQrV66ctm7dqiFDhqh69erp7iPZetcsX75cX331lZo2bSovLy95enqqevXqGjZsmHbu3KlevXo55PuTGyZOnKiVK1eqS5cuKlWqlNzc3FS5cmWNGDFC+/fvt/flcYZ58+bp448/VvPmzeXr6yuzmX+qAgAAoHAzGYZh5HURAAAAAADHS14q7O2339bEiRMddp4nn3xSs2fPVqVKlXTq1CmHnGP9+vVq27atJGndunVq06aNQ84DAAAAOALLmgEAAAAAHMJqtWrfvn325zVr1hsD4hIAAADjSURBVJSrq+stH+/y5cu6fPmyJNtyZwAAAEBBRTgDAAAAAHCICxcuqH79+vbnt9s/ZurUqZo0aVIuVAYAAADkLRbyBQAAAAAAAAAAcCJ6zgAAAADAHcJZPWcAAAAAZI6ZMwAAAAAAAAAAAE5EzxkAAAAAuEOwcAIAAACQPzBzBgAAAAAAAAAAwIkIZwAAAAAAAAAAAJyIcAYAAAAAAAAAAMCJCGcAAAAAAAAAAACciHAGAAAAAAAAAADAiQhnAAAAAAAAAAAAnIhwBgAAAAAAAAAAwIkIZwAAAAAAAAAAAJzo/wHKpST47pdFVAAAAABJRU5ErkJggg==",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"fig, ax = plt.subplots(1, 1, figsize = (20, 7))\n",
"plot_df = AirPassengersPanel[AirPassengersPanel.unique_id=='Airline1'].set_index('ds')\n",
@@ -522,7 +1110,100 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " unique_id | \n",
+ " ds | \n",
+ " y | \n",
+ " trend | \n",
+ " y_[lag12] | \n",
+ " month | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " Airline1 | \n",
+ " 1949-01-31 | \n",
+ " 112.0 | \n",
+ " 0 | \n",
+ " 112.0 | \n",
+ " -0.500000 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " Airline1 | \n",
+ " 1949-02-28 | \n",
+ " 118.0 | \n",
+ " 1 | \n",
+ " 118.0 | \n",
+ " -0.409091 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " Airline1 | \n",
+ " 1949-03-31 | \n",
+ " 132.0 | \n",
+ " 2 | \n",
+ " 132.0 | \n",
+ " -0.318182 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " Airline1 | \n",
+ " 1949-04-30 | \n",
+ " 129.0 | \n",
+ " 3 | \n",
+ " 129.0 | \n",
+ " -0.227273 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " Airline1 | \n",
+ " 1949-05-31 | \n",
+ " 121.0 | \n",
+ " 4 | \n",
+ " 121.0 | \n",
+ " -0.136364 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
+ `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
+ `val_size`: int, validation size for temporal cross-validation.
+ `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
+ `test_size`: int, test size for temporal cross-validation.
+ """
+ return self._fit(
+ dataset=dataset,
+ batch_size=self.batch_size,
+ valid_batch_size=self.valid_batch_size,
+ val_size=val_size,
+ test_size=test_size,
+ random_seed=random_seed,
+ distributed_config=distributed_config,
+ )
+
+ def predict(
+ self,
+ dataset,
+ test_size=None,
+ step_size=1,
+ random_seed=None,
+ quantiles=None,
+ **data_module_kwargs,
+ ):
+ """Predict.
+
+ Neural network prediction with PL's `Trainer` execution of `predict_step`.
+
+ **Parameters:**
+ `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
+ `test_size`: int=None, test size for temporal cross-validation.
+ `step_size`: int=1, Step size between each window.
+ `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
+ `quantiles`: list of floats, optional (default=None), target quantiles to predict.
+ `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).
+ """
+ self._check_exog(dataset)
+ self._restart_seed(random_seed)
+ if "quantile" in data_module_kwargs:
+ warnings.warn(
+ "The 'quantile' argument will be deprecated, use 'quantiles' instead."
+ )
+ if quantiles is not None:
+ raise ValueError("You can't specify quantile and quantiles.")
+ quantiles = [data_module_kwargs.pop("quantile")]
+ self._set_quantiles(quantiles)
+
+ self.predict_step_size = step_size
+ self.decompose_forecast = False
+ datamodule = TimeSeriesDataModule(
+ dataset=dataset,
+ valid_batch_size=self.valid_batch_size,
+ **data_module_kwargs,
+ )
+
+ # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.
+ pred_trainer_kwargs = self.trainer_kwargs.copy()
+ if (pred_trainer_kwargs.get("accelerator", None) == "gpu") and (
+ torch.cuda.device_count() > 1
+ ):
+ pred_trainer_kwargs["devices"] = [0]
+
+ trainer = pl.Trainer(**pred_trainer_kwargs)
+ fcsts = trainer.predict(self, datamodule=datamodule)
+ fcsts = torch.vstack(fcsts)
+
+ if self.MULTIVARIATE:
+ # [B, h, n_series (, Q)] -> [n_series, B, h (, Q)]
+ fcsts = fcsts.swapaxes(0, 2)
+ fcsts = fcsts.swapaxes(1, 2)
+
+ fcsts = fcsts.numpy().flatten()
+ fcsts = fcsts.reshape(-1, len(self.loss.output_names))
+ return fcsts
+
+ def decompose(
+ self,
+ dataset,
+ step_size=1,
+ random_seed=None,
+ quantiles=None,
+ **data_module_kwargs,
+ ):
+ """Decompose Predictions.
+
+ Decompose the predictions through the network's layers.
+ Available methods are `ESRNN`, `NHITS`, `NBEATS`, and `NBEATSx`.
+
+ **Parameters:**
+ `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation here](https://nixtla.github.io/neuralforecast/tsdataset.html).
+ `step_size`: int=1, step size between each window of temporal data.
+ `quantiles`: list of floats, optional (default=None), target quantiles to predict.
+ `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).
+ """
+ # Restart random seed
+ if random_seed is None:
+ random_seed = self.random_seed
+ torch.manual_seed(random_seed)
+ self._set_quantiles(quantiles)
+
+ self.predict_step_size = step_size
+ self.decompose_forecast = True
+ datamodule = TimeSeriesDataModule(
+ dataset=dataset,
+ valid_batch_size=self.valid_batch_size,
+ **data_module_kwargs,
+ )
+ trainer = pl.Trainer(**self.trainer_kwargs)
+ fcsts = trainer.predict(self, datamodule=datamodule)
+ self.decompose_forecast = False # Default decomposition back to false
+ return torch.vstack(fcsts).numpy()
diff --git a/neuralforecast/common/_base_multivariate.py b/neuralforecast/common/_base_multivariate.py
deleted file mode 100644
index 0fdc3b94d..000000000
--- a/neuralforecast/common/_base_multivariate.py
+++ /dev/null
@@ -1,608 +0,0 @@
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/common.base_multivariate.ipynb.
-
-# %% auto 0
-__all__ = ['BaseMultivariate']
-
-# %% ../../nbs/common.base_multivariate.ipynb 5
-import numpy as np
-import torch
-import torch.nn as nn
-import pytorch_lightning as pl
-import neuralforecast.losses.pytorch as losses
-
-from ._base_model import BaseModel
-from ._scalers import TemporalNorm
-from ..tsdataset import TimeSeriesDataModule
-from ..utils import get_indexer_raise_missing
-
-# %% ../../nbs/common.base_multivariate.ipynb 6
-class BaseMultivariate(BaseModel):
- """Base Multivariate
-
- Base class for all multivariate models. The forecasts for all time-series are produced simultaneously
- within each window, which are randomly sampled during training.
-
- This class implements the basic functionality for all windows-based models, including:
- - PyTorch Lightning's methods training_step, validation_step, predict_step.
- - fit and predict methods used by NeuralForecast.core class.
- - sampling and wrangling methods to generate multivariate windows.
- """
-
- def __init__(
- self,
- h,
- input_size,
- loss,
- valid_loss,
- learning_rate,
- max_steps,
- val_check_steps,
- n_series,
- batch_size,
- step_size=1,
- num_lr_decays=0,
- early_stop_patience_steps=-1,
- scaler_type="robust",
- futr_exog_list=None,
- hist_exog_list=None,
- stat_exog_list=None,
- num_workers_loader=0,
- drop_last_loader=False,
- random_seed=1,
- alias=None,
- optimizer=None,
- optimizer_kwargs=None,
- lr_scheduler=None,
- lr_scheduler_kwargs=None,
- dataloader_kwargs=None,
- **trainer_kwargs,
- ):
- super().__init__(
- random_seed=random_seed,
- loss=loss,
- valid_loss=valid_loss,
- optimizer=optimizer,
- optimizer_kwargs=optimizer_kwargs,
- lr_scheduler=lr_scheduler,
- lr_scheduler_kwargs=lr_scheduler_kwargs,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
- max_steps=max_steps,
- early_stop_patience_steps=early_stop_patience_steps,
- **trainer_kwargs,
- )
-
- # Padder to complete train windows,
- # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]
- self.h = h
- self.input_size = input_size
- self.n_series = n_series
- self.padder = nn.ConstantPad1d(padding=(0, self.h), value=0.0)
-
- # Multivariate models do not support these loss functions yet.
- unsupported_losses = (
- losses.sCRPS,
- losses.MQLoss,
- losses.DistributionLoss,
- losses.PMM,
- losses.GMM,
- losses.HuberMQLoss,
- losses.MASE,
- losses.relMSE,
- losses.NBMM,
- )
- if isinstance(self.loss, unsupported_losses):
- raise Exception(f"{self.loss} is not supported in a Multivariate model.")
- if isinstance(self.valid_loss, unsupported_losses):
- raise Exception(
- f"{self.valid_loss} is not supported in a Multivariate model."
- )
-
- self.batch_size = batch_size
-
- # Optimization
- self.learning_rate = learning_rate
- self.max_steps = max_steps
- self.num_lr_decays = num_lr_decays
- self.lr_decay_steps = (
- max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7
- )
- self.early_stop_patience_steps = early_stop_patience_steps
- self.val_check_steps = val_check_steps
- self.step_size = step_size
-
- # Scaler
- self.scaler = TemporalNorm(
- scaler_type=scaler_type, dim=2
- ) # Time dimension is in the second axis
-
- # Fit arguments
- self.val_size = 0
- self.test_size = 0
-
- # Model state
- self.decompose_forecast = False
-
- # DataModule arguments
- self.num_workers_loader = num_workers_loader
- self.dataloader_kwargs = dataloader_kwargs
- self.drop_last_loader = drop_last_loader
- # used by on_validation_epoch_end hook
- self.validation_step_outputs = []
- self.alias = alias
-
- def _create_windows(self, batch, step):
- # Parse common data
- window_size = self.input_size + self.h
- temporal_cols = batch["temporal_cols"]
- temporal = batch["temporal"]
-
- if step == "train":
- if self.val_size + self.test_size > 0:
- cutoff = -self.val_size - self.test_size
- temporal = temporal[:, :, :cutoff]
-
- temporal = self.padder(temporal)
- windows = temporal.unfold(
- dimension=-1, size=window_size, step=self.step_size
- )
- # [n_series, C, Ws, L+H] 0, 1, 2, 3
-
- # Sample and Available conditions
- available_idx = temporal_cols.get_loc("available_mask")
- sample_condition = windows[:, available_idx, :, -self.h :]
- sample_condition = torch.sum(sample_condition, axis=2) # Sum over time
- sample_condition = torch.sum(
- sample_condition, axis=0
- ) # Sum over time-series
- available_condition = windows[:, available_idx, :, : -self.h]
- available_condition = torch.sum(
- available_condition, axis=2
- ) # Sum over time
- available_condition = torch.sum(
- available_condition, axis=0
- ) # Sum over time-series
- final_condition = (sample_condition > 0) & (
- available_condition > 0
- ) # Of shape [Ws]
- windows = windows[:, :, final_condition, :]
-
- # Get Static data
- static = batch.get("static", None)
- static_cols = batch.get("static_cols", None)
-
- # Protection of empty windows
- if final_condition.sum() == 0:
- raise Exception("No windows available for training")
-
- # Sample windows
- n_windows = windows.shape[2]
- if self.batch_size is not None:
- w_idxs = np.random.choice(
- n_windows,
- size=self.batch_size,
- replace=(n_windows < self.batch_size),
- )
- windows = windows[:, :, w_idxs, :]
-
- windows = windows.permute(2, 1, 3, 0) # [Ws, C, L+H, n_series]
-
- windows_batch = dict(
- temporal=windows,
- temporal_cols=temporal_cols,
- static=static,
- static_cols=static_cols,
- )
-
- return windows_batch
-
- elif step in ["predict", "val"]:
-
- if step == "predict":
- predict_step_size = self.predict_step_size
- cutoff = -self.input_size - self.test_size
- temporal = batch["temporal"][:, :, cutoff:]
-
- elif step == "val":
- predict_step_size = self.step_size
- cutoff = -self.input_size - self.val_size - self.test_size
- if self.test_size > 0:
- temporal = batch["temporal"][:, :, cutoff : -self.test_size]
- else:
- temporal = batch["temporal"][:, :, cutoff:]
-
- if (
- (step == "predict")
- and (self.test_size == 0)
- and (len(self.futr_exog_list) == 0)
- ):
- temporal = self.padder(temporal)
-
- windows = temporal.unfold(
- dimension=-1, size=window_size, step=predict_step_size
- )
- # [n_series, C, Ws, L+H] -> [Ws, C, L+H, n_series]
- windows = windows.permute(2, 1, 3, 0)
-
- # Get Static data
- static = batch.get("static", None)
- static_cols = batch.get("static_cols", None)
-
- windows_batch = dict(
- temporal=windows,
- temporal_cols=temporal_cols,
- static=static,
- static_cols=static_cols,
- )
-
- return windows_batch
- else:
- raise ValueError(f"Unknown step {step}")
-
- def _normalization(self, windows, y_idx):
-
- # windows are already filtered by train/validation/test
- # from the `create_windows_method` nor leakage risk
- temporal = windows["temporal"] # [Ws, C, L+H, n_series]
- temporal_cols = windows["temporal_cols"].copy() # [Ws, C, L+H, n_series]
-
- # To avoid leakage uses only the lags
- temporal_data_cols = self._get_temporal_exogenous_cols(
- temporal_cols=temporal_cols
- )
- temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)
- temporal_idxs = np.append(y_idx, temporal_idxs)
- temporal_data = temporal[:, temporal_idxs, :, :]
- temporal_mask = temporal[
- :, temporal_cols.get_loc("available_mask"), :, :
- ].clone()
- temporal_mask[:, -self.h :, :] = 0.0
-
- # Normalize. self.scaler stores the shift and scale for inverse transform
- temporal_mask = temporal_mask.unsqueeze(
- 1
- ) # Add channel dimension for scaler.transform.
- temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)
- # Replace values in windows dict
- temporal[:, temporal_idxs, :, :] = temporal_data
- windows["temporal"] = temporal
-
- return windows
-
- def _inv_normalization(self, y_hat, temporal_cols, y_idx):
- # Receives window predictions [Ws, H, n_series]
- # Broadcasts outputs and inverts normalization
-
- # Add C dimension
- # if y_hat.ndim == 2:
- # remove_dimension = True
- # y_hat = y_hat.unsqueeze(-1)
- # else:
- # remove_dimension = False
-
- y_scale = self.scaler.x_scale[:, [y_idx], :].squeeze(1)
- y_loc = self.scaler.x_shift[:, [y_idx], :].squeeze(1)
-
- # y_scale = torch.repeat_interleave(y_scale, repeats=y_hat.shape[-1], dim=-1)
- # y_loc = torch.repeat_interleave(y_loc, repeats=y_hat.shape[-1], dim=-1)
-
- y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)
-
- # if remove_dimension:
- # y_hat = y_hat.squeeze(-1)
- # y_loc = y_loc.squeeze(-1)
- # y_scale = y_scale.squeeze(-1)
-
- return y_hat, y_loc, y_scale
-
- def _parse_windows(self, batch, windows):
- # Temporal: [Ws, C, L+H, n_series]
-
- # Filter insample lags from outsample horizon
- mask_idx = batch["temporal_cols"].get_loc("available_mask")
- y_idx = batch["y_idx"]
- insample_y = windows["temporal"][:, y_idx, : -self.h, :]
- insample_mask = windows["temporal"][:, mask_idx, : -self.h, :]
- outsample_y = windows["temporal"][:, y_idx, -self.h :, :]
- outsample_mask = windows["temporal"][:, mask_idx, -self.h :, :]
-
- # Filter historic exogenous variables
- if len(self.hist_exog_list):
- hist_exog_idx = get_indexer_raise_missing(
- windows["temporal_cols"], self.hist_exog_list
- )
- hist_exog = windows["temporal"][:, hist_exog_idx, : -self.h, :]
- else:
- hist_exog = None
-
- # Filter future exogenous variables
- if len(self.futr_exog_list):
- futr_exog_idx = get_indexer_raise_missing(
- windows["temporal_cols"], self.futr_exog_list
- )
- futr_exog = windows["temporal"][:, futr_exog_idx, :, :]
- else:
- futr_exog = None
-
- # Filter static variables
- if len(self.stat_exog_list):
- static_idx = get_indexer_raise_missing(
- windows["static_cols"], self.stat_exog_list
- )
- stat_exog = windows["static"][:, static_idx]
- else:
- stat_exog = None
-
- return (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- )
-
- def training_step(self, batch, batch_idx):
- # Create and normalize windows [batch_size, n_series, C, L+H]
- windows = self._create_windows(batch, step="train")
- y_idx = batch["y_idx"]
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- ) = self._parse_windows(batch, windows)
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L, n_series]
- insample_mask=insample_mask, # [Ws, L, n_series]
- futr_exog=futr_exog, # [Ws, F, L + h, n_series]
- hist_exog=hist_exog, # [Ws, X, L, n_series]
- stat_exog=stat_exog,
- ) # [n_series, S]
-
- # Model Predictions
- output = self(windows_batch)
- if self.loss.is_distribution_output:
- outsample_y, y_loc, y_scale = self._inv_normalization(
- y_hat=outsample_y, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)
- else:
- loss = self.loss(y=outsample_y, y_hat=output, mask=outsample_mask)
-
- if torch.isnan(loss):
- print("Model Parameters", self.hparams)
- print("insample_y", torch.isnan(insample_y).sum())
- print("outsample_y", torch.isnan(outsample_y).sum())
- print("output", torch.isnan(output).sum())
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "train_loss",
- loss.detach().item(),
- batch_size=outsample_y.size(0),
- prog_bar=True,
- on_epoch=True,
- )
- self.train_trajectories.append((self.global_step, loss.detach().item()))
- return loss
-
- def validation_step(self, batch, batch_idx):
- if self.val_size == 0:
- return np.nan
-
- # Create and normalize windows [Ws, L+H, C]
- windows = self._create_windows(batch, step="val")
- y_idx = batch["y_idx"]
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- ) = self._parse_windows(batch, windows)
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L, n_series]
- insample_mask=insample_mask, # [Ws, L, n_series]
- futr_exog=futr_exog, # [Ws, F, L + h, n_series]
- hist_exog=hist_exog, # [Ws, X, L, n_series]
- stat_exog=stat_exog,
- ) # [n_series, S]
-
- # Model Predictions
- output = self(windows_batch)
- if self.loss.is_distribution_output:
- outsample_y, y_loc, y_scale = self._inv_normalization(
- y_hat=outsample_y, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
-
- if str(type(self.valid_loss)) in [
- "",
- "",
- ]:
- _, output = self.loss.sample(distr_args=distr_args)
-
- # Validation Loss evaluation
- if self.valid_loss.is_distribution_output:
- valid_loss = self.valid_loss(
- y=outsample_y, distr_args=distr_args, mask=outsample_mask
- )
- else:
- valid_loss = self.valid_loss(
- y=outsample_y, y_hat=output, mask=outsample_mask
- )
-
- if torch.isnan(valid_loss):
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "valid_loss",
- valid_loss.detach().item(),
- batch_size=outsample_y.size(0),
- prog_bar=True,
- on_epoch=True,
- )
- self.validation_step_outputs.append(valid_loss)
- return valid_loss
-
- def predict_step(self, batch, batch_idx):
- # Create and normalize windows [Ws, L+H, C]
- windows = self._create_windows(batch, step="predict")
- y_idx = batch["y_idx"]
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- insample_y, insample_mask, _, _, hist_exog, futr_exog, stat_exog = (
- self._parse_windows(batch, windows)
- )
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L, n_series]
- insample_mask=insample_mask, # [Ws, L, n_series]
- futr_exog=futr_exog, # [Ws, F, L + h, n_series]
- hist_exog=hist_exog, # [Ws, X, L, n_series]
- stat_exog=stat_exog,
- ) # [n_series, S]
-
- # Model Predictions
- output = self(windows_batch)
- if self.loss.is_distribution_output:
- _, y_loc, y_scale = self._inv_normalization(
- y_hat=torch.empty(
- size=(insample_y.shape[0], self.h, self.n_series),
- dtype=output[0].dtype,
- device=output[0].device,
- ),
- temporal_cols=batch["temporal_cols"],
- y_idx=y_idx,
- )
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- _, y_hat = self.loss.sample(distr_args=distr_args)
-
- if self.loss.return_params:
- distr_args = torch.stack(distr_args, dim=-1)
- distr_args = torch.reshape(
- distr_args, (len(windows["temporal"]), self.h, -1)
- )
- y_hat = torch.concat((y_hat, distr_args), axis=2)
- else:
- y_hat, _, _ = self._inv_normalization(
- y_hat=output, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- return y_hat
-
- def fit(
- self,
- dataset,
- val_size=0,
- test_size=0,
- random_seed=None,
- distributed_config=None,
- ):
- """Fit.
-
- The `fit` method, optimizes the neural network's weights using the
- initialization parameters (`learning_rate`, `windows_batch_size`, ...)
- and the `loss` function as defined during the initialization.
- Within `fit` we use a PyTorch Lightning `Trainer` that
- inherits the initialization's `self.trainer_kwargs`, to customize
- its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
-
- The method is designed to be compatible with SKLearn-like classes
- and in particular to be compatible with the StatsForecast library.
-
- By default the `model` is not saving training checkpoints to protect
- disk memory, to get them change `enable_checkpointing=True` in `__init__`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `val_size`: int, validation size for temporal cross-validation.
- `test_size`: int, test size for temporal cross-validation.
- """
- if distributed_config is not None:
- raise ValueError(
- "multivariate models cannot be trained using distributed data parallel."
- )
- return self._fit(
- dataset=dataset,
- batch_size=self.n_series,
- valid_batch_size=self.n_series,
- val_size=val_size,
- test_size=test_size,
- random_seed=random_seed,
- shuffle_train=False,
- distributed_config=None,
- )
-
- def predict(
- self,
- dataset,
- test_size=None,
- step_size=1,
- random_seed=None,
- **data_module_kwargs,
- ):
- """Predict.
-
- Neural network prediction with PL's `Trainer` execution of `predict_step`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `test_size`: int=None, test size for temporal cross-validation.
- `step_size`: int=1, Step size between each window.
- `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).
- """
- self._check_exog(dataset)
- self._restart_seed(random_seed)
- data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)
-
- self.predict_step_size = step_size
- self.decompose_forecast = False
- datamodule = TimeSeriesDataModule(
- dataset=dataset,
- valid_batch_size=self.n_series,
- batch_size=self.n_series,
- **data_module_kwargs,
- )
-
- # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.
- pred_trainer_kwargs = self.trainer_kwargs.copy()
- if (pred_trainer_kwargs.get("accelerator", None) == "gpu") and (
- torch.cuda.device_count() > 1
- ):
- pred_trainer_kwargs["devices"] = [0]
-
- trainer = pl.Trainer(**pred_trainer_kwargs)
- fcsts = trainer.predict(self, datamodule=datamodule)
- fcsts = torch.vstack(fcsts).numpy()
-
- fcsts = np.transpose(fcsts, (2, 0, 1))
- fcsts = fcsts.flatten()
- fcsts = fcsts.reshape(-1, len(self.loss.output_names))
- return fcsts
-
- def decompose(self, dataset, step_size=1, random_seed=None, **data_module_kwargs):
- raise NotImplementedError("decompose")
diff --git a/neuralforecast/common/_base_recurrent.py b/neuralforecast/common/_base_recurrent.py
deleted file mode 100644
index 0479996c1..000000000
--- a/neuralforecast/common/_base_recurrent.py
+++ /dev/null
@@ -1,593 +0,0 @@
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/common.base_recurrent.ipynb.
-
-# %% auto 0
-__all__ = ['BaseRecurrent']
-
-# %% ../../nbs/common.base_recurrent.ipynb 6
-import numpy as np
-import torch
-import torch.nn as nn
-import pytorch_lightning as pl
-import neuralforecast.losses.pytorch as losses
-
-from ._base_model import BaseModel
-from ._scalers import TemporalNorm
-from ..tsdataset import TimeSeriesDataModule
-from ..utils import get_indexer_raise_missing
-
-# %% ../../nbs/common.base_recurrent.ipynb 7
-class BaseRecurrent(BaseModel):
- """Base Recurrent
-
- Base class for all recurrent-based models. The forecasts are produced sequentially between
- windows.
-
- This class implements the basic functionality for all windows-based models, including:
- - PyTorch Lightning's methods training_step, validation_step, predict_step.
- - fit and predict methods used by NeuralForecast.core class.
- - sampling and wrangling methods to sequential windows.
- """
-
- def __init__(
- self,
- h,
- input_size,
- inference_input_size,
- loss,
- valid_loss,
- learning_rate,
- max_steps,
- val_check_steps,
- batch_size,
- valid_batch_size,
- scaler_type="robust",
- num_lr_decays=0,
- early_stop_patience_steps=-1,
- futr_exog_list=None,
- hist_exog_list=None,
- stat_exog_list=None,
- num_workers_loader=0,
- drop_last_loader=False,
- random_seed=1,
- alias=None,
- optimizer=None,
- optimizer_kwargs=None,
- lr_scheduler=None,
- lr_scheduler_kwargs=None,
- dataloader_kwargs=None,
- **trainer_kwargs,
- ):
- super().__init__(
- random_seed=random_seed,
- loss=loss,
- valid_loss=valid_loss,
- optimizer=optimizer,
- optimizer_kwargs=optimizer_kwargs,
- lr_scheduler=lr_scheduler,
- lr_scheduler_kwargs=lr_scheduler_kwargs,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
- max_steps=max_steps,
- early_stop_patience_steps=early_stop_patience_steps,
- **trainer_kwargs,
- )
-
- # Padder to complete train windows,
- # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]
- self.h = h
- self.input_size = input_size
- self.inference_input_size = inference_input_size
- self.padder = nn.ConstantPad1d(padding=(0, self.h), value=0.0)
-
- unsupported_distributions = ["Bernoulli", "ISQF"]
- if (
- isinstance(self.loss, losses.DistributionLoss)
- and self.loss.distribution in unsupported_distributions
- ):
- raise Exception(
- f"Distribution {self.loss.distribution} not available for Recurrent-based models. Please choose another distribution."
- )
-
- # Valid batch_size
- self.batch_size = batch_size
- if valid_batch_size is None:
- self.valid_batch_size = batch_size
- else:
- self.valid_batch_size = valid_batch_size
-
- # Optimization
- self.learning_rate = learning_rate
- self.max_steps = max_steps
- self.num_lr_decays = num_lr_decays
- self.lr_decay_steps = (
- max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7
- )
- self.early_stop_patience_steps = early_stop_patience_steps
- self.val_check_steps = val_check_steps
-
- # Scaler
- self.scaler = TemporalNorm(
- scaler_type=scaler_type,
- dim=-1, # Time dimension is -1.
- num_features=1 + len(self.hist_exog_list) + len(self.futr_exog_list),
- )
-
- # Fit arguments
- self.val_size = 0
- self.test_size = 0
-
- # DataModule arguments
- self.num_workers_loader = num_workers_loader
- self.dataloader_kwargs = dataloader_kwargs
- self.drop_last_loader = drop_last_loader
- # used by on_validation_epoch_end hook
- self.validation_step_outputs = []
- self.alias = alias
-
- def _normalization(self, batch, val_size=0, test_size=0):
- temporal = batch["temporal"] # B, C, T
- temporal_cols = batch["temporal_cols"].copy()
- y_idx = batch["y_idx"]
-
- # Separate data and mask
- temporal_data_cols = self._get_temporal_exogenous_cols(
- temporal_cols=temporal_cols
- )
- temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)
- temporal_idxs = np.append(y_idx, temporal_idxs)
- temporal_data = temporal[:, temporal_idxs, :]
- temporal_mask = temporal[:, temporal_cols.get_loc("available_mask"), :].clone()
-
- # Remove validation and test set to prevent leakeage
- if val_size + test_size > 0:
- cutoff = val_size + test_size
- temporal_mask[:, -cutoff:] = 0
-
- # Normalize. self.scaler stores the shift and scale for inverse transform
- temporal_mask = temporal_mask.unsqueeze(
- 1
- ) # Add channel dimension for scaler.transform.
- temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)
-
- # Replace values in windows dict
- temporal[:, temporal_idxs, :] = temporal_data
- batch["temporal"] = temporal
-
- return batch
-
- def _inv_normalization(self, y_hat, temporal_cols, y_idx):
- # Receives window predictions [B, seq_len, H, output]
- # Broadcasts outputs and inverts normalization
-
- # Get 'y' scale and shift, and add W dimension
- y_loc = self.scaler.x_shift[:, [y_idx], 0].flatten() # [B,C,T] -> [B]
- y_scale = self.scaler.x_scale[:, [y_idx], 0].flatten() # [B,C,T] -> [B]
-
- # Expand scale and shift to y_hat dimensions
- y_loc = y_loc.view(*y_loc.shape, *(1,) * (y_hat.ndim - 1)) # .expand(y_hat)
- y_scale = y_scale.view(
- *y_scale.shape, *(1,) * (y_hat.ndim - 1)
- ) # .expand(y_hat)
-
- y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)
-
- return y_hat, y_loc, y_scale
-
- def _create_windows(self, batch, step):
- temporal = batch["temporal"]
- temporal_cols = batch["temporal_cols"]
-
- if step == "train":
- if self.val_size + self.test_size > 0:
- cutoff = -self.val_size - self.test_size
- temporal = temporal[:, :, :cutoff]
- temporal = self.padder(temporal)
-
- # Truncate batch to shorter time-series
- av_condition = torch.nonzero(
- torch.min(
- temporal[:, temporal_cols.get_loc("available_mask")], axis=0
- ).values
- )
- min_time_stamp = int(av_condition.min())
-
- available_ts = temporal.shape[-1] - min_time_stamp
- if available_ts < 1 + self.h:
- raise Exception(
- "Time series too short for given input and output size. \n"
- f"Available timestamps: {available_ts}"
- )
-
- temporal = temporal[:, :, min_time_stamp:]
-
- if step == "val":
- if self.test_size > 0:
- temporal = temporal[:, :, : -self.test_size]
- temporal = self.padder(temporal)
-
- if step == "predict":
- if (self.test_size == 0) and (len(self.futr_exog_list) == 0):
- temporal = self.padder(temporal)
-
- # Test size covers all data, pad left one timestep with zeros
- if temporal.shape[-1] == self.test_size:
- padder_left = nn.ConstantPad1d(padding=(1, 0), value=0.0)
- temporal = padder_left(temporal)
-
- # Parse batch
- window_size = 1 + self.h # 1 for current t and h for future
- windows = temporal.unfold(dimension=-1, size=window_size, step=1)
-
- # Truncated backprogatation/inference (shorten sequence where RNNs unroll)
- n_windows = windows.shape[2]
- input_size = -1
- if (step == "train") and (self.input_size > 0):
- input_size = self.input_size
- if (input_size > 0) and (n_windows > input_size):
- max_sampleable_time = n_windows - self.input_size + 1
- start = np.random.choice(max_sampleable_time)
- windows = windows[:, :, start : (start + input_size), :]
-
- if (step == "val") and (self.inference_input_size > 0):
- cutoff = self.inference_input_size + self.val_size
- windows = windows[:, :, -cutoff:, :]
-
- if (step == "predict") and (self.inference_input_size > 0):
- cutoff = self.inference_input_size + self.test_size
- windows = windows[:, :, -cutoff:, :]
-
- # [B, C, input_size, 1+H]
- windows_batch = dict(
- temporal=windows,
- temporal_cols=temporal_cols,
- static=batch.get("static", None),
- static_cols=batch.get("static_cols", None),
- )
-
- return windows_batch
-
- def _parse_windows(self, batch, windows):
- # [B, C, seq_len, 1+H]
- # Filter insample lags from outsample horizon
- mask_idx = batch["temporal_cols"].get_loc("available_mask")
- y_idx = batch["y_idx"]
- insample_y = windows["temporal"][:, y_idx, :, : -self.h]
- insample_mask = windows["temporal"][:, mask_idx, :, : -self.h]
- outsample_y = windows["temporal"][:, y_idx, :, -self.h :].contiguous()
- outsample_mask = windows["temporal"][:, mask_idx, :, -self.h :].contiguous()
-
- # Filter historic exogenous variables
- if len(self.hist_exog_list):
- hist_exog_idx = get_indexer_raise_missing(
- windows["temporal_cols"], self.hist_exog_list
- )
- hist_exog = windows["temporal"][:, hist_exog_idx, :, : -self.h]
- else:
- hist_exog = None
-
- # Filter future exogenous variables
- if len(self.futr_exog_list):
- futr_exog_idx = get_indexer_raise_missing(
- windows["temporal_cols"], self.futr_exog_list
- )
- futr_exog = windows["temporal"][:, futr_exog_idx, :, :]
- else:
- futr_exog = None
- # Filter static variables
- if len(self.stat_exog_list):
- static_idx = get_indexer_raise_missing(
- windows["static_cols"], self.stat_exog_list
- )
- stat_exog = windows["static"][:, static_idx]
- else:
- stat_exog = None
-
- return (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- )
-
- def training_step(self, batch, batch_idx):
- # Create and normalize windows [Ws, L+H, C]
- batch = self._normalization(
- batch, val_size=self.val_size, test_size=self.test_size
- )
- windows = self._create_windows(batch, step="train")
-
- # Parse windows
- (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- ) = self._parse_windows(batch, windows)
-
- windows_batch = dict(
- insample_y=insample_y, # [B, seq_len, 1]
- insample_mask=insample_mask, # [B, seq_len, 1]
- futr_exog=futr_exog, # [B, F, seq_len, 1+H]
- hist_exog=hist_exog, # [B, C, seq_len]
- stat_exog=stat_exog,
- ) # [B, S]
-
- # Model predictions
- output = self(windows_batch) # tuple([B, seq_len, H, output])
- if self.loss.is_distribution_output:
- outsample_y, y_loc, y_scale = self._inv_normalization(
- y_hat=outsample_y,
- temporal_cols=batch["temporal_cols"],
- y_idx=batch["y_idx"],
- )
- B = output[0].size()[0]
- T = output[0].size()[1]
- H = output[0].size()[2]
- output = [arg.view(-1, *(arg.size()[2:])) for arg in output]
- outsample_y = outsample_y.view(B * T, H)
- outsample_mask = outsample_mask.view(B * T, H)
- y_loc = y_loc.repeat_interleave(repeats=T, dim=0).squeeze(-1)
- y_scale = y_scale.repeat_interleave(repeats=T, dim=0).squeeze(-1)
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)
- else:
- loss = self.loss(y=outsample_y, y_hat=output, mask=outsample_mask)
-
- if torch.isnan(loss):
- print("Model Parameters", self.hparams)
- print("insample_y", torch.isnan(insample_y).sum())
- print("outsample_y", torch.isnan(outsample_y).sum())
- print("output", torch.isnan(output).sum())
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "train_loss",
- loss.detach().item(),
- batch_size=outsample_y.size(0),
- prog_bar=True,
- on_epoch=True,
- )
- self.train_trajectories.append((self.global_step, loss.detach().item()))
- return loss
-
- def validation_step(self, batch, batch_idx):
- if self.val_size == 0:
- return np.nan
-
- # Create and normalize windows [Ws, L+H, C]
- batch = self._normalization(
- batch, val_size=self.val_size, test_size=self.test_size
- )
- windows = self._create_windows(batch, step="val")
- y_idx = batch["y_idx"]
-
- # Parse windows
- (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- ) = self._parse_windows(batch, windows)
-
- windows_batch = dict(
- insample_y=insample_y, # [B, seq_len, 1]
- insample_mask=insample_mask, # [B, seq_len, 1]
- futr_exog=futr_exog, # [B, F, seq_len, 1+H]
- hist_exog=hist_exog, # [B, C, seq_len]
- stat_exog=stat_exog,
- ) # [B, S]
-
- # Remove train y_hat (+1 and -1 for padded last window with zeros)
- # tuple([B, seq_len, H, output]) -> tuple([B, validation_size, H, output])
- val_windows = (self.val_size) + 1
- outsample_y = outsample_y[:, -val_windows:-1, :]
- outsample_mask = outsample_mask[:, -val_windows:-1, :]
-
- # Model predictions
- output = self(windows_batch) # tuple([B, seq_len, H, output])
- if self.loss.is_distribution_output:
- output = [arg[:, -val_windows:-1] for arg in output]
- outsample_y, y_loc, y_scale = self._inv_normalization(
- y_hat=outsample_y, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- B = output[0].size()[0]
- T = output[0].size()[1]
- H = output[0].size()[2]
- output = [arg.reshape(-1, *(arg.size()[2:])) for arg in output]
- outsample_y = outsample_y.reshape(B * T, H)
- outsample_mask = outsample_mask.reshape(B * T, H)
- y_loc = y_loc.repeat_interleave(repeats=T, dim=0).squeeze(-1)
- y_scale = y_scale.repeat_interleave(repeats=T, dim=0).squeeze(-1)
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- _, sample_mean, quants = self.loss.sample(distr_args=distr_args)
-
- if str(type(self.valid_loss)) in [
- "",
- "",
- ]:
- output = quants
- elif str(type(self.valid_loss)) in [
- ""
- ]:
- output = torch.unsqueeze(sample_mean, dim=-1) # [N,H,1] -> [N,H]
-
- else:
- output = output[:, -val_windows:-1, :]
-
- # Validation Loss evaluation
- if self.valid_loss.is_distribution_output:
- valid_loss = self.valid_loss(
- y=outsample_y, distr_args=distr_args, mask=outsample_mask
- )
- else:
- outsample_y, _, _ = self._inv_normalization(
- y_hat=outsample_y, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- output, _, _ = self._inv_normalization(
- y_hat=output, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- valid_loss = self.valid_loss(
- y=outsample_y, y_hat=output, mask=outsample_mask
- )
-
- if torch.isnan(valid_loss):
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "valid_loss",
- valid_loss.detach().item(),
- batch_size=outsample_y.size(0),
- prog_bar=True,
- on_epoch=True,
- )
- self.validation_step_outputs.append(valid_loss)
- return valid_loss
-
- def predict_step(self, batch, batch_idx):
- # Create and normalize windows [Ws, L+H, C]
- batch = self._normalization(batch, val_size=0, test_size=self.test_size)
- windows = self._create_windows(batch, step="predict")
- y_idx = batch["y_idx"]
-
- # Parse windows
- insample_y, insample_mask, _, _, hist_exog, futr_exog, stat_exog = (
- self._parse_windows(batch, windows)
- )
-
- windows_batch = dict(
- insample_y=insample_y, # [B, seq_len, 1]
- insample_mask=insample_mask, # [B, seq_len, 1]
- futr_exog=futr_exog, # [B, F, seq_len, 1+H]
- hist_exog=hist_exog, # [B, C, seq_len]
- stat_exog=stat_exog,
- ) # [B, S]
-
- # Model Predictions
- output = self(windows_batch) # tuple([B, seq_len, H], ...)
- if self.loss.is_distribution_output:
- _, y_loc, y_scale = self._inv_normalization(
- y_hat=output[0], temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- B = output[0].size()[0]
- T = output[0].size()[1]
- H = output[0].size()[2]
- output = [arg.reshape(-1, *(arg.size()[2:])) for arg in output]
- y_loc = y_loc.repeat_interleave(repeats=T, dim=0).squeeze(-1)
- y_scale = y_scale.repeat_interleave(repeats=T, dim=0).squeeze(-1)
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- _, sample_mean, quants = self.loss.sample(distr_args=distr_args)
- y_hat = torch.concat((sample_mean, quants), axis=2)
- y_hat = y_hat.view(B, T, H, -1)
-
- if self.loss.return_params:
- distr_args = torch.stack(distr_args, dim=-1)
- distr_args = torch.reshape(distr_args, (B, T, H, -1))
- y_hat = torch.concat((y_hat, distr_args), axis=3)
- else:
- y_hat, _, _ = self._inv_normalization(
- y_hat=output, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- return y_hat
-
- def fit(
- self,
- dataset,
- val_size=0,
- test_size=0,
- random_seed=None,
- distributed_config=None,
- ):
- """Fit.
-
- The `fit` method, optimizes the neural network's weights using the
- initialization parameters (`learning_rate`, `batch_size`, ...)
- and the `loss` function as defined during the initialization.
- Within `fit` we use a PyTorch Lightning `Trainer` that
- inherits the initialization's `self.trainer_kwargs`, to customize
- its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
-
- The method is designed to be compatible with SKLearn-like classes
- and in particular to be compatible with the StatsForecast library.
-
- By default the `model` is not saving training checkpoints to protect
- disk memory, to get them change `enable_checkpointing=True` in `__init__`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `val_size`: int, validation size for temporal cross-validation.
- `test_size`: int, test size for temporal cross-validation.
- `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
- """
- return self._fit(
- dataset=dataset,
- batch_size=self.batch_size,
- valid_batch_size=self.valid_batch_size,
- val_size=val_size,
- test_size=test_size,
- random_seed=random_seed,
- distributed_config=distributed_config,
- )
-
- def predict(self, dataset, step_size=1, random_seed=None, **data_module_kwargs):
- """Predict.
-
- Neural network prediction with PL's `Trainer` execution of `predict_step`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `step_size`: int=1, Step size between each window.
- `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
- `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).
- """
- self._check_exog(dataset)
- self._restart_seed(random_seed)
- data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)
-
- if step_size > 1:
- raise Exception("Recurrent models do not support step_size > 1")
-
- # fcsts (window, batch, h)
- # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.
- pred_trainer_kwargs = self.trainer_kwargs.copy()
- if (pred_trainer_kwargs.get("accelerator", None) == "gpu") and (
- torch.cuda.device_count() > 1
- ):
- pred_trainer_kwargs["devices"] = [0]
-
- trainer = pl.Trainer(**pred_trainer_kwargs)
-
- datamodule = TimeSeriesDataModule(
- dataset=dataset,
- valid_batch_size=self.valid_batch_size,
- num_workers=self.num_workers_loader,
- **data_module_kwargs,
- )
- fcsts = trainer.predict(self, datamodule=datamodule)
- if self.test_size > 0:
- # Remove warmup windows (from train and validation)
- # [N,T,H,output], avoid indexing last dim for univariate output compatibility
- fcsts = torch.vstack(
- [fcst[:, -(1 + self.test_size - self.h) :, :] for fcst in fcsts]
- )
- fcsts = fcsts.numpy().flatten()
- fcsts = fcsts.reshape(-1, len(self.loss.output_names))
- else:
- fcsts = torch.vstack([fcst[:, -1:, :] for fcst in fcsts]).numpy().flatten()
- fcsts = fcsts.reshape(-1, len(self.loss.output_names))
- return fcsts
diff --git a/neuralforecast/common/_base_windows.py b/neuralforecast/common/_base_windows.py
deleted file mode 100644
index dd4a4c869..000000000
--- a/neuralforecast/common/_base_windows.py
+++ /dev/null
@@ -1,744 +0,0 @@
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/common.base_windows.ipynb.
-
-# %% auto 0
-__all__ = ['BaseWindows']
-
-# %% ../../nbs/common.base_windows.ipynb 5
-import numpy as np
-import torch
-import torch.nn as nn
-import pytorch_lightning as pl
-
-from ._base_model import BaseModel
-from ._scalers import TemporalNorm
-from ..tsdataset import TimeSeriesDataModule
-from ..utils import get_indexer_raise_missing
-
-# %% ../../nbs/common.base_windows.ipynb 6
-class BaseWindows(BaseModel):
- """Base Windows
-
- Base class for all windows-based models. The forecasts are produced separately
- for each window, which are randomly sampled during training.
-
- This class implements the basic functionality for all windows-based models, including:
- - PyTorch Lightning's methods training_step, validation_step, predict_step.
- - fit and predict methods used by NeuralForecast.core class.
- - sampling and wrangling methods to generate windows.
- """
-
- def __init__(
- self,
- h,
- input_size,
- loss,
- valid_loss,
- learning_rate,
- max_steps,
- val_check_steps,
- batch_size,
- valid_batch_size,
- windows_batch_size,
- inference_windows_batch_size,
- start_padding_enabled,
- step_size=1,
- num_lr_decays=0,
- early_stop_patience_steps=-1,
- scaler_type="identity",
- futr_exog_list=None,
- hist_exog_list=None,
- stat_exog_list=None,
- exclude_insample_y=False,
- num_workers_loader=0,
- drop_last_loader=False,
- random_seed=1,
- alias=None,
- optimizer=None,
- optimizer_kwargs=None,
- lr_scheduler=None,
- lr_scheduler_kwargs=None,
- dataloader_kwargs=None,
- **trainer_kwargs,
- ):
- super().__init__(
- random_seed=random_seed,
- loss=loss,
- valid_loss=valid_loss,
- optimizer=optimizer,
- optimizer_kwargs=optimizer_kwargs,
- lr_scheduler=lr_scheduler,
- lr_scheduler_kwargs=lr_scheduler_kwargs,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
- max_steps=max_steps,
- early_stop_patience_steps=early_stop_patience_steps,
- **trainer_kwargs,
- )
-
- # Padder to complete train windows,
- # example y=[1,2,3,4,5] h=3 -> last y_output = [5,0,0]
- self.h = h
- self.input_size = input_size
- self.windows_batch_size = windows_batch_size
- self.start_padding_enabled = start_padding_enabled
- if start_padding_enabled:
- self.padder_train = nn.ConstantPad1d(
- padding=(self.input_size - 1, self.h), value=0.0
- )
- else:
- self.padder_train = nn.ConstantPad1d(padding=(0, self.h), value=0.0)
-
- # Batch sizes
- self.batch_size = batch_size
- if valid_batch_size is None:
- self.valid_batch_size = batch_size
- else:
- self.valid_batch_size = valid_batch_size
- if inference_windows_batch_size is None:
- self.inference_windows_batch_size = windows_batch_size
- else:
- self.inference_windows_batch_size = inference_windows_batch_size
-
- # Optimization
- self.learning_rate = learning_rate
- self.max_steps = max_steps
- self.num_lr_decays = num_lr_decays
- self.lr_decay_steps = (
- max(max_steps // self.num_lr_decays, 1) if self.num_lr_decays > 0 else 10e7
- )
- self.early_stop_patience_steps = early_stop_patience_steps
- self.val_check_steps = val_check_steps
- self.windows_batch_size = windows_batch_size
- self.step_size = step_size
-
- self.exclude_insample_y = exclude_insample_y
-
- # Scaler
- self.scaler = TemporalNorm(
- scaler_type=scaler_type,
- dim=1, # Time dimension is 1.
- num_features=1 + len(self.hist_exog_list) + len(self.futr_exog_list),
- )
-
- # Fit arguments
- self.val_size = 0
- self.test_size = 0
-
- # Model state
- self.decompose_forecast = False
-
- # DataModule arguments
- self.num_workers_loader = num_workers_loader
- self.dataloader_kwargs = dataloader_kwargs
- self.drop_last_loader = drop_last_loader
- # used by on_validation_epoch_end hook
- self.validation_step_outputs = []
- self.alias = alias
-
- def _create_windows(self, batch, step, w_idxs=None):
- # Parse common data
- window_size = self.input_size + self.h
- temporal_cols = batch["temporal_cols"]
- temporal = batch["temporal"]
-
- if step == "train":
- if self.val_size + self.test_size > 0:
- cutoff = -self.val_size - self.test_size
- temporal = temporal[:, :, :cutoff]
-
- temporal = self.padder_train(temporal)
- if temporal.shape[-1] < window_size:
- raise Exception(
- "Time series is too short for training, consider setting a smaller input size or set start_padding_enabled=True"
- )
- windows = temporal.unfold(
- dimension=-1, size=window_size, step=self.step_size
- )
-
- # [B, C, Ws, L+H] 0, 1, 2, 3
- # -> [B * Ws, L+H, C] 0, 2, 3, 1
- windows_per_serie = windows.shape[2]
- windows = windows.permute(0, 2, 3, 1).contiguous()
- windows = windows.reshape(-1, window_size, len(temporal_cols))
-
- # Sample and Available conditions
- available_idx = temporal_cols.get_loc("available_mask")
- available_condition = windows[:, : self.input_size, available_idx]
- available_condition = torch.sum(available_condition, axis=1)
- final_condition = available_condition > 0
- if self.h > 0:
- sample_condition = windows[:, self.input_size :, available_idx]
- sample_condition = torch.sum(sample_condition, axis=1)
- final_condition = (sample_condition > 0) & (available_condition > 0)
- windows = windows[final_condition]
-
- # Parse Static data to match windows
- # [B, S_in] -> [B, Ws, S_in] -> [B*Ws, S_in]
- static = batch.get("static", None)
- static_cols = batch.get("static_cols", None)
- if static is not None:
- static = torch.repeat_interleave(
- static, repeats=windows_per_serie, dim=0
- )
- static = static[final_condition]
-
- # Protection of empty windows
- if final_condition.sum() == 0:
- raise Exception("No windows available for training")
-
- # Sample windows
- n_windows = len(windows)
- if self.windows_batch_size is not None:
- w_idxs = np.random.choice(
- n_windows,
- size=self.windows_batch_size,
- replace=(n_windows < self.windows_batch_size),
- )
- windows = windows[w_idxs]
-
- if static is not None:
- static = static[w_idxs]
-
- # think about interaction available * sample mask
- # [B, C, Ws, L+H]
- windows_batch = dict(
- temporal=windows,
- temporal_cols=temporal_cols,
- static=static,
- static_cols=static_cols,
- )
- return windows_batch
-
- elif step in ["predict", "val"]:
-
- if step == "predict":
- initial_input = temporal.shape[-1] - self.test_size
- if (
- initial_input <= self.input_size
- ): # There is not enough data to predict first timestamp
- padder_left = nn.ConstantPad1d(
- padding=(self.input_size - initial_input, 0), value=0.0
- )
- temporal = padder_left(temporal)
- predict_step_size = self.predict_step_size
- cutoff = -self.input_size - self.test_size
- temporal = temporal[:, :, cutoff:]
-
- elif step == "val":
- predict_step_size = self.step_size
- cutoff = -self.input_size - self.val_size - self.test_size
- if self.test_size > 0:
- temporal = batch["temporal"][:, :, cutoff : -self.test_size]
- else:
- temporal = batch["temporal"][:, :, cutoff:]
- if temporal.shape[-1] < window_size:
- initial_input = temporal.shape[-1] - self.val_size
- padder_left = nn.ConstantPad1d(
- padding=(self.input_size - initial_input, 0), value=0.0
- )
- temporal = padder_left(temporal)
-
- if (
- (step == "predict")
- and (self.test_size == 0)
- and (len(self.futr_exog_list) == 0)
- ):
- padder_right = nn.ConstantPad1d(padding=(0, self.h), value=0.0)
- temporal = padder_right(temporal)
-
- windows = temporal.unfold(
- dimension=-1, size=window_size, step=predict_step_size
- )
-
- # [batch, channels, windows, window_size] 0, 1, 2, 3
- # -> [batch * windows, window_size, channels] 0, 2, 3, 1
- windows_per_serie = windows.shape[2]
- windows = windows.permute(0, 2, 3, 1).contiguous()
- windows = windows.reshape(-1, window_size, len(temporal_cols))
-
- static = batch.get("static", None)
- static_cols = batch.get("static_cols", None)
- if static is not None:
- static = torch.repeat_interleave(
- static, repeats=windows_per_serie, dim=0
- )
-
- # Sample windows for batched prediction
- if w_idxs is not None:
- windows = windows[w_idxs]
- if static is not None:
- static = static[w_idxs]
-
- windows_batch = dict(
- temporal=windows,
- temporal_cols=temporal_cols,
- static=static,
- static_cols=static_cols,
- )
- return windows_batch
- else:
- raise ValueError(f"Unknown step {step}")
-
- def _normalization(self, windows, y_idx):
- # windows are already filtered by train/validation/test
- # from the `create_windows_method` nor leakage risk
- temporal = windows["temporal"] # B, L+H, C
- temporal_cols = windows["temporal_cols"].copy() # B, L+H, C
-
- # To avoid leakage uses only the lags
- # temporal_data_cols = temporal_cols.drop('available_mask').tolist()
- temporal_data_cols = self._get_temporal_exogenous_cols(
- temporal_cols=temporal_cols
- )
- temporal_idxs = get_indexer_raise_missing(temporal_cols, temporal_data_cols)
- temporal_idxs = np.append(y_idx, temporal_idxs)
- temporal_data = temporal[:, :, temporal_idxs]
- temporal_mask = temporal[:, :, temporal_cols.get_loc("available_mask")].clone()
- if self.h > 0:
- temporal_mask[:, -self.h :] = 0.0
-
- # Normalize. self.scaler stores the shift and scale for inverse transform
- temporal_mask = temporal_mask.unsqueeze(
- -1
- ) # Add channel dimension for scaler.transform.
- temporal_data = self.scaler.transform(x=temporal_data, mask=temporal_mask)
-
- # Replace values in windows dict
- temporal[:, :, temporal_idxs] = temporal_data
- windows["temporal"] = temporal
-
- return windows
-
- def _inv_normalization(self, y_hat, temporal_cols, y_idx):
- # Receives window predictions [B, H, output]
- # Broadcasts outputs and inverts normalization
-
- # Add C dimension
- if y_hat.ndim == 2:
- remove_dimension = True
- y_hat = y_hat.unsqueeze(-1)
- else:
- remove_dimension = False
-
- y_scale = self.scaler.x_scale[:, :, [y_idx]]
- y_loc = self.scaler.x_shift[:, :, [y_idx]]
-
- y_scale = torch.repeat_interleave(y_scale, repeats=y_hat.shape[-1], dim=-1).to(
- y_hat.device
- )
- y_loc = torch.repeat_interleave(y_loc, repeats=y_hat.shape[-1], dim=-1).to(
- y_hat.device
- )
-
- y_hat = self.scaler.inverse_transform(z=y_hat, x_scale=y_scale, x_shift=y_loc)
- y_loc = y_loc.to(y_hat.device)
- y_scale = y_scale.to(y_hat.device)
-
- if remove_dimension:
- y_hat = y_hat.squeeze(-1)
- y_loc = y_loc.squeeze(-1)
- y_scale = y_scale.squeeze(-1)
-
- return y_hat, y_loc, y_scale
-
- def _parse_windows(self, batch, windows):
- # Filter insample lags from outsample horizon
- y_idx = batch["y_idx"]
- mask_idx = batch["temporal_cols"].get_loc("available_mask")
-
- insample_y = windows["temporal"][:, : self.input_size, y_idx]
- insample_mask = windows["temporal"][:, : self.input_size, mask_idx]
-
- # Declare additional information
- outsample_y = None
- outsample_mask = None
- hist_exog = None
- futr_exog = None
- stat_exog = None
-
- if self.h > 0:
- outsample_y = windows["temporal"][:, self.input_size :, y_idx]
- outsample_mask = windows["temporal"][:, self.input_size :, mask_idx]
-
- if len(self.hist_exog_list):
- hist_exog_idx = get_indexer_raise_missing(
- windows["temporal_cols"], self.hist_exog_list
- )
- hist_exog = windows["temporal"][:, : self.input_size, hist_exog_idx]
-
- if len(self.futr_exog_list):
- futr_exog_idx = get_indexer_raise_missing(
- windows["temporal_cols"], self.futr_exog_list
- )
- futr_exog = windows["temporal"][:, :, futr_exog_idx]
-
- if len(self.stat_exog_list):
- static_idx = get_indexer_raise_missing(
- windows["static_cols"], self.stat_exog_list
- )
- stat_exog = windows["static"][:, static_idx]
-
- # TODO: think a better way of removing insample_y features
- if self.exclude_insample_y:
- insample_y = insample_y * 0
-
- return (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- )
-
- def training_step(self, batch, batch_idx):
- # Create and normalize windows [Ws, L+H, C]
- windows = self._create_windows(batch, step="train")
- y_idx = batch["y_idx"]
- original_outsample_y = torch.clone(windows["temporal"][:, -self.h :, y_idx])
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- (
- insample_y,
- insample_mask,
- outsample_y,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- ) = self._parse_windows(batch, windows)
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L]
- insample_mask=insample_mask, # [Ws, L]
- futr_exog=futr_exog, # [Ws, L + h, F]
- hist_exog=hist_exog, # [Ws, L, X]
- stat_exog=stat_exog,
- ) # [Ws, S]
-
- # Model Predictions
- output = self(windows_batch)
- if self.loss.is_distribution_output:
- _, y_loc, y_scale = self._inv_normalization(
- y_hat=outsample_y, temporal_cols=batch["temporal_cols"], y_idx=y_idx
- )
- outsample_y = original_outsample_y
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- loss = self.loss(y=outsample_y, distr_args=distr_args, mask=outsample_mask)
- else:
- loss = self.loss(y=outsample_y, y_hat=output, mask=outsample_mask)
-
- if torch.isnan(loss):
- print("Model Parameters", self.hparams)
- print("insample_y", torch.isnan(insample_y).sum())
- print("outsample_y", torch.isnan(outsample_y).sum())
- print("output", torch.isnan(output).sum())
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "train_loss",
- loss.detach().item(),
- batch_size=outsample_y.size(0),
- prog_bar=True,
- on_epoch=True,
- )
- self.train_trajectories.append((self.global_step, loss.detach().item()))
- return loss
-
- def _compute_valid_loss(
- self, outsample_y, output, outsample_mask, temporal_cols, y_idx
- ):
- if self.loss.is_distribution_output:
- _, y_loc, y_scale = self._inv_normalization(
- y_hat=outsample_y, temporal_cols=temporal_cols, y_idx=y_idx
- )
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- _, sample_mean, quants = self.loss.sample(distr_args=distr_args)
-
- if str(type(self.valid_loss)) in [
- "",
- "",
- ]:
- output = quants
- elif str(type(self.valid_loss)) in [
- ""
- ]:
- output = torch.unsqueeze(sample_mean, dim=-1) # [N,H,1] -> [N,H]
-
- # Validation Loss evaluation
- if self.valid_loss.is_distribution_output:
- valid_loss = self.valid_loss(
- y=outsample_y, distr_args=distr_args, mask=outsample_mask
- )
- else:
- output, _, _ = self._inv_normalization(
- y_hat=output, temporal_cols=temporal_cols, y_idx=y_idx
- )
- valid_loss = self.valid_loss(
- y=outsample_y, y_hat=output, mask=outsample_mask
- )
- return valid_loss
-
- def validation_step(self, batch, batch_idx):
- if self.val_size == 0:
- return np.nan
-
- # TODO: Hack to compute number of windows
- windows = self._create_windows(batch, step="val")
- n_windows = len(windows["temporal"])
- y_idx = batch["y_idx"]
-
- # Number of windows in batch
- windows_batch_size = self.inference_windows_batch_size
- if windows_batch_size < 0:
- windows_batch_size = n_windows
- n_batches = int(np.ceil(n_windows / windows_batch_size))
-
- valid_losses = []
- batch_sizes = []
- for i in range(n_batches):
- # Create and normalize windows [Ws, L+H, C]
- w_idxs = np.arange(
- i * windows_batch_size, min((i + 1) * windows_batch_size, n_windows)
- )
- windows = self._create_windows(batch, step="val", w_idxs=w_idxs)
- original_outsample_y = torch.clone(windows["temporal"][:, -self.h :, y_idx])
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- (
- insample_y,
- insample_mask,
- _,
- outsample_mask,
- hist_exog,
- futr_exog,
- stat_exog,
- ) = self._parse_windows(batch, windows)
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L]
- insample_mask=insample_mask, # [Ws, L]
- futr_exog=futr_exog, # [Ws, L + h, F]
- hist_exog=hist_exog, # [Ws, L, X]
- stat_exog=stat_exog,
- ) # [Ws, S]
-
- # Model Predictions
- output_batch = self(windows_batch)
- valid_loss_batch = self._compute_valid_loss(
- outsample_y=original_outsample_y,
- output=output_batch,
- outsample_mask=outsample_mask,
- temporal_cols=batch["temporal_cols"],
- y_idx=batch["y_idx"],
- )
- valid_losses.append(valid_loss_batch)
- batch_sizes.append(len(output_batch))
-
- valid_loss = torch.stack(valid_losses)
- batch_sizes = torch.tensor(batch_sizes, device=valid_loss.device)
- batch_size = torch.sum(batch_sizes)
- valid_loss = torch.sum(valid_loss * batch_sizes) / batch_size
-
- if torch.isnan(valid_loss):
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "valid_loss",
- valid_loss.detach().item(),
- batch_size=batch_size,
- prog_bar=True,
- on_epoch=True,
- )
- self.validation_step_outputs.append(valid_loss)
- return valid_loss
-
- def predict_step(self, batch, batch_idx):
-
- # TODO: Hack to compute number of windows
- windows = self._create_windows(batch, step="predict")
- n_windows = len(windows["temporal"])
- y_idx = batch["y_idx"]
-
- # Number of windows in batch
- windows_batch_size = self.inference_windows_batch_size
- if windows_batch_size < 0:
- windows_batch_size = n_windows
- n_batches = int(np.ceil(n_windows / windows_batch_size))
-
- y_hats = []
- for i in range(n_batches):
- # Create and normalize windows [Ws, L+H, C]
- w_idxs = np.arange(
- i * windows_batch_size, min((i + 1) * windows_batch_size, n_windows)
- )
- windows = self._create_windows(batch, step="predict", w_idxs=w_idxs)
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- insample_y, insample_mask, _, _, hist_exog, futr_exog, stat_exog = (
- self._parse_windows(batch, windows)
- )
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L]
- insample_mask=insample_mask, # [Ws, L]
- futr_exog=futr_exog, # [Ws, L + h, F]
- hist_exog=hist_exog, # [Ws, L, X]
- stat_exog=stat_exog,
- ) # [Ws, S]
-
- # Model Predictions
- output_batch = self(windows_batch)
- # Inverse normalization and sampling
- if self.loss.is_distribution_output:
- _, y_loc, y_scale = self._inv_normalization(
- y_hat=torch.empty(
- size=(insample_y.shape[0], self.h),
- dtype=output_batch[0].dtype,
- device=output_batch[0].device,
- ),
- temporal_cols=batch["temporal_cols"],
- y_idx=y_idx,
- )
- distr_args = self.loss.scale_decouple(
- output=output_batch, loc=y_loc, scale=y_scale
- )
- _, sample_mean, quants = self.loss.sample(distr_args=distr_args)
- y_hat = torch.concat((sample_mean, quants), axis=2)
-
- if self.loss.return_params:
- distr_args = torch.stack(distr_args, dim=-1)
- distr_args = torch.reshape(
- distr_args, (len(windows["temporal"]), self.h, -1)
- )
- y_hat = torch.concat((y_hat, distr_args), axis=2)
- else:
- y_hat, _, _ = self._inv_normalization(
- y_hat=output_batch,
- temporal_cols=batch["temporal_cols"],
- y_idx=y_idx,
- )
- y_hats.append(y_hat)
- y_hat = torch.cat(y_hats, dim=0)
- return y_hat
-
- def fit(
- self,
- dataset,
- val_size=0,
- test_size=0,
- random_seed=None,
- distributed_config=None,
- ):
- """Fit.
-
- The `fit` method, optimizes the neural network's weights using the
- initialization parameters (`learning_rate`, `windows_batch_size`, ...)
- and the `loss` function as defined during the initialization.
- Within `fit` we use a PyTorch Lightning `Trainer` that
- inherits the initialization's `self.trainer_kwargs`, to customize
- its inputs, see [PL's trainer arguments](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
-
- The method is designed to be compatible with SKLearn-like classes
- and in particular to be compatible with the StatsForecast library.
-
- By default the `model` is not saving training checkpoints to protect
- disk memory, to get them change `enable_checkpointing=True` in `__init__`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `val_size`: int, validation size for temporal cross-validation.
- `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
- `test_size`: int, test size for temporal cross-validation.
- """
- return self._fit(
- dataset=dataset,
- batch_size=self.batch_size,
- valid_batch_size=self.valid_batch_size,
- val_size=val_size,
- test_size=test_size,
- random_seed=random_seed,
- distributed_config=distributed_config,
- )
-
- def predict(
- self,
- dataset,
- test_size=None,
- step_size=1,
- random_seed=None,
- **data_module_kwargs,
- ):
- """Predict.
-
- Neural network prediction with PL's `Trainer` execution of `predict_step`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `test_size`: int=None, test size for temporal cross-validation.
- `step_size`: int=1, Step size between each window.
- `random_seed`: int=None, random_seed for pytorch initializer and numpy generators, overwrites model.__init__'s.
- `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).
- """
- self._check_exog(dataset)
- self._restart_seed(random_seed)
- data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)
-
- self.predict_step_size = step_size
- self.decompose_forecast = False
- datamodule = TimeSeriesDataModule(
- dataset=dataset,
- valid_batch_size=self.valid_batch_size,
- **data_module_kwargs,
- )
-
- # Protect when case of multiple gpu. PL does not support return preds with multiple gpu.
- pred_trainer_kwargs = self.trainer_kwargs.copy()
- if (pred_trainer_kwargs.get("accelerator", None) == "gpu") and (
- torch.cuda.device_count() > 1
- ):
- pred_trainer_kwargs["devices"] = [0]
-
- trainer = pl.Trainer(**pred_trainer_kwargs)
- fcsts = trainer.predict(self, datamodule=datamodule)
- fcsts = torch.vstack(fcsts).numpy().flatten()
- fcsts = fcsts.reshape(-1, len(self.loss.output_names))
- return fcsts
-
- def decompose(self, dataset, step_size=1, random_seed=None, **data_module_kwargs):
- """Decompose Predictions.
-
- Decompose the predictions through the network's layers.
- Available methods are `ESRNN`, `NHITS`, `NBEATS`, and `NBEATSx`.
-
- **Parameters:**
- `dataset`: NeuralForecast's `TimeSeriesDataset`, see [documentation here](https://nixtla.github.io/neuralforecast/tsdataset.html).
- `step_size`: int=1, step size between each window of temporal data.
- `**data_module_kwargs`: PL's TimeSeriesDataModule args, see [documentation](https://pytorch-lightning.readthedocs.io/en/1.6.1/extensions/datamodules.html#using-a-datamodule).
- """
- # Restart random seed
- if random_seed is None:
- random_seed = self.random_seed
- torch.manual_seed(random_seed)
- data_module_kwargs = self._set_quantile_for_iqloss(**data_module_kwargs)
-
- self.predict_step_size = step_size
- self.decompose_forecast = True
- datamodule = TimeSeriesDataModule(
- dataset=dataset,
- valid_batch_size=self.valid_batch_size,
- **data_module_kwargs,
- )
- trainer = pl.Trainer(**self.trainer_kwargs)
- fcsts = trainer.predict(self, datamodule=datamodule)
- self.decompose_forecast = False # Default decomposition back to false
- return torch.vstack(fcsts).numpy()
diff --git a/neuralforecast/common/_model_checks.py b/neuralforecast/common/_model_checks.py
new file mode 100644
index 000000000..ab387c0ff
--- /dev/null
+++ b/neuralforecast/common/_model_checks.py
@@ -0,0 +1,224 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/common.model_checks.ipynb.
+
+# %% auto 0
+__all__ = ['seed', 'test_size', 'FREQ', 'N_SERIES_1', 'df', 'max_ds', 'Y_TRAIN_DF_1', 'Y_TEST_DF_1', 'N_SERIES_2', 'Y_TRAIN_DF_2',
+ 'Y_TEST_DF_2', 'N_SERIES_3', 'STATIC_3', 'Y_TRAIN_DF_3', 'Y_TEST_DF_3', 'N_SERIES_4', 'STATIC_4',
+ 'Y_TRAIN_DF_4', 'Y_TEST_DF_4', 'check_loss_functions', 'check_airpassengers', 'check_model']
+
+# %% ../../nbs/common.model_checks.ipynb 4
+import pandas as pd
+import neuralforecast.losses.pytorch as losses
+
+from .. import NeuralForecast
+from neuralforecast.utils import (
+ AirPassengersPanel,
+ AirPassengersStatic,
+ generate_series,
+)
+
+# %% ../../nbs/common.model_checks.ipynb 5
+seed = 0
+test_size = 14
+FREQ = "D"
+
+# 1 series, no exogenous
+N_SERIES_1 = 1
+df = generate_series(n_series=N_SERIES_1, seed=seed, freq=FREQ, equal_ends=True)
+max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)
+Y_TRAIN_DF_1 = df[df.ds < max_ds]
+Y_TEST_DF_1 = df[df.ds >= max_ds]
+
+# 5 series, no exogenous
+N_SERIES_2 = 5
+df = generate_series(n_series=N_SERIES_2, seed=seed, freq=FREQ, equal_ends=True)
+max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)
+Y_TRAIN_DF_2 = df[df.ds < max_ds]
+Y_TEST_DF_2 = df[df.ds >= max_ds]
+
+# 1 series, with static and temporal exogenous
+N_SERIES_3 = 1
+df, STATIC_3 = generate_series(
+ n_series=N_SERIES_3,
+ n_static_features=2,
+ n_temporal_features=2,
+ seed=seed,
+ freq=FREQ,
+ equal_ends=True,
+)
+max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)
+Y_TRAIN_DF_3 = df[df.ds < max_ds]
+Y_TEST_DF_3 = df[df.ds >= max_ds]
+
+# 5 series, with static and temporal exogenous
+N_SERIES_4 = 5
+df, STATIC_4 = generate_series(
+ n_series=N_SERIES_4,
+ n_static_features=2,
+ n_temporal_features=2,
+ seed=seed,
+ freq=FREQ,
+ equal_ends=True,
+)
+max_ds = df.ds.max() - pd.Timedelta(test_size, FREQ)
+Y_TRAIN_DF_4 = df[df.ds < max_ds]
+Y_TEST_DF_4 = df[df.ds >= max_ds]
+
+
+# Generic test for a given config for a model
+def _run_model_tests(model_class, config):
+ if model_class.RECURRENT:
+ config["inference_input_size"] = config["input_size"]
+
+ # DF_1
+ if model_class.MULTIVARIATE:
+ config["n_series"] = N_SERIES_1
+ if isinstance(config["loss"], losses.relMSE):
+ config["loss"].y_train = Y_TRAIN_DF_1["y"].values
+ if isinstance(config["valid_loss"], losses.relMSE):
+ config["valid_loss"].y_train = Y_TRAIN_DF_1["y"].values
+
+ model = model_class(**config)
+ fcst = NeuralForecast(models=[model], freq=FREQ)
+ fcst.fit(df=Y_TRAIN_DF_1, val_size=24)
+ _ = fcst.predict(futr_df=Y_TEST_DF_1)
+ # DF_2
+ if model_class.MULTIVARIATE:
+ config["n_series"] = N_SERIES_2
+ if isinstance(config["loss"], losses.relMSE):
+ config["loss"].y_train = Y_TRAIN_DF_2["y"].values
+ if isinstance(config["valid_loss"], losses.relMSE):
+ config["valid_loss"].y_train = Y_TRAIN_DF_2["y"].values
+ model = model_class(**config)
+ fcst = NeuralForecast(models=[model], freq=FREQ)
+ fcst.fit(df=Y_TRAIN_DF_2, val_size=24)
+ _ = fcst.predict(futr_df=Y_TEST_DF_2)
+
+ if model.EXOGENOUS_STAT and model.EXOGENOUS_FUTR:
+ # DF_3
+ if model_class.MULTIVARIATE:
+ config["n_series"] = N_SERIES_3
+ if isinstance(config["loss"], losses.relMSE):
+ config["loss"].y_train = Y_TRAIN_DF_3["y"].values
+ if isinstance(config["valid_loss"], losses.relMSE):
+ config["valid_loss"].y_train = Y_TRAIN_DF_3["y"].values
+ model = model_class(**config)
+ fcst = NeuralForecast(models=[model], freq=FREQ)
+ fcst.fit(df=Y_TRAIN_DF_3, static_df=STATIC_3, val_size=24)
+ _ = fcst.predict(futr_df=Y_TEST_DF_3)
+
+ # DF_4
+ if model_class.MULTIVARIATE:
+ config["n_series"] = N_SERIES_4
+ if isinstance(config["loss"], losses.relMSE):
+ config["loss"].y_train = Y_TRAIN_DF_4["y"].values
+ if isinstance(config["valid_loss"], losses.relMSE):
+ config["valid_loss"].y_train = Y_TRAIN_DF_4["y"].values
+ model = model_class(**config)
+ fcst = NeuralForecast(models=[model], freq=FREQ)
+ fcst.fit(df=Y_TRAIN_DF_4, static_df=STATIC_4, val_size=24)
+ _ = fcst.predict(futr_df=Y_TEST_DF_4)
+
+
+# Tests a model against every loss function
+def check_loss_functions(model_class):
+ loss_list = [
+ losses.MAE(),
+ losses.MSE(),
+ losses.RMSE(),
+ losses.MAPE(),
+ losses.SMAPE(),
+ losses.MASE(seasonality=7),
+ losses.QuantileLoss(q=0.5),
+ losses.MQLoss(),
+ losses.IQLoss(),
+ losses.DistributionLoss("Normal"),
+ losses.DistributionLoss("StudentT"),
+ losses.DistributionLoss("Poisson"),
+ losses.DistributionLoss("NegativeBinomial"),
+ losses.DistributionLoss("Tweedie", rho=1.5),
+ losses.DistributionLoss("ISQF"),
+ losses.PMM(),
+ losses.PMM(weighted=True),
+ losses.GMM(),
+ losses.GMM(weighted=True),
+ losses.NBMM(),
+ losses.NBMM(weighted=True),
+ losses.HuberLoss(),
+ losses.TukeyLoss(),
+ losses.HuberQLoss(q=0.5),
+ losses.HuberMQLoss(),
+ ]
+ for loss in loss_list:
+ test_name = f"{model_class.__name__}: checking {loss._get_name()}"
+ print(f"{test_name}")
+ config = {
+ "max_steps": 2,
+ "h": 7,
+ "input_size": 28,
+ "loss": loss,
+ "valid_loss": None,
+ "enable_progress_bar": False,
+ "enable_model_summary": False,
+ "val_check_steps": 2,
+ }
+ try:
+ _run_model_tests(model_class, config)
+ except RuntimeError:
+ raise Exception(f"{test_name} failed.")
+ except Exception:
+ print(f"{test_name} skipped on raised Exception.")
+ pass
+
+
+# Tests a model against the AirPassengers dataset
+def check_airpassengers(model_class):
+ print(f"{model_class.__name__}: checking forecast AirPassengers dataset")
+ Y_train_df = AirPassengersPanel[
+ AirPassengersPanel.ds < AirPassengersPanel["ds"].values[-12]
+ ] # 132 train
+ Y_test_df = AirPassengersPanel[
+ AirPassengersPanel.ds >= AirPassengersPanel["ds"].values[-12]
+ ].reset_index(
+ drop=True
+ ) # 12 test
+
+ config = {
+ "max_steps": 2,
+ "h": 12,
+ "input_size": 24,
+ "enable_progress_bar": False,
+ "enable_model_summary": False,
+ "val_check_steps": 2,
+ }
+
+ if model_class.MULTIVARIATE:
+ config["n_series"] = Y_train_df["unique_id"].nunique()
+ # Normal forecast
+ fcst = NeuralForecast(models=[model_class(**config)], freq="M")
+ fcst.fit(df=Y_train_df, static_df=AirPassengersStatic)
+ _ = fcst.predict(futr_df=Y_test_df)
+
+ # Cross-validation
+ fcst = NeuralForecast(models=[model_class(**config)], freq="M")
+ _ = fcst.cross_validation(
+ df=AirPassengersPanel, static_df=AirPassengersStatic, n_windows=2, step_size=12
+ )
+
+
+# Add unit test functions to this function
+def check_model(model_class, checks=["losses", "airpassengers"]):
+ """
+ Check model with various tests. Options for checks are:
+ "losses": test the model against all loss functions
+ "airpassengers": test the model against the airpassengers dataset for forecasting and cross-validation
+
+ """
+ if "losses" in checks:
+ check_loss_functions(model_class)
+ if "airpassengers" in checks:
+ try:
+ check_airpassengers(model_class)
+ except RuntimeError:
+ raise Exception(
+ f"{model_class.__name__}: AirPassengers forecast test failed."
+ )
diff --git a/neuralforecast/common/_modules.py b/neuralforecast/common/_modules.py
index d50228b87..852968bd0 100644
--- a/neuralforecast/common/_modules.py
+++ b/neuralforecast/common/_modules.py
@@ -4,7 +4,7 @@
__all__ = ['ACTIVATIONS', 'MLP', 'Chomp1d', 'CausalConv1d', 'TemporalConvolutionEncoder', 'TransEncoderLayer', 'TransEncoder',
'TransDecoderLayer', 'TransDecoder', 'AttentionLayer', 'PositionalEmbedding', 'TokenEmbedding',
'TimeFeatureEmbedding', 'FixedEmbedding', 'TemporalEmbedding', 'DataEmbedding', 'MovingAvg', 'SeriesDecomp',
- 'RevIN']
+ 'RevIN', 'RevINMultivariate']
# %% ../../nbs/common.modules.ipynb 3
import math
@@ -601,3 +601,66 @@ def _denormalize(self, x):
else:
x = x + self.mean
return x
+
+# %% ../../nbs/common.modules.ipynb 21
+class RevINMultivariate(nn.Module):
+ """
+ ReversibleInstanceNorm1d for Multivariate models
+ """
+
+ def __init__(
+ self,
+ num_features: int,
+ eps=1e-5,
+ affine=False,
+ subtract_last=False,
+ non_norm=False,
+ ):
+ super().__init__()
+ self.num_features = num_features
+ self.eps = eps
+ self.affine = affine
+ if self.affine:
+ self._init_params()
+
+ def forward(self, x, mode: str):
+ if mode == "norm":
+ x = self._normalize(x)
+ elif mode == "denorm":
+ x = self._denormalize(x)
+ else:
+ raise NotImplementedError
+ return x
+
+ def _init_params(self):
+ # initialize RevIN params: (C,)
+ self.affine_weight = nn.Parameter(torch.ones((1, 1, self.num_features)))
+ self.affine_bias = nn.Parameter(torch.zeros((1, 1, self.num_features)))
+
+ def _normalize(self, x):
+ # Batch statistics
+ self.batch_mean = torch.mean(x, axis=1, keepdim=True).detach()
+ self.batch_std = torch.sqrt(
+ torch.var(x, axis=1, keepdim=True, unbiased=False) + self.eps
+ ).detach()
+
+ # Instance normalization
+ x = x - self.batch_mean
+ x = x / self.batch_std
+
+ if self.affine:
+ x = x * self.affine_weight
+ x = x + self.affine_bias
+
+ return x
+
+ def _denormalize(self, x):
+ # Reverse the normalization
+ if self.affine:
+ x = x - self.affine_bias
+ x = x / self.affine_weight
+
+ x = x * self.batch_std
+ x = x + self.batch_mean
+
+ return x
diff --git a/neuralforecast/common/_scalers.py b/neuralforecast/common/_scalers.py
index c45b58d62..f11187d21 100644
--- a/neuralforecast/common/_scalers.py
+++ b/neuralforecast/common/_scalers.py
@@ -402,11 +402,11 @@ def __init__(self, scaler_type="robust", dim=-1, eps=1e-6, num_features=None):
def _init_params(self, num_features):
# Initialize RevIN scaler params to broadcast:
if self.dim == 1: # [B,T,C] [1,1,C]
- self.revin_bias = nn.Parameter(torch.zeros(1, 1, num_features))
- self.revin_weight = nn.Parameter(torch.ones(1, 1, num_features))
+ self.revin_bias = nn.Parameter(torch.zeros(1, 1, num_features, 1))
+ self.revin_weight = nn.Parameter(torch.ones(1, 1, num_features, 1))
elif self.dim == -1: # [B,C,T] [1,C,1]
- self.revin_bias = nn.Parameter(torch.zeros(1, num_features, 1))
- self.revin_weight = nn.Parameter(torch.ones(1, num_features, 1))
+ self.revin_bias = nn.Parameter(torch.zeros(1, num_features, 1, 1))
+ self.revin_weight = nn.Parameter(torch.ones(1, num_features, 1, 1))
# @torch.no_grad()
def transform(self, x, mask):
diff --git a/neuralforecast/core.py b/neuralforecast/core.py
index fffe4bde5..d53267ba7 100644
--- a/neuralforecast/core.py
+++ b/neuralforecast/core.py
@@ -29,6 +29,7 @@
from .common._base_model import DistributedConfig
from .compat import SparkDataFrame
+from .losses.pytorch import IQLoss
from neuralforecast.tsdataset import (
_FilesDataset,
TimeSeriesDataset,
@@ -69,7 +70,12 @@
RMoK,
)
from .common._base_auto import BaseAuto, MockTrial
-from .utils import PredictionIntervals, get_prediction_interval_method
+from neuralforecast.utils import (
+ PredictionIntervals,
+ get_prediction_interval_method,
+ level_to_quantiles,
+ quantiles_to_level,
+)
# %% ../nbs/core.ipynb 5
# this disables warnings about the number of workers in the dataloaders
@@ -264,6 +270,7 @@ def __init__(
# Flags and attributes
self._fitted = False
self._reset_models()
+ self._add_level = False
def _scalers_fit_transform(self, dataset: TimeSeriesDataset) -> None:
self.scalers_ = {}
@@ -681,13 +688,16 @@ def _get_model_names(self, add_level=False) -> List[str]:
names: List[str] = []
count_names = {"model": 0}
for model in self.models:
- if add_level and model.loss.outputsize_multiplier > 1:
- continue
-
model_name = repr(model)
count_names[model_name] = count_names.get(model_name, -1) + 1
if count_names[model_name] > 0:
model_name += str(count_names[model_name])
+
+ if add_level and (
+ model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss)
+ ):
+ continue
+
names.extend(model_name + n for n in model.loss.output_names)
return names
@@ -815,6 +825,7 @@ def predict(
verbose: bool = False,
engine=None,
level: Optional[List[Union[int, float]]] = None,
+ quantiles: Optional[List[float]] = None,
**data_kwargs,
):
"""Predict with core.NeuralForecast.
@@ -838,6 +849,8 @@ def predict(
Distributed engine for inference. Only used if df is a spark dataframe or if fit was called on a spark dataframe.
level : list of ints or floats, optional (default=None)
Confidence levels between 0 and 100.
+ quantiles : list of floats, optional (default=None)
+ Alternative to level, target quantiles to predict.
data_kwargs : kwargs
Extra arguments to be passed to the dataset within each model.
@@ -853,6 +866,22 @@ def predict(
if not self._fitted:
raise Exception("You must fit the model before predicting.")
+ quantiles_ = None
+ level_ = None
+ has_level = False
+ if level is not None:
+ has_level = True
+ if quantiles is not None:
+ raise ValueError("You can't set both level and quantiles.")
+ level_ = sorted(list(set(level)))
+ quantiles_ = level_to_quantiles(level_)
+
+ if quantiles is not None:
+ if level is not None:
+ raise ValueError("You can't set both level and quantiles.")
+ quantiles_ = sorted(list(set(quantiles)))
+ level_ = quantiles_to_level(quantiles_)
+
needed_futr_exog = self._get_needed_futr_exog()
if needed_futr_exog:
if futr_df is None:
@@ -905,8 +934,6 @@ def predict(
if verbose:
print("Using stored dataset.")
- cols = self._get_model_names()
-
# Placeholder dataframe for predictions with unique_id and ds
fcsts_df = ufp.make_future_dataframe(
uids=uids,
@@ -949,27 +976,20 @@ def predict(
self._scalers_transform(futr_dataset)
dataset = dataset.append(futr_dataset)
- col_idx = 0
- fcsts = np.full(
- (self.h * len(uids), len(cols)), fill_value=np.nan, dtype=np.float32
+ fcsts, cols = self._generate_forecasts(
+ dataset=dataset,
+ uids=uids,
+ quantiles_=quantiles_,
+ level_=level_,
+ has_level=has_level,
+ **data_kwargs,
)
- for model in self.models:
- old_test_size = model.get_test_size()
- model.set_test_size(self.h) # To predict h steps ahead
- model_fcsts = model.predict(dataset=dataset, **data_kwargs)
- # Append predictions in memory placeholder
- output_length = len(model.loss.output_names)
- fcsts[:, col_idx : col_idx + output_length] = model_fcsts
- col_idx += output_length
- model.set_test_size(old_test_size) # Set back to original value
+
if self.scalers_:
indptr = np.append(0, np.full(len(uids), self.h).cumsum())
fcsts = self._scalers_target_inverse_transform(fcsts, indptr)
# Declare predictions pd.DataFrame
- cols = (
- self._get_model_names()
- ) # Needed for IQLoss as column names may have changed during the call to .predict()
if isinstance(fcsts_df, pl_DataFrame):
fcsts = pl_DataFrame(dict(zip(cols, fcsts.T)))
else:
@@ -979,29 +999,6 @@ def predict(
_warn_id_as_idx()
fcsts_df = fcsts_df.set_index(self.id_col)
- # add prediction intervals
- if level is not None:
- if self._cs_df is None or self.prediction_intervals is None:
- raise Exception(
- "You must fit the model with prediction_intervals to use level."
- )
- else:
- level_ = sorted(level)
- model_names = self._get_model_names(add_level=True)
- prediction_interval_method = get_prediction_interval_method(
- self.prediction_intervals.method
- )
-
- fcsts_df = prediction_interval_method(
- fcsts_df,
- self._cs_df,
- model_names=list(model_names),
- level=level_,
- cs_n_windows=self.prediction_intervals.n_windows,
- n_series=len(uids),
- horizon=self.h,
- )
-
return fcsts_df
def _reset_models(self):
@@ -1050,15 +1047,6 @@ def _no_refit_cross_validation(
"Validation and test sets are larger than the shorter time-series."
)
- cols = []
- count_names = {"model": 0}
- for model in self.models:
- model_name = repr(model)
- count_names[model_name] = count_names.get(model_name, -1) + 1
- if count_names[model_name] > 0:
- model_name += str(count_names[model_name])
- cols += [model_name + n for n in model.loss.output_names]
-
fcsts_df = ufp.cv_times(
times=self.ds,
uids=self.uids,
@@ -1072,23 +1060,22 @@ def _no_refit_cross_validation(
# the cv_times is sorted by window and then id
fcsts_df = ufp.sort(fcsts_df, [id_col, "cutoff", time_col])
- col_idx = 0
- fcsts = np.full(
- (self.dataset.n_groups * self.h * n_windows, len(cols)),
- np.nan,
- dtype=np.float32,
- )
-
+ fcsts_list: List = []
for model in self.models:
+ if self._add_level and (
+ model.loss.outputsize_multiplier > 1 or isinstance(model.loss, IQLoss)
+ ):
+ continue
+
model.fit(dataset=self.dataset, val_size=val_size, test_size=test_size)
model_fcsts = model.predict(
self.dataset, step_size=step_size, **data_kwargs
)
# Append predictions in memory placeholder
- output_length = len(model.loss.output_names)
- fcsts[:, col_idx : (col_idx + output_length)] = model_fcsts
- col_idx += output_length
+ fcsts_list.append(model_fcsts)
+
+ fcsts = np.concatenate(fcsts_list, axis=-1)
# we may have allocated more space than needed
# each serie can produce at most (serie.size - 1) // self.h CV windows
effective_sizes = ufp.counts_by_id(fcsts_df, id_col)["counts"].to_numpy()
@@ -1116,6 +1103,7 @@ def _no_refit_cross_validation(
self._fitted = True
# Add predictions to forecasts DataFrame
+ cols = self._get_model_names(add_level=self._add_level)
if isinstance(self.uids, pl_Series):
fcsts = pl_DataFrame(dict(zip(cols, fcsts.T)))
else:
@@ -1151,6 +1139,7 @@ def cross_validation(
target_col: str = "y",
prediction_intervals: Optional[PredictionIntervals] = None,
level: Optional[List[Union[int, float]]] = None,
+ quantiles: Optional[List[float]] = None,
**data_kwargs,
) -> DataFrame:
"""Temporal Cross-Validation with core.NeuralForecast.
@@ -1192,7 +1181,9 @@ def cross_validation(
prediction_intervals : PredictionIntervals, optional (default=None)
Configuration to calibrate prediction intervals (Conformal Prediction).
level : list of ints or floats, optional (default=None)
- Confidence levels between 0 and 100. Use with prediction_intervals.
+ Confidence levels between 0 and 100.
+ quantiles : list of floats, optional (default=None)
+ Alternative to level, target quantiles to predict.
data_kwargs : kwargs
Extra arguments to be passed to the dataset within each model.
@@ -1225,17 +1216,19 @@ def cross_validation(
df = df.reset_index(id_col)
# Checks for prediction intervals
- if prediction_intervals is not None or level is not None:
- if level is None:
- warnings.warn("Level not provided, using level=[90].")
- level = [90]
- if prediction_intervals is None:
- raise Exception("You must set prediction_intervals to use level.")
+ if prediction_intervals is not None:
+ if level is None and quantiles is None:
+ raise Exception(
+ "When passing prediction_intervals you need to set the level or quantiles argument."
+ )
if not refit:
raise Exception(
- "Passing prediction_intervals and/or level is only supported with refit=True."
+ "Passing prediction_intervals is only supported with refit=True."
)
+ if level is not None and quantiles is not None:
+ raise ValueError("You can't set both level and quantiles argument.")
+
if not refit:
return self._no_refit_cross_validation(
@@ -1296,6 +1289,7 @@ def cross_validation(
sort_df=sort_df,
verbose=verbose,
level=level,
+ quantiles=quantiles,
**data_kwargs,
)
preds = ufp.join(preds, cutoffs, on=id_col, how="left")
@@ -1317,7 +1311,7 @@ def cross_validation(
out = out.set_index(id_col)
return out
- def predict_insample(self, step_size: int = 1):
+ def predict_insample(self, step_size: int = 1, **data_kwargs):
"""Predict insample with core.NeuralForecast.
`core.NeuralForecast`'s `predict_insample` uses stored fitted `models`
@@ -1338,26 +1332,6 @@ def predict_insample(self, step_size: int = 1):
"The models must be fitted first with `fit` or `cross_validation`."
)
- for model in self.models:
- if model.SAMPLING_TYPE == "recurrent":
- warnings.warn(
- f"Predict insample might not provide accurate predictions for \
- recurrent model {repr(model)} class yet due to scaling."
- )
- print(
- f"WARNING: Predict insample might not provide accurate predictions for \
- recurrent model {repr(model)} class yet due to scaling."
- )
-
- cols = []
- count_names = {"model": 0}
- for model in self.models:
- model_name = repr(model)
- count_names[model_name] = count_names.get(model_name, -1) + 1
- if count_names[model_name] > 0:
- model_name += str(count_names[model_name])
- cols += [model_name + n for n in model.loss.output_names]
-
# Remove test set from dataset and last dates
test_size = self.models[0].get_test_size()
@@ -1396,9 +1370,7 @@ def predict_insample(self, step_size: int = 1):
time_col=self.time_col,
)
- col_idx = 0
- fcsts = np.full((len(fcsts_df), len(cols)), np.nan, dtype=np.float32)
-
+ fcsts_list: List = []
for model in self.models:
# Test size is the number of periods to forecast (full size of trimmed dataset)
model.set_test_size(test_size=trimmed_dataset.max_size)
@@ -1406,10 +1378,9 @@ def predict_insample(self, step_size: int = 1):
# Predict
model_fcsts = model.predict(trimmed_dataset, step_size=step_size)
# Append predictions in memory placeholder
- output_length = len(model.loss.output_names)
- fcsts[:, col_idx : (col_idx + output_length)] = model_fcsts
- col_idx += output_length
+ fcsts_list.append(model_fcsts)
model.set_test_size(test_size=test_size) # Set original test_size
+ fcsts = np.concatenate(fcsts_list, axis=-1)
# original y
original_y = {
@@ -1419,6 +1390,7 @@ def predict_insample(self, step_size: int = 1):
}
# Add predictions to forecasts DataFrame
+ cols = self._get_model_names()
if isinstance(self.uids, pl_Series):
fcsts = pl_DataFrame(dict(zip(cols, fcsts.T)))
Y_df = pl_DataFrame(original_y)
@@ -1698,6 +1670,7 @@ def _conformity_scores(
"Please reduce the number of windows, horizon or remove those series."
)
+ self._add_level = True
cv_results = self.cross_validation(
df=df,
static_df=static_df,
@@ -1706,6 +1679,7 @@ def _conformity_scores(
time_col=time_col,
target_col=target_col,
)
+ self._add_level = False
kept = [time_col, id_col, "cutoff"]
# conformity score for each model
@@ -1717,3 +1691,126 @@ def _conformity_scores(
cv_results = ufp.assign_columns(cv_results, model, abs_err)
dropped = list(set(cv_results.columns) - set(kept))
return ufp.drop_columns(cv_results, dropped)
+
+ def _generate_forecasts(
+ self,
+ dataset: TimeSeriesDataset,
+ uids: Series,
+ quantiles_: Optional[List[float]] = None,
+ level_: Optional[List[Union[int, float]]] = None,
+ has_level: Optional[bool] = False,
+ **data_kwargs,
+ ) -> np.array:
+ fcsts_list: List = []
+ cols = []
+ count_names = {"model": 0}
+ for model in self.models:
+ old_test_size = model.get_test_size()
+ model.set_test_size(self.h) # To predict h steps ahead
+
+ # Increment model name if the same model is used more than once
+ model_name = repr(model)
+ count_names[model_name] = count_names.get(model_name, -1) + 1
+ if count_names[model_name] > 0:
+ model_name += str(count_names[model_name])
+
+ # Predict for every quantile or level if requested and the loss function supports it
+ # case 1: DistributionLoss and MixtureLosses
+ if (
+ quantiles_ is not None
+ and not isinstance(model.loss, IQLoss)
+ and hasattr(model.loss, "update_quantile")
+ and callable(model.loss.update_quantile)
+ ):
+ model_fcsts = model.predict(
+ dataset=dataset, quantiles=quantiles_, **data_kwargs
+ )
+ fcsts_list.append(model_fcsts)
+ col_names = []
+ for i, quantile in enumerate(quantiles_):
+ col_name = self._get_column_name(model_name, quantile, has_level)
+ if i == 0:
+ col_names.extend([f"{model_name}", col_name])
+ else:
+ col_names.extend([col_name])
+ if hasattr(model.loss, "return_params") and model.loss.return_params:
+ cols.extend(
+ col_names
+ + [
+ model_name + param_name
+ for param_name in model.loss.param_names
+ ]
+ )
+ else:
+ cols.extend(col_names)
+ # case 2: IQLoss
+ elif quantiles_ is not None and isinstance(model.loss, IQLoss):
+ # IQLoss does not give monotonically increasing quantiles, so we apply a hack: compute all quantiles, and take the quantile over the quantiles
+ quantiles_iqloss = np.linspace(0.01, 0.99, 20)
+ fcsts_list_iqloss = []
+ for i, quantile in enumerate(quantiles_iqloss):
+ model_fcsts = model.predict(
+ dataset=dataset, quantiles=[quantile], **data_kwargs
+ )
+ fcsts_list_iqloss.append(model_fcsts)
+ fcsts_iqloss = np.concatenate(fcsts_list_iqloss, axis=-1)
+
+ # Get the actual requested quantiles
+ model_fcsts = np.quantile(fcsts_iqloss, quantiles_, axis=-1).T
+ fcsts_list.append(model_fcsts)
+
+ # Get the right column names
+ col_names = []
+ for i, quantile in enumerate(quantiles_):
+ col_name = self._get_column_name(model_name, quantile, has_level)
+ col_names.extend([col_name])
+ cols.extend(col_names)
+ # case 3: PointLoss via prediction intervals
+ elif quantiles_ is not None and model.loss.outputsize_multiplier == 1:
+ if self.prediction_intervals is None:
+ raise AttributeError(
+ f"You have trained {model_name} with loss={type(model.loss).__name__}(). \n"
+ " You then must set `prediction_intervals` during fit to use level or quantiles during predict."
+ )
+ model_fcsts = model.predict(
+ dataset=dataset, quantiles=quantiles_, **data_kwargs
+ )
+ prediction_interval_method = get_prediction_interval_method(
+ self.prediction_intervals.method
+ )
+ fcsts_with_intervals, out_cols = prediction_interval_method(
+ model_fcsts,
+ self._cs_df,
+ model=model_name,
+ level=level_ if has_level else None,
+ cs_n_windows=self.prediction_intervals.n_windows,
+ n_series=len(uids),
+ horizon=self.h,
+ quantiles=quantiles_ if not has_level else None,
+ )
+ fcsts_list.append(fcsts_with_intervals)
+ cols.extend([model_name] + out_cols)
+ # base case: quantiles or levels are not supported or provided as arguments
+ else:
+ model_fcsts = model.predict(dataset=dataset, **data_kwargs)
+ fcsts_list.append(model_fcsts)
+ cols.extend(model_name + n for n in model.loss.output_names)
+ model.set_test_size(old_test_size) # Set back to original value
+ fcsts = np.concatenate(fcsts_list, axis=-1)
+
+ return fcsts, cols
+
+ @staticmethod
+ def _get_column_name(model_name, quantile, has_level) -> str:
+ if not has_level:
+ col_name = f"{model_name}_ql{quantile}"
+ elif quantile < 0.5:
+ level_lo = int(round(100 - 200 * quantile))
+ col_name = f"{model_name}-lo-{level_lo}"
+ elif quantile > 0.5:
+ level_hi = int(round(100 - 200 * (1 - quantile)))
+ col_name = f"{model_name}-hi-{level_hi}"
+ else:
+ col_name = f"{model_name}-median"
+
+ return col_name
diff --git a/neuralforecast/losses/pytorch.py b/neuralforecast/losses/pytorch.py
index a713b5b31..6e6e98e8c 100644
--- a/neuralforecast/losses/pytorch.py
+++ b/neuralforecast/losses/pytorch.py
@@ -6,9 +6,8 @@
'Accuracy', 'sCRPS']
# %% ../../nbs/losses.pytorch.ipynb 4
-from typing import Optional, Union, Tuple
+from typing import Optional, Union, Tuple, List
-import math
import numpy as np
import torch
@@ -22,6 +21,9 @@
Poisson,
NegativeBinomial,
Beta,
+ Gamma,
+ MixtureSameFamily,
+ Categorical,
AffineTransform,
TransformedDistribution,
)
@@ -55,7 +57,9 @@ class BasePointLoss(torch.nn.Module):
`output_names`: Names of the outputs.
"""
- def __init__(self, horizon_weight, outputsize_multiplier, output_names):
+ def __init__(
+ self, horizon_weight=None, outputsize_multiplier=None, output_names=None
+ ):
super(BasePointLoss, self).__init__()
if horizon_weight is not None:
horizon_weight = torch.Tensor(horizon_weight.flatten())
@@ -66,10 +70,13 @@ def __init__(self, horizon_weight, outputsize_multiplier, output_names):
def domain_map(self, y_hat: torch.Tensor):
"""
- Univariate loss operates in dimension [B,T,H]/[B,H]
- This changes the network's output from [B,H,1]->[B,H]
+ Input:
+ Univariate: [B, H, 1]
+ Multivariate: [B, H, N]
+
+ Output: [B, H, N]
"""
- return y_hat.squeeze(-1)
+ return y_hat
def _compute_weights(self, y, mask):
"""
@@ -78,17 +85,18 @@ def _compute_weights(self, y, mask):
If set, check that it has the same length as the horizon in x.
"""
if mask is None:
- mask = torch.ones_like(y, device=y.device)
+ mask = torch.ones_like(y)
if self.horizon_weight is None:
- self.horizon_weight = torch.ones(mask.shape[-1])
+ weights = torch.ones_like(mask)
else:
- assert mask.shape[-1] == len(
+ assert mask.shape[1] == len(
self.horizon_weight
), "horizon_weight must have same length as Y"
+ weights = self.horizon_weight.clone()
+ weights = weights[None, :, None].to(mask.device)
+ weights = torch.ones_like(mask, device=mask.device) * weights
- weights = self.horizon_weight.clone()
- weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)
return weights * mask
# %% ../../nbs/losses.pytorch.ipynb 11
@@ -118,7 +126,8 @@ def __call__(
y: torch.Tensor,
y_hat: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ y_insample: Union[torch.Tensor, None] = None,
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -158,8 +167,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -203,7 +213,8 @@ def __call__(
y: torch.Tensor,
y_hat: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ y_insample: Union[torch.Tensor, None] = None,
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -248,8 +259,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -298,7 +310,8 @@ def __call__(
y: torch.Tensor,
y_hat: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ y_insample: Union[torch.Tensor, None] = None,
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -348,12 +361,12 @@ def __call__(
y_hat: torch.Tensor,
y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor (batch_size, output_size), Actual values.
`y_hat`: tensor (batch_size, output_size)), Predicted values.
- `y_insample`: tensor (batch_size, input_size), Actual insample Seasonal Naive predictions.
+ `y_insample`: tensor (batch_size, input_size), Actual insample values.
`mask`: tensor, Specifies date stamps per serie to consider in loss.
**Returns:**
@@ -366,7 +379,7 @@ def __call__(
),
axis=1,
)
- losses = _divide_no_nan(delta_y, scale[:, None])
+ losses = _divide_no_nan(delta_y, scale[:, None, None])
weights = self._compute_weights(y=y, mask=mask)
return _weighted_mean(losses=losses, weights=weights)
@@ -375,11 +388,11 @@ class relMSE(BasePointLoss):
"""Relative Mean Squared Error
Computes Relative Mean Squared Error (relMSE), as proposed by Hyndman & Koehler (2006)
as an alternative to percentage errors, to avoid measure unstability.
- $$ \mathrm{relMSE}(\\mathbf{y}, \\mathbf{\hat{y}}, \\mathbf{\hat{y}}^{naive1}) =
- \\frac{\mathrm{MSE}(\\mathbf{y}, \\mathbf{\hat{y}})}{\mathrm{MSE}(\\mathbf{y}, \\mathbf{\hat{y}}^{naive1})} $$
+ $$ \mathrm{relMSE}(\\mathbf{y}, \\mathbf{\hat{y}}, \\mathbf{\hat{y}}^{benchmark}) =
+ \\frac{\mathrm{MSE}(\\mathbf{y}, \\mathbf{\hat{y}})}{\mathrm{MSE}(\\mathbf{y}, \\mathbf{\hat{y}}^{benchmark})} $$
**Parameters:**
- `y_train`: numpy array, Training values.
+ `y_train`: numpy array, deprecated.
`horizon_weight`: Tensor of size h, weight for each timestamp of the forecasting window.
**References:**
@@ -391,34 +404,32 @@ class relMSE(BasePointLoss):
Submitted to the International Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)
"""
- def __init__(self, y_train, horizon_weight=None):
+ def __init__(self, y_train=None, horizon_weight=None):
super(relMSE, self).__init__(
horizon_weight=horizon_weight, outputsize_multiplier=1, output_names=[""]
)
- self.y_train = y_train
+ if y_train is not None:
+ raise DeprecationWarning("y_train will be deprecated in a future release.")
self.mse = MSE(horizon_weight=horizon_weight)
def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_benchmark: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor (batch_size, output_size), Actual values.
`y_hat`: tensor (batch_size, output_size)), Predicted values.
- `y_insample`: tensor (batch_size, input_size), Actual insample Seasonal Naive predictions.
+ `y_benchmark`: tensor (batch_size, output_size), Benchmark predicted values.
`mask`: tensor, Specifies date stamps per serie to consider in loss.
**Returns:**
`relMSE`: tensor (single value).
"""
- horizon = y.shape[-1]
- last_col = self.y_train[:, -1].unsqueeze(1)
- y_naive = last_col.repeat(1, horizon)
-
- norm = self.mse(y=y, y_hat=y_naive, mask=mask) # Already weighted
+ norm = self.mse(y=y, y_hat=y_benchmark, mask=mask) # Already weighted
norm = norm + 1e-5 # Numerical stability
loss = self.mse(y=y, y_hat=y_hat, mask=mask) # Already weighted
loss = _divide_no_nan(loss, norm)
@@ -456,8 +467,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -549,38 +561,48 @@ def __init__(self, level=[80, 90], quantiles=None, horizon_weight=None):
def domain_map(self, y_hat: torch.Tensor):
"""
- Identity domain map [B,T,H,Q]/[B,H,Q]
+ Input:
+ Univariate: [B, H, 1 * Q]
+ Multivariate: [B, H, N * Q]
+
+ Output: [B, H, N, Q]
"""
- return y_hat
+ output = y_hat.reshape(
+ y_hat.shape[0], y_hat.shape[1], -1, self.outputsize_multiplier
+ )
+
+ return output
def _compute_weights(self, y, mask):
"""
Compute final weights for each datapoint (based on all weights and all masks)
Set horizon_weight to a ones[H] tensor if not set.
If set, check that it has the same length as the horizon in x.
+
+ y: [B, h, N, 1]
+ mask: [B, h, N, 1]
"""
- if mask is None:
- mask = torch.ones_like(y, device=y.device)
- else:
- mask = mask.unsqueeze(1) # Add Q dimension.
if self.horizon_weight is None:
- self.horizon_weight = torch.ones(mask.shape[-1])
+ weights = torch.ones_like(mask)
else:
- assert mask.shape[-1] == len(
+ assert mask.shape[1] == len(
self.horizon_weight
), "horizon_weight must have same length as Y"
+ weights = self.horizon_weight.clone()
+ weights = weights[None, :, None, None]
+ weights = weights.to(mask.device)
+ weights = torch.ones_like(mask, device=mask.device) * weights
- weights = self.horizon_weight.clone()
- weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)
return weights * mask
def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -590,26 +612,24 @@ def __call__(
**Returns:**
`mqloss`: tensor (single value).
"""
+ # [B, h, N] -> [B, h, N, 1]
+ if y_hat.ndim == 3:
+ y_hat = y_hat.unsqueeze(-1)
+
+ y = y.unsqueeze(-1)
+ if mask is not None:
+ mask = mask.unsqueeze(-1)
+ else:
+ mask = torch.ones_like(y, device=y.device)
+
+ error = y_hat - y
- error = y_hat - y.unsqueeze(-1)
sq = torch.maximum(-error, torch.zeros_like(error))
s1_q = torch.maximum(error, torch.zeros_like(error))
- losses = (1 / len(self.quantiles)) * (
- self.quantiles * sq + (1 - self.quantiles) * s1_q
- )
-
- if y_hat.ndim == 3: # BaseWindows
- losses = losses.swapaxes(
- -2, -1
- ) # [B,H,Q] -> [B,Q,H] (needed for horizon weighting, H at the end)
- elif y_hat.ndim == 4: # BaseRecurrent
- losses = losses.swapaxes(-2, -1)
- losses = losses.swapaxes(
- -2, -3
- ) # [B,seq_len,H,Q] -> [B,Q,seq_len,H] (needed for horizon weighting, H at the end)
+ quantiles = self.quantiles[None, None, None, :]
+ losses = (1 / len(quantiles)) * (quantiles * sq + (1 - quantiles) * s1_q)
weights = self._compute_weights(y=losses, mask=mask) # Use losses for extra dim
- # NOTE: Weights do not have Q dimension.
return _weighted_mean(losses=losses, weights=weights)
@@ -700,9 +720,9 @@ def _init_sampling_distribution(self, device):
concentration0=concentration0, concentration1=concentration1
)
- def update_quantile(self, q: float = 0.5):
- self.q = q
- self.output_names = [f"_ql{q}"]
+ def update_quantile(self, q: List[float] = [0.5]):
+ self.q = q[0]
+ self.output_names = [f"_ql{q[0]}"]
self.has_predicted = True
def domain_map(self, y_hat):
@@ -711,9 +731,8 @@ def domain_map(self, y_hat):
Input shapes to this function:
- base_windows: y_hat = [B, h, 1]
- base_multivariate: y_hat = [B, h, n_series]
- base_recurrent: y_hat = [B, seq_len, h, n_series]
+ Univariate: y_hat = [B, h, 1]
+ Multivariate: y_hat = [B, h, N]
"""
if self.eval() and self.has_predicted:
quantiles = torch.full(
@@ -734,7 +753,7 @@ def domain_map(self, y_hat):
emb_outputs = self.output_layer(emb_inputs)
# Domain map
- y_hat = emb_outputs.squeeze(-1).squeeze(-1)
+ y_hat = emb_outputs.squeeze(-1)
return y_hat
@@ -767,20 +786,6 @@ def weighted_average(
return x.mean(dim=dim)
# %% ../../nbs/losses.pytorch.ipynb 65
-def bernoulli_domain_map(input: torch.Tensor):
- """Bernoulli Domain Map
- Maps input into distribution constraints, by construction input's
- last dimension is of matching `distr_args` length.
-
- **Parameters:**
- `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
-
- **Returns:**
- `(probs,)`: tuple with tensors of Poisson distribution arguments.
- """
- return (input.squeeze(-1),)
-
-
def bernoulli_scale_decouple(output, loc=None, scale=None):
"""Bernoulli Scale Decouple
@@ -795,22 +800,6 @@ def bernoulli_scale_decouple(output, loc=None, scale=None):
return (probs,)
-def student_domain_map(input: torch.Tensor):
- """Student T Domain Map
- Maps input into distribution constraints, by construction input's
- last dimension is of matching `distr_args` length.
-
- **Parameters:**
- `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
- `eps`: float, helps the initialization of scale for easier optimization.
-
- **Returns:**
- `(df, loc, scale)`: tuple with tensors of StudentT distribution arguments.
- """
- df, loc, scale = torch.tensor_split(input, 3, dim=-1)
- return df.squeeze(-1), loc.squeeze(-1), scale.squeeze(-1)
-
-
def student_scale_decouple(output, loc=None, scale=None, eps: float = 0.1):
"""Normal Scale Decouple
@@ -827,22 +816,6 @@ def student_scale_decouple(output, loc=None, scale=None, eps: float = 0.1):
return (df, mean, tscale)
-def normal_domain_map(input: torch.Tensor):
- """Normal Domain Map
- Maps input into distribution constraints, by construction input's
- last dimension is of matching `distr_args` length.
-
- **Parameters:**
- `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
- `eps`: float, helps the initialization of scale for easier optimization.
-
- **Returns:**
- `(mean, std)`: tuple with tensors of Normal distribution arguments.
- """
- mean, std = torch.tensor_split(input, 2, dim=-1)
- return mean.squeeze(-1), std.squeeze(-1)
-
-
def normal_scale_decouple(output, loc=None, scale=None, eps: float = 0.2):
"""Normal Scale Decouple
@@ -858,20 +831,6 @@ def normal_scale_decouple(output, loc=None, scale=None, eps: float = 0.2):
return (mean, std)
-def poisson_domain_map(input: torch.Tensor):
- """Poisson Domain Map
- Maps input into distribution constraints, by construction input's
- last dimension is of matching `distr_args` length.
-
- **Parameters:**
- `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
-
- **Returns:**
- `(rate,)`: tuple with tensors of Poisson distribution arguments.
- """
- return (input.squeeze(-1),)
-
-
def poisson_scale_decouple(output, loc=None, scale=None):
"""Poisson Scale Decouple
@@ -887,21 +846,6 @@ def poisson_scale_decouple(output, loc=None, scale=None):
return (rate,)
-def nbinomial_domain_map(input: torch.Tensor):
- """Negative Binomial Domain Map
- Maps input into distribution constraints, by construction input's
- last dimension is of matching `distr_args` length.
-
- **Parameters:**
- `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
-
- **Returns:**
- `(total_count, alpha)`: tuple with tensors of N.Binomial distribution arguments.
- """
- mu, alpha = torch.tensor_split(input, 2, dim=-1)
- return mu.squeeze(-1), alpha.squeeze(-1)
-
-
def nbinomial_scale_decouple(output, loc=None, scale=None):
"""Negative Binomial Scale Decouple
@@ -964,10 +908,12 @@ class Tweedie(Distribution):
Series B (Methodological), 49(2), 127–162. http://www.jstor.org/stable/2345415](http://www.jstor.org/stable/2345415)
"""
+ arg_constraints = {"log_mu": constraints.real}
+ support = constraints.nonnegative
+
def __init__(self, log_mu, rho, validate_args=None):
# TODO: add sigma2 dispersion
# TODO add constraints
- # arg_constraints = {'log_mu': constraints.real, 'rho': constraints.positive}
# support = constraints.real
self.log_mu = log_mu
self.rho = rho
@@ -1001,7 +947,7 @@ def sample(self, sample_shape=torch.Size()):
beta = beta.expand(shape)
N = torch.poisson(rate) + 1e-5
- gamma = torch.distributions.gamma.Gamma(N * alpha, beta)
+ gamma = Gamma(N * alpha, beta)
samples = gamma.sample()
samples[N == 0] = 0
@@ -1017,12 +963,12 @@ def log_prob(self, y_true):
return a - b
-def tweedie_domain_map(input: torch.Tensor):
+def tweedie_domain_map(input: torch.Tensor, rho: float = 1.5):
"""
Maps output of neural network to domain of distribution loss
"""
- return (input.squeeze(-1),)
+ return (input, rho)
def tweedie_scale_decouple(output, loc=None, scale=None):
@@ -1032,14 +978,14 @@ def tweedie_scale_decouple(output, loc=None, scale=None):
count and logits based on anchoring `loc`, `scale`.
Also adds Tweedie domain protection to the distribution parameters.
"""
- log_mu = output[0]
+ log_mu, rho = output
log_mu = F.softplus(log_mu)
log_mu = torch.clamp(log_mu, 1e-9, 37)
if (loc is not None) and (scale is not None):
log_mu += torch.log(loc)
log_mu = torch.clamp(log_mu, 1e-9, 37)
- return (log_mu,)
+ return (log_mu, rho)
# %% ../../nbs/losses.pytorch.ipynb 67
# Code adapted from: https://github.com/awslabs/gluonts/blob/61133ef6e2d88177b32ace4afc6843ab9a7bc8cd/src/gluonts/torch/distributions/isqf.py
@@ -1097,6 +1043,14 @@ def crps(self, y: torch.Tensor) -> torch.Tensor:
p = self.base_dist.crps(z)
return p * scale
+ @property
+ def mean(self):
+ """
+ Function used to compute the empirical mean
+ """
+ samples = self.sample([1000])
+ return samples.mean(dim=0)
+
class BaseISQF(Distribution):
"""
@@ -1753,7 +1707,7 @@ def isqf_domain_map(
last dimension is of matching `distr_args` length.
**Parameters:**
- `input`: tensor, of dimensions [B,T,H,theta] or [B,H,theta].
+ `input`: tensor, of dimensions [B, H, N * n_outputs].
`tol`: float, tolerance.
`quantiles`: tensor, quantiles used for ISQF (i.e. x-positions for the knots).
`num_pieces`: int, num_pieces used for each quantile spline.
@@ -1768,6 +1722,10 @@ def isqf_domain_map(
# Because in this case the spline knots could be squeezed together
# and cause overflow in spline CRPS computation
num_qk = len(quantiles)
+ n_outputs = 2 * (num_qk - 1) * num_pieces + 2 + num_qk
+
+ # Reshape: [B, h, N * n_outputs] -> [B, h, N, n_outputs]
+ input = input.reshape(input.shape[0], input.shape[1], -1, n_outputs)
start_index = 0
spline_knots = input[..., start_index : start_index + (num_qk - 1) * num_pieces]
start_index += (num_qk - 1) * num_pieces
@@ -1777,26 +1735,19 @@ def isqf_domain_map(
start_index += 1
beta_r = input[..., start_index : start_index + 1]
start_index += 1
- quantile_knots = input[..., start_index : start_index + num_qk]
-
- qk_y = torch.cat(
- [
- quantile_knots[..., 0:1],
- torch.abs(quantile_knots[..., 1:]) + tol,
- ],
- dim=-1,
- )
- qk_y = torch.cumsum(qk_y, dim=-1)
+ quantile_knots = F.softplus(input[..., start_index : start_index + num_qk]) + tol
+
+ qk_y = torch.cumsum(quantile_knots, dim=-1)
# Prevent overflow when we compute 1/beta
- beta_l = torch.abs(beta_l.squeeze(-1)) + tol
- beta_r = torch.abs(beta_r.squeeze(-1)) + tol
+ beta_l = F.softplus(beta_l.squeeze(-1)) + tol
+ beta_r = F.softplus(beta_r.squeeze(-1)) + tol
# Reshape spline arguments
batch_shape = spline_knots.shape[:-1]
# repeat qk_x from (num_qk,) to (*batch_shape, num_qk)
- qk_x_repeat = torch.sort(quantiles).values.repeat(*batch_shape, 1).to(input.device)
+ qk_x_repeat = quantiles.repeat(*batch_shape, 1).to(input.device)
# knots and heights have shape (*batch_shape, (num_qk-1)*num_pieces)
# reshape them to (*batch_shape, (num_qk-1), num_pieces)
@@ -1902,15 +1853,6 @@ def __init__(
Tweedie=Tweedie,
ISQF=ISQF,
)
- domain_maps = dict(
- Bernoulli=bernoulli_domain_map,
- Normal=normal_domain_map,
- Poisson=poisson_domain_map,
- StudentT=student_domain_map,
- NegativeBinomial=nbinomial_domain_map,
- Tweedie=tweedie_domain_map,
- ISQF=partial(isqf_domain_map, quantiles=qs, num_pieces=num_pieces),
- )
scale_decouples = dict(
Bernoulli=bernoulli_scale_decouple,
Normal=normal_scale_decouple,
@@ -1935,9 +1877,23 @@ def __init__(
assert (
distribution in available_distributions.keys()
), f"{distribution} not available"
+ if distribution == "ISQF":
+ quantiles = torch.sort(qs).values
+ self.domain_map = partial(
+ isqf_domain_map, quantiles=quantiles, num_pieces=num_pieces
+ )
+ if return_params:
+ raise Exception("ISQF does not support 'return_params=True'")
+ elif distribution == "Tweedie":
+ rho = distribution_kwargs.pop("rho")
+ self.domain_map = partial(tweedie_domain_map, rho=rho)
+ if return_params:
+ raise Exception("Tweedie does not support 'return_params=True'")
+ else:
+ self.domain_map = self._domain_map
+
self.distribution = distribution
self._base_distribution = available_distributions[distribution]
- self.domain_map = domain_maps[distribution]
self.scale_decouple = scale_decouples[distribution]
self.distribution_kwargs = distribution_kwargs
self.num_samples = num_samples
@@ -1953,6 +1909,16 @@ def __init__(
self.outputsize_multiplier = len(self.param_names)
self.is_distribution_output = True
+ self.has_predicted = False
+
+ def _domain_map(self, input: torch.Tensor):
+ """
+ Maps output of neural network to domain of distribution loss
+
+ """
+ output = torch.tensor_split(input, self.outputsize_multiplier, dim=2)
+
+ return output
def get_distribution(self, distr_args, **distribution_kwargs) -> Distribution:
"""
@@ -1965,10 +1931,10 @@ def get_distribution(self, distr_args, **distribution_kwargs) -> Distribution:
**Returns**
`Distribution`: AffineTransformed distribution.
"""
- # TransformedDistribution(distr, [AffineTransform(loc=loc, scale=scale)])
distr = self._base_distribution(*distr_args, **distribution_kwargs)
+ self.distr_mean = distr.mean
- if self.distribution == "Poisson":
+ if self.distribution in ("Poisson", "NegativeBinomial"):
distr.support = constraints.nonnegative
return distr
@@ -1979,7 +1945,7 @@ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):
**Parameters**
`distr_args`: Constructor arguments for the underlying Distribution type.
- `num_samples`: int=500, overwrite number of samples for the empirical quantiles.
+ `num_samples`: int, overwrite number of samples for the empirical quantiles.
**Returns**
`samples`: tensor, shape [B,H,`num_samples`].
@@ -1988,29 +1954,39 @@ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):
if num_samples is None:
num_samples = self.num_samples
- # print(distr_args[0].size())
- B, H = distr_args[0].shape[:2]
- Q = len(self.quantiles)
-
# Instantiate Scaled Decoupled Distribution
distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)
samples = distr.sample(sample_shape=(num_samples,))
- samples = samples.permute(1, 2, 0) # [samples,B,H] -> [B,H,samples]
- samples = samples.view(B * H, num_samples)
- sample_mean = torch.mean(samples, dim=-1)
+ samples = samples.permute(
+ 1, 2, 3, 0
+ ) # [samples, B, H, N] -> [B, H, N, samples]
+
+ sample_mean = torch.mean(samples, dim=-1, keepdim=True)
# Compute quantiles
quantiles_device = self.quantiles.to(distr_args[0].device)
- quants = torch.quantile(input=samples, q=quantiles_device, dim=1)
- quants = quants.permute((1, 0)) # [Q, B*H] -> [B*H, Q]
-
- # Final reshapes
- samples = samples.view(B, H, num_samples)
- sample_mean = sample_mean.view(B, H, 1)
- quants = quants.view(B, H, Q)
+ quants = torch.quantile(input=samples, q=quantiles_device, dim=-1)
+ quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]
return samples, sample_mean, quants
+ def update_quantile(self, q: Optional[List[float]] = None):
+ if q is not None:
+ self.quantiles = nn.Parameter(
+ torch.tensor(q, dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = (
+ [""]
+ + [f"_ql{q_i}" for q_i in q]
+ + self.return_params * self.param_names
+ )
+ self.has_predicted = True
+ elif q is None and self.has_predicted:
+ self.quantiles = nn.Parameter(
+ torch.tensor([0.5], dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = ["", "-median"] + self.return_params * self.param_names
+
def __call__(
self,
y: torch.Tensor,
@@ -2029,10 +2005,6 @@ def __call__(
**Parameters**
`y`: tensor, Actual values.
`distr_args`: Constructor arguments for the underlying Distribution type.
- `loc`: Optional tensor, of the same shape as the batch_shape + event_shape
- of the resulting distribution.
- `scale`: Optional tensor, of the same shape as the batch_shape+event_shape
- of the resulting distribution.
`mask`: tensor, Specifies date stamps per serie to consider in loss.
**Returns**
@@ -2079,6 +2051,7 @@ def __init__(
return_params=False,
batch_correlation=False,
horizon_correlation=False,
+ weighted=False,
):
super(PMM, self).__init__()
# Transform level to MQLoss parameters
@@ -2093,21 +2066,36 @@ def __init__(
self.num_samples = num_samples
self.batch_correlation = batch_correlation
self.horizon_correlation = horizon_correlation
+ self.weighted = weighted
# If True, predict_step will return Distribution's parameters
self.return_params = return_params
+
+ lambda_names = [f"-lambda-{i}" for i in range(1, n_components + 1)]
+ if weighted:
+ weight_names = [f"-weight-{i}" for i in range(1, n_components + 1)]
+ self.param_names = [i for j in zip(lambda_names, weight_names) for i in j]
+ else:
+ self.param_names = lambda_names
+
if self.return_params:
- self.param_names = [f"-lambda-{i}" for i in range(1, n_components + 1)]
self.output_names = self.output_names + self.param_names
# Add first output entry for the sample_mean
self.output_names.insert(0, "")
- self.outputsize_multiplier = n_components
+ self.n_outputs = 1 + weighted
+ self.n_components = n_components
+ self.outputsize_multiplier = self.n_outputs * n_components
self.is_distribution_output = True
+ self.has_predicted = False
def domain_map(self, output: torch.Tensor):
- return (output,) # , weights
+ output = output.reshape(
+ output.shape[0], output.shape[1], -1, self.outputsize_multiplier
+ )
+
+ return torch.tensor_split(output, self.n_outputs, dim=-1)
def scale_decouple(
self,
@@ -2121,26 +2109,61 @@ def scale_decouple(
variance and residual location based on anchoring `loc`, `scale`.
Also adds domain protection to the distribution parameters.
"""
- lambdas = output[0]
+ if self.weighted:
+ lambdas, weights = output
+ weights = F.softmax(weights, dim=-1)
+ else:
+ lambdas = output[0]
+
if (loc is not None) and (scale is not None):
- loc = loc.view(lambdas.size(dim=0), 1, -1)
- scale = scale.view(lambdas.size(dim=0), 1, -1)
+ if loc.ndim == 3:
+ loc = loc.unsqueeze(-1)
+ scale = scale.unsqueeze(-1)
lambdas = (lambdas * scale) + loc
- lambdas = F.softplus(lambdas)
- return (lambdas,)
- def sample(self, distr_args, num_samples=None):
+ lambdas = F.softplus(lambdas) + 1e-3
+
+ if self.weighted:
+ return (lambdas, weights)
+ else:
+ return (lambdas,)
+
+ def get_distribution(self, distr_args) -> Distribution:
+ """
+ Construct the associated Pytorch Distribution, given the collection of
+ constructor arguments and, optionally, location and scale tensors.
+
+ **Parameters**
+ `distr_args`: Constructor arguments for the underlying Distribution type.
+
+ **Returns**
+ `Distribution`: AffineTransformed distribution.
+ """
+ if self.weighted:
+ lambdas, weights = distr_args
+ else:
+ lambdas = distr_args[0]
+ weights = torch.full_like(lambdas, fill_value=1 / self.n_components)
+
+ mix = Categorical(weights)
+ components = Poisson(rate=lambdas)
+ components.support = constraints.nonnegative
+ distr = MixtureSameFamily(
+ mixture_distribution=mix, component_distribution=components
+ )
+
+ self.distr_mean = distr.mean
+
+ return distr
+
+ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):
"""
Construct the empirical quantiles from the estimated Distribution,
sampling from it `num_samples` independently.
**Parameters**
`distr_args`: Constructor arguments for the underlying Distribution type.
- `loc`: Optional tensor, of the same shape as the batch_shape + event_shape
- of the resulting distribution.
- `scale`: Optional tensor, of the same shape as the batch_shape+event_shape
- of the resulting distribution.
- `num_samples`: int=500, overwrites number of samples for the empirical quantiles.
+ `num_samples`: int, overwrite number of samples for the empirical quantiles.
**Returns**
`samples`: tensor, shape [B,H,`num_samples`].
@@ -2149,100 +2172,75 @@ def sample(self, distr_args, num_samples=None):
if num_samples is None:
num_samples = self.num_samples
- lambdas = distr_args[0]
- B, H, K = lambdas.size()
- Q = len(self.quantiles)
-
- # Sample K ~ Mult(weights)
- # shared across B, H
- # weights = torch.repeat_interleave(input=weights, repeats=H, dim=2)
- weights = (1 / K) * torch.ones_like(lambdas, device=lambdas.device)
-
- # Avoid loop, vectorize
- weights = weights.reshape(-1, K)
- lambdas = lambdas.flatten()
-
- # Vectorization trick to recover row_idx
- sample_idxs = torch.multinomial(
- input=weights, num_samples=num_samples, replacement=True
- )
- aux_col_idx = (
- torch.unsqueeze(torch.arange(B * H, device=lambdas.device), -1) * K
- )
-
- # To device
- sample_idxs = sample_idxs.to(lambdas.device)
-
- sample_idxs = sample_idxs + aux_col_idx
- sample_idxs = sample_idxs.flatten()
-
- sample_lambdas = lambdas[sample_idxs]
+ # Instantiate Scaled Decoupled Distribution
+ distr = self.get_distribution(distr_args=distr_args)
+ samples = distr.sample(sample_shape=(num_samples,))
+ samples = samples.permute(
+ 1, 2, 3, 0
+ ) # [samples, B, H, N] -> [B, H, N, samples]
- # Sample y ~ Poisson(lambda) independently
- samples = torch.poisson(sample_lambdas).to(lambdas.device)
- samples = samples.view(B * H, num_samples)
- sample_mean = torch.mean(samples, dim=-1)
+ sample_mean = torch.mean(samples, dim=-1, keepdim=True)
# Compute quantiles
- quantiles_device = self.quantiles.to(lambdas.device)
- quants = torch.quantile(input=samples, q=quantiles_device, dim=1)
- quants = quants.permute((1, 0)) # Q, B*H
-
- # Final reshapes
- samples = samples.view(B, H, num_samples)
- sample_mean = sample_mean.view(B, H, 1)
- quants = quants.view(B, H, Q)
+ quantiles_device = self.quantiles.to(distr_args[0].device)
+ quants = torch.quantile(input=samples, q=quantiles_device, dim=-1)
+ quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]
return samples, sample_mean, quants
- def neglog_likelihood(
+ def update_quantile(self, q: Optional[List[float]] = None):
+ if q is not None:
+ self.quantiles = nn.Parameter(
+ torch.tensor(q, dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = (
+ [""]
+ + [f"_ql{q_i}" for q_i in q]
+ + self.return_params * self.param_names
+ )
+ self.has_predicted = True
+ elif q is None and self.has_predicted:
+ self.quantiles = nn.Parameter(
+ torch.tensor([0.5], dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = ["", "-median"] + self.return_params * self.param_names
+
+ def __call__(
self,
y: torch.Tensor,
- distr_args: Tuple[torch.Tensor],
+ distr_args: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
):
- if mask is None:
- mask = (y > 0) * 1
- else:
- mask = mask * ((y > 0) * 1)
-
- eps = 1e-10
- lambdas = distr_args[0]
- B, H, K = lambdas.size()
-
- weights = (1 / K) * torch.ones_like(lambdas, device=lambdas.device)
+ """
+ Computes the negative log-likelihood objective function.
+ To estimate the following predictive distribution:
- y = y[:, :, None]
- mask = mask[:, :, None]
+ $$\mathrm{P}(\mathbf{y}_{\\tau}\,|\,\\theta) \\quad \mathrm{and} \\quad -\log(\mathrm{P}(\mathbf{y}_{\\tau}\,|\,\\theta))$$
- y = y * mask # Protect y negative entries
+ where $\\theta$ represents the distributions parameters. It aditionally
+ summarizes the objective signal using a weighted average using the `mask` tensor.
- # Single Poisson likelihood
- log_pi = y.xlogy(lambdas + eps) - lambdas - (y + 1).lgamma()
+ **Parameters**
+ `y`: tensor, Actual values.
+ `distr_args`: Constructor arguments for the underlying Distribution type.
+ `mask`: tensor, Specifies date stamps per serie to consider in loss.
+ **Returns**
+ `loss`: scalar, weighted loss function against which backpropagation will be performed.
+ """
+ # Instantiate Scaled Decoupled Distribution
+ distr = self.get_distribution(distr_args=distr_args)
+ x = distr._pad(y)
+ log_prob_x = distr.component_distribution.log_prob(x)
+ log_mix_prob = torch.log_softmax(distr.mixture_distribution.logits, dim=-1)
if self.batch_correlation:
- log_pi = torch.sum(log_pi, dim=0, keepdim=True)
-
+ log_prob_x = torch.sum(log_prob_x, dim=0, keepdim=True)
if self.horizon_correlation:
- log_pi = torch.sum(log_pi, dim=1, keepdim=True)
-
- # Numerically Stable Mixture loglikelihood
- loglik = torch.logsumexp((torch.log(weights) + log_pi), dim=2, keepdim=True)
- loglik = loglik * mask
+ log_prob_x = torch.sum(log_prob_x, dim=1, keepdim=True)
- mean = torch.sum(weights * lambdas, axis=-1, keepdims=True)
- reglrz = torch.mean(torch.square(y - mean) * mask)
- loss = -torch.mean(loglik) + 0.001 * reglrz
- return loss
-
- def __call__(
- self,
- y: torch.Tensor,
- distr_args: Tuple[torch.Tensor],
- mask: Union[torch.Tensor, None] = None,
- ):
+ loss_values = -torch.logsumexp(log_prob_x + log_mix_prob, dim=-1)
- return self.neglog_likelihood(y=y, distr_args=distr_args, mask=mask)
+ return weighted_average(loss_values, weights=mask)
# %% ../../nbs/losses.pytorch.ipynb 82
class GMM(torch.nn.Module):
@@ -2280,6 +2278,7 @@ def __init__(
return_params=False,
batch_correlation=False,
horizon_correlation=False,
+ weighted=False,
):
super(GMM, self).__init__()
# Transform level to MQLoss parameters
@@ -2294,24 +2293,39 @@ def __init__(
self.num_samples = num_samples
self.batch_correlation = batch_correlation
self.horizon_correlation = horizon_correlation
+ self.weighted = weighted
# If True, predict_step will return Distribution's parameters
self.return_params = return_params
+
+ mu_names = [f"-mu-{i}" for i in range(1, n_components + 1)]
+ std_names = [f"-std-{i}" for i in range(1, n_components + 1)]
+ if weighted:
+ weight_names = [f"-weight-{i}" for i in range(1, n_components + 1)]
+ self.param_names = [
+ i for j in zip(mu_names, std_names, weight_names) for i in j
+ ]
+ else:
+ self.param_names = [i for j in zip(mu_names, std_names) for i in j]
+
if self.return_params:
- mu_names = [f"-mu-{i}" for i in range(1, n_components + 1)]
- std_names = [f"-std-{i}" for i in range(1, n_components + 1)]
- mu_std_names = [i for j in zip(mu_names, std_names) for i in j]
- self.output_names = self.output_names + mu_std_names
+ self.output_names = self.output_names + self.param_names
# Add first output entry for the sample_mean
self.output_names.insert(0, "")
- self.outputsize_multiplier = 2 * n_components
+ self.n_outputs = 2 + weighted
+ self.n_components = n_components
+ self.outputsize_multiplier = self.n_outputs * n_components
self.is_distribution_output = True
+ self.has_predicted = False
def domain_map(self, output: torch.Tensor):
- means, stds = torch.tensor_split(output, 2, dim=-1)
- return (means, stds)
+ output = output.reshape(
+ output.shape[0], output.shape[1], -1, self.outputsize_multiplier
+ )
+
+ return torch.tensor_split(output, self.n_outputs, dim=-1)
def scale_decouple(
self,
@@ -2326,130 +2340,136 @@ def scale_decouple(
variance and residual location based on anchoring `loc`, `scale`.
Also adds domain protection to the distribution parameters.
"""
- means, stds = output
+ if self.weighted:
+ means, stds, weights = output
+ weights = F.softmax(weights, dim=-1)
+ else:
+ means, stds = output
+
stds = F.softplus(stds)
if (loc is not None) and (scale is not None):
- loc = loc.view(means.size(dim=0), 1, -1)
- scale = scale.view(means.size(dim=0), 1, -1)
+ if loc.ndim == 3:
+ loc = loc.unsqueeze(-1)
+ scale = scale.unsqueeze(-1)
means = (means * scale) + loc
stds = (stds + eps) * scale
- return (means, stds)
- def sample(self, distr_args, num_samples=None):
+ if self.weighted:
+ return (means, stds, weights)
+ else:
+ return (means, stds)
+
+ def get_distribution(self, distr_args) -> Distribution:
"""
- Construct the empirical quantiles from the estimated Distribution,
- sampling from it `num_samples` independently.
+ Construct the associated Pytorch Distribution, given the collection of
+ constructor arguments and, optionally, location and scale tensors.
**Parameters**
`distr_args`: Constructor arguments for the underlying Distribution type.
- `loc`: Optional tensor, of the same shape as the batch_shape + event_shape
- of the resulting distribution.
- `scale`: Optional tensor, of the same shape as the batch_shape+event_shape
- of the resulting distribution.
- `num_samples`: int=500, number of samples for the empirical quantiles.
**Returns**
- `samples`: tensor, shape [B,H,`num_samples`].
- `quantiles`: tensor, empirical quantiles defined by `levels`.
+ `Distribution`: AffineTransformed distribution.
"""
- if num_samples is None:
- num_samples = self.num_samples
-
- means, stds = distr_args
- B, H, K = means.size()
- Q = len(self.quantiles)
- assert means.shape == stds.shape
+ if self.weighted:
+ means, stds, weights = distr_args
+ else:
+ means, stds = distr_args
+ weights = torch.full_like(means, fill_value=1 / self.n_components)
- # Sample K ~ Mult(weights)
- # shared across B, H
- # weights = torch.repeat_interleave(input=weights, repeats=H, dim=2)
+ mix = Categorical(weights)
+ components = Normal(loc=means, scale=stds)
+ distr = MixtureSameFamily(
+ mixture_distribution=mix, component_distribution=components
+ )
- weights = (1 / K) * torch.ones_like(means, device=means.device)
+ self.distr_mean = distr.mean
- # Avoid loop, vectorize
- weights = weights.reshape(-1, K)
- means = means.flatten()
- stds = stds.flatten()
+ return distr
- # Vectorization trick to recover row_idx
- sample_idxs = torch.multinomial(
- input=weights, num_samples=num_samples, replacement=True
- )
- aux_col_idx = torch.unsqueeze(torch.arange(B * H, device=means.device), -1) * K
+ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):
+ """
+ Construct the empirical quantiles from the estimated Distribution,
+ sampling from it `num_samples` independently.
- # To device
- sample_idxs = sample_idxs.to(means.device)
+ **Parameters**
+ `distr_args`: Constructor arguments for the underlying Distribution type.
+ `num_samples`: int, overwrite number of samples for the empirical quantiles.
- sample_idxs = sample_idxs + aux_col_idx
- sample_idxs = sample_idxs.flatten()
+ **Returns**
+ `samples`: tensor, shape [B,H,`num_samples`].
+ `quantiles`: tensor, empirical quantiles defined by `levels`.
+ """
+ if num_samples is None:
+ num_samples = self.num_samples
- sample_means = means[sample_idxs]
- sample_stds = stds[sample_idxs]
+ # Instantiate Scaled Decoupled Distribution
+ distr = self.get_distribution(distr_args=distr_args)
+ samples = distr.sample(sample_shape=(num_samples,))
+ samples = samples.permute(
+ 1, 2, 3, 0
+ ) # [samples, B, H, N] -> [B, H, N, samples]
- # Sample y ~ Normal(mu, std) independently
- samples = torch.normal(sample_means, sample_stds).to(means.device)
- samples = samples.view(B * H, num_samples)
- sample_mean = torch.mean(samples, dim=-1)
+ sample_mean = torch.mean(samples, dim=-1, keepdim=True)
# Compute quantiles
- quantiles_device = self.quantiles.to(means.device)
- quants = torch.quantile(input=samples, q=quantiles_device, dim=1)
- quants = quants.permute((1, 0)) # Q, B*H
-
- # Final reshapes
- samples = samples.view(B, H, num_samples)
- sample_mean = sample_mean.view(B, H, 1)
- quants = quants.view(B, H, Q)
+ quantiles_device = self.quantiles.to(distr_args[0].device)
+ quants = torch.quantile(input=samples, q=quantiles_device, dim=-1)
+ quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]
return samples, sample_mean, quants
- def neglog_likelihood(
+ def update_quantile(self, q: Optional[List[float]] = None):
+ if q is not None:
+ self.quantiles = nn.Parameter(
+ torch.tensor(q, dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = (
+ [""]
+ + [f"_ql{q_i}" for q_i in q]
+ + self.return_params * self.param_names
+ )
+ self.has_predicted = True
+ elif q is None and self.has_predicted:
+ self.quantiles = nn.Parameter(
+ torch.tensor([0.5], dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = ["", "-median"] + self.return_params * self.param_names
+
+ def __call__(
self,
y: torch.Tensor,
- distr_args: Tuple[torch.Tensor, torch.Tensor],
+ distr_args: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
):
+ """
+ Computes the negative log-likelihood objective function.
+ To estimate the following predictive distribution:
- if mask is None:
- mask = torch.ones_like(y)
-
- means, stds = distr_args
- B, H, K = means.size()
-
- weights = (1 / K) * torch.ones_like(means, device=means.device)
+ $$\mathrm{P}(\mathbf{y}_{\\tau}\,|\,\\theta) \\quad \mathrm{and} \\quad -\log(\mathrm{P}(\mathbf{y}_{\\tau}\,|\,\\theta))$$
- y = y[:, :, None]
- mask = mask[:, :, None]
+ where $\\theta$ represents the distributions parameters. It aditionally
+ summarizes the objective signal using a weighted average using the `mask` tensor.
- var = stds**2
- log_stds = torch.log(stds)
- log_pi = (
- -((y - means) ** 2 / (2 * var))
- - log_stds
- - math.log(math.sqrt(2 * math.pi))
- )
+ **Parameters**
+ `y`: tensor, Actual values.
+ `distr_args`: Constructor arguments for the underlying Distribution type.
+ `mask`: tensor, Specifies date stamps per serie to consider in loss.
+ **Returns**
+ `loss`: scalar, weighted loss function against which backpropagation will be performed.
+ """
+ # Instantiate Scaled Decoupled Distribution
+ distr = self.get_distribution(distr_args=distr_args)
+ x = distr._pad(y)
+ log_prob_x = distr.component_distribution.log_prob(x)
+ log_mix_prob = torch.log_softmax(distr.mixture_distribution.logits, dim=-1)
if self.batch_correlation:
- log_pi = torch.sum(log_pi, dim=0, keepdim=True)
-
+ log_prob_x = torch.sum(log_prob_x, dim=0, keepdim=True)
if self.horizon_correlation:
- log_pi = torch.sum(log_pi, dim=1, keepdim=True)
-
- # Numerically Stable Mixture loglikelihood
- loglik = torch.logsumexp((torch.log(weights) + log_pi), dim=2, keepdim=True)
- loglik = loglik * mask
-
- loss = -torch.mean(loglik)
- return loss
-
- def __call__(
- self,
- y: torch.Tensor,
- distr_args: Tuple[torch.Tensor, torch.Tensor],
- mask: Union[torch.Tensor, None] = None,
- ):
+ log_prob_x = torch.sum(log_prob_x, dim=1, keepdim=True)
+ loss_values = -torch.logsumexp(log_prob_x + log_mix_prob, dim=-1)
- return self.neglog_likelihood(y=y, distr_args=distr_args, mask=mask)
+ return weighted_average(loss_values, weights=mask)
# %% ../../nbs/losses.pytorch.ipynb 90
class NBMM(torch.nn.Module):
@@ -2483,6 +2503,7 @@ def __init__(
quantiles=None,
num_samples=1000,
return_params=False,
+ weighted=False,
):
super(NBMM, self).__init__()
# Transform level to MQLoss parameters
@@ -2495,26 +2516,41 @@ def __init__(
qs = torch.Tensor(quantiles)
self.quantiles = torch.nn.Parameter(qs, requires_grad=False)
self.num_samples = num_samples
+ self.weighted = weighted
# If True, predict_step will return Distribution's parameters
self.return_params = return_params
- if self.return_params:
- total_count_names = [
- f"-total_count-{i}" for i in range(1, n_components + 1)
+
+ total_count_names = [f"-total_count-{i}" for i in range(1, n_components + 1)]
+ probs_names = [f"-probs-{i}" for i in range(1, n_components + 1)]
+ if weighted:
+ weight_names = [f"-weight-{i}" for i in range(1, n_components + 1)]
+ self.param_names = [
+ i for j in zip(total_count_names, probs_names, weight_names) for i in j
+ ]
+ else:
+ self.param_names = [
+ i for j in zip(total_count_names, probs_names) for i in j
]
- probs_names = [f"-probs-{i}" for i in range(1, n_components + 1)]
- param_names = [i for j in zip(total_count_names, probs_names) for i in j]
- self.output_names = self.output_names + param_names
+
+ if self.return_params:
+ self.output_names = self.output_names + self.param_names
# Add first output entry for the sample_mean
self.output_names.insert(0, "")
- self.outputsize_multiplier = 2 * n_components
+ self.n_outputs = 2 + weighted
+ self.n_components = n_components
+ self.outputsize_multiplier = self.n_outputs * n_components
self.is_distribution_output = True
+ self.has_predicted = False
def domain_map(self, output: torch.Tensor):
- mu, alpha = torch.tensor_split(output, 2, dim=-1)
- return (mu, alpha)
+ output = output.reshape(
+ output.shape[0], output.shape[1], -1, self.outputsize_multiplier
+ )
+
+ return torch.tensor_split(output, self.n_outputs, dim=-1)
def scale_decouple(
self,
@@ -2530,11 +2566,18 @@ def scale_decouple(
Also adds domain protection to the distribution parameters.
"""
# Efficient NBinomial parametrization
- mu, alpha = output
+ if self.weighted:
+ mu, alpha, weights = output
+ weights = F.softmax(weights, dim=-1)
+ else:
+ mu, alpha = output
+
mu = F.softplus(mu) + 1e-8
alpha = F.softplus(alpha) + 1e-8 # alpha = 1/total_counts
if (loc is not None) and (scale is not None):
- loc = loc.view(mu.size(dim=0), 1, -1)
+ if loc.ndim == 3:
+ loc = loc.unsqueeze(-1)
+ scale = scale.unsqueeze(-1)
mu *= loc
alpha /= loc + 1.0
@@ -2543,20 +2586,47 @@ def scale_decouple(
# => probs = mu / [total_count * (1 + mu * (1/total_count))]
total_count = 1.0 / alpha
probs = (mu * alpha / (1.0 + mu * alpha)) + 1e-8
- return (total_count, probs)
+ if self.weighted:
+ return (total_count, probs, weights)
+ else:
+ return (total_count, probs)
+
+ def get_distribution(self, distr_args) -> Distribution:
+ """
+ Construct the associated Pytorch Distribution, given the collection of
+ constructor arguments and, optionally, location and scale tensors.
+
+ **Parameters**
+ `distr_args`: Constructor arguments for the underlying Distribution type.
+
+ **Returns**
+ `Distribution`: AffineTransformed distribution.
+ """
+ if self.weighted:
+ total_count, probs, weights = distr_args
+ else:
+ total_count, probs = distr_args
+ weights = torch.full_like(total_count, fill_value=1 / self.n_components)
+
+ mix = Categorical(weights)
+ components = NegativeBinomial(total_count, probs)
+ components.support = constraints.nonnegative
+ distr = MixtureSameFamily(
+ mixture_distribution=mix, component_distribution=components
+ )
+
+ self.distr_mean = distr.mean
+
+ return distr
- def sample(self, distr_args, num_samples=None):
+ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):
"""
Construct the empirical quantiles from the estimated Distribution,
sampling from it `num_samples` independently.
**Parameters**
`distr_args`: Constructor arguments for the underlying Distribution type.
- `loc`: Optional tensor, of the same shape as the batch_shape + event_shape
- of the resulting distribution.
- `scale`: Optional tensor, of the same shape as the batch_shape+event_shape
- of the resulting distribution.
- `num_samples`: int=500, number of samples for the empirical quantiles.
+ `num_samples`: int, overwrite number of samples for the empirical quantiles.
**Returns**
`samples`: tensor, shape [B,H,`num_samples`].
@@ -2565,105 +2635,68 @@ def sample(self, distr_args, num_samples=None):
if num_samples is None:
num_samples = self.num_samples
- total_count, probs = distr_args
- B, H, K = total_count.size()
- Q = len(self.quantiles)
- assert total_count.shape == probs.shape
-
- # Sample K ~ Mult(weights)
- # shared across B, H
- # weights = torch.repeat_interleave(input=weights, repeats=H, dim=2)
-
- weights = (1 / K) * torch.ones_like(probs, device=probs.device)
-
- # Avoid loop, vectorize
- weights = weights.reshape(-1, K)
- total_count = total_count.flatten()
- probs = probs.flatten()
-
- # Vectorization trick to recover row_idx
- sample_idxs = torch.multinomial(
- input=weights, num_samples=num_samples, replacement=True
- )
- aux_col_idx = torch.unsqueeze(torch.arange(B * H, device=probs.device), -1) * K
-
- # To device
- sample_idxs = sample_idxs.to(probs.device)
-
- sample_idxs = sample_idxs + aux_col_idx
- sample_idxs = sample_idxs.flatten()
-
- sample_total_count = total_count[sample_idxs]
- sample_probs = probs[sample_idxs]
+ # Instantiate Scaled Decoupled Distribution
+ distr = self.get_distribution(distr_args=distr_args)
+ samples = distr.sample(sample_shape=(num_samples,))
+ samples = samples.permute(
+ 1, 2, 3, 0
+ ) # [samples, B, H, N] -> [B, H, N, samples]
- # Sample y ~ NBinomial(total_count, probs) independently
- dist = NegativeBinomial(total_count=sample_total_count, probs=sample_probs)
- samples = dist.sample(sample_shape=(1,)).to(probs.device)[0]
- samples = samples.view(B * H, num_samples)
- sample_mean = torch.mean(samples, dim=-1)
+ sample_mean = torch.mean(samples, dim=-1, keepdim=True)
# Compute quantiles
- quantiles_device = self.quantiles.to(probs.device)
- quants = torch.quantile(input=samples, q=quantiles_device, dim=1)
- quants = quants.permute((1, 0)) # Q, B*H
-
- # Final reshapes
- samples = samples.view(B, H, num_samples)
- sample_mean = sample_mean.view(B, H, 1)
- quants = quants.view(B, H, Q)
+ quantiles_device = self.quantiles.to(distr_args[0].device)
+ quants = torch.quantile(input=samples, q=quantiles_device, dim=-1)
+ quants = quants.permute(1, 2, 3, 0) # [Q, B, H, N] -> [B, H, N, Q]
return samples, sample_mean, quants
- def neglog_likelihood(
+ def update_quantile(self, q: Optional[List[float]] = None):
+ if q is not None:
+ self.quantiles = nn.Parameter(
+ torch.tensor(q, dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = (
+ [""]
+ + [f"_ql{q_i}" for q_i in q]
+ + self.return_params * self.param_names
+ )
+ self.has_predicted = True
+ elif q is None and self.has_predicted:
+ self.quantiles = nn.Parameter(
+ torch.tensor([0.5], dtype=torch.float32), requires_grad=False
+ )
+ self.output_names = ["", "-median"] + self.return_params * self.param_names
+
+ def __call__(
self,
y: torch.Tensor,
- distr_args: Tuple[torch.Tensor, torch.Tensor],
+ distr_args: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
):
+ """
+ Computes the negative log-likelihood objective function.
+ To estimate the following predictive distribution:
- if mask is None:
- mask = torch.ones_like(y)
-
- total_count, probs = distr_args
- B, H, K = total_count.size()
-
- weights = (1 / K) * torch.ones_like(probs, device=probs.device)
-
- y = y[:, :, None]
- mask = mask[:, :, None]
-
- log_unnormalized_prob = total_count * torch.log(1.0 - probs) + y * torch.log(
- probs
- )
- log_normalization = (
- -torch.lgamma(total_count + y)
- + torch.lgamma(1.0 + y)
- + torch.lgamma(total_count)
- )
- log_normalization[total_count + y == 0.0] = 0.0
- log = log_unnormalized_prob - log_normalization
-
- # log = torch.sum(log, dim=0, keepdim=True) # Joint within batch/group
- # log = torch.sum(log, dim=1, keepdim=True) # Joint within horizon
-
- # Numerical stability mixture and loglik
- log_max = torch.amax(log, dim=2, keepdim=True) # [1,1,K] (collapsed joints)
- lik = weights * torch.exp(log - log_max) # Take max
- loglik = torch.log(torch.sum(lik, dim=2, keepdim=True)) + log_max # Return max
+ $$\mathrm{P}(\mathbf{y}_{\\tau}\,|\,\\theta) \\quad \mathrm{and} \\quad -\log(\mathrm{P}(\mathbf{y}_{\\tau}\,|\,\\theta))$$
- loglik = loglik * mask # replace with mask
+ where $\\theta$ represents the distributions parameters. It aditionally
+ summarizes the objective signal using a weighted average using the `mask` tensor.
- loss = -torch.mean(loglik)
- return loss
+ **Parameters**
+ `y`: tensor, Actual values.
+ `distr_args`: Constructor arguments for the underlying Distribution type.
+ `mask`: tensor, Specifies date stamps per serie to consider in loss.
- def __call__(
- self,
- y: torch.Tensor,
- distr_args: Tuple[torch.Tensor, torch.Tensor],
- mask: Union[torch.Tensor, None] = None,
- ):
+ **Returns**
+ `loss`: scalar, weighted loss function against which backpropagation will be performed.
+ """
+ # Instantiate Scaled Decoupled Distribution
+ distr = self.get_distribution(distr_args=distr_args)
+ loss_values = -distr.log_prob(y)
+ loss_weights = mask
- return self.neglog_likelihood(y=y, distr_args=distr_args, mask=mask)
+ return weighted_average(loss_values, weights=loss_weights)
# %% ../../nbs/losses.pytorch.ipynb 97
class HuberLoss(BasePointLoss):
@@ -2702,8 +2735,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -2718,7 +2752,7 @@ def __call__(
return _weighted_mean(losses=losses, weights=weights)
# %% ../../nbs/losses.pytorch.ipynb 102
-class TukeyLoss(torch.nn.Module):
+class TukeyLoss(BasePointLoss):
""" Tukey Loss
The Tukey loss function, also known as Tukey's biweight function, is a
@@ -2758,10 +2792,14 @@ def __init__(self, c: float = 4.685, normalize: bool = True):
def domain_map(self, y_hat: torch.Tensor):
"""
- Univariate loss operates in dimension [B,T,H]/[B,H]
- This changes the network's output from [B,H,1]->[B,H]
+ Input:
+ Univariate: [B, H, 1]
+ Multivariate: [B, H, N]
+
+ Output: [B, H, N]
"""
- return y_hat.squeeze(-1)
+
+ return y_hat
def masked_mean(self, x, mask, dim):
x_nan = x.masked_fill(mask < 1, float("nan"))
@@ -2773,8 +2811,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -2844,8 +2883,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -2855,6 +2895,7 @@ def __call__(
**Returns:**
`huber_qloss`: tensor (single value).
"""
+
error = y_hat - y
zero_error = torch.zeros_like(error)
sq = torch.maximum(-error, zero_error)
@@ -2914,9 +2955,17 @@ def __init__(
def domain_map(self, y_hat: torch.Tensor):
"""
- Identity domain map [B,T,H,Q]/[B,H,Q]
+ Input:
+ Univariate: [B, H, 1 * Q]
+ Multivariate: [B, H, N * Q]
+
+ Output: [B, H, N, Q]
"""
- return y_hat
+ output = y_hat.reshape(
+ y_hat.shape[0], y_hat.shape[1], -1, self.outputsize_multiplier
+ )
+
+ return output
def _compute_weights(self, y, mask):
"""
@@ -2924,28 +2973,26 @@ def _compute_weights(self, y, mask):
Set horizon_weight to a ones[H] tensor if not set.
If set, check that it has the same length as the horizon in x.
"""
- if mask is None:
- mask = torch.ones_like(y, device=y.device)
- else:
- mask = mask.unsqueeze(1) # Add Q dimension.
if self.horizon_weight is None:
- self.horizon_weight = torch.ones(mask.shape[-1])
+ weights = torch.ones_like(mask)
else:
- assert mask.shape[-1] == len(
+ assert mask.shape[1] == len(
self.horizon_weight
), "horizon_weight must have same length as Y"
+ weights = self.horizon_weight.clone()
+ weights = weights[None, :, None, None].to(mask.device)
+ weights = torch.ones_like(mask, device=mask.device) * weights
- weights = self.horizon_weight.clone()
- weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)
return weights * mask
def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -2955,35 +3002,33 @@ def __call__(
**Returns:**
`hmqloss`: tensor (single value).
"""
+ y = y.unsqueeze(-1)
+
+ if mask is not None:
+ mask = mask.unsqueeze(-1)
+ else:
+ mask = torch.ones_like(y, device=y.device)
+
+ error = y_hat - y
- error = y_hat - y.unsqueeze(-1)
zero_error = torch.zeros_like(error)
sq = torch.maximum(-error, torch.zeros_like(error))
s1_q = torch.maximum(error, torch.zeros_like(error))
+
+ quantiles = self.quantiles[None, None, None, :]
losses = F.huber_loss(
- self.quantiles * sq, zero_error, reduction="none", delta=self.delta
+ quantiles * sq, zero_error, reduction="none", delta=self.delta
) + F.huber_loss(
- (1 - self.quantiles) * s1_q, zero_error, reduction="none", delta=self.delta
+ (1 - quantiles) * s1_q, zero_error, reduction="none", delta=self.delta
)
- losses = (1 / len(self.quantiles)) * losses
-
- if y_hat.ndim == 3: # BaseWindows
- losses = losses.swapaxes(
- -2, -1
- ) # [B,H,Q] -> [B,Q,H] (needed for horizon weighting, H at the end)
- elif y_hat.ndim == 4: # BaseRecurrent
- losses = losses.swapaxes(-2, -1)
- losses = losses.swapaxes(
- -2, -3
- ) # [B,seq_len,H,Q] -> [B,Q,seq_len,H] (needed for horizon weighting, H at the end)
+ losses = (1 / len(quantiles)) * losses
- weights = self._compute_weights(y=losses, mask=mask) # Use losses for extra dim
- # NOTE: Weights do not have Q dimension.
+ weights = self._compute_weights(y=losses, mask=mask)
return _weighted_mean(losses=losses, weights=weights)
# %% ../../nbs/losses.pytorch.ipynb 118
-class Accuracy(torch.nn.Module):
+class Accuracy(BasePointLoss):
"""Accuracy
Computes the accuracy between categorical `y` and `y_hat`.
@@ -2999,20 +3044,26 @@ def __init__(
):
super(Accuracy, self).__init__()
self.is_distribution_output = False
+ self.outputsize_multiplier = 1
def domain_map(self, y_hat: torch.Tensor):
"""
- Univariate loss operates in dimension [B,T,H]/[B,H]
- This changes the network's output from [B,H,1]->[B,H]
+ Input:
+ Univariate: [B, H, 1]
+ Multivariate: [B, H, N]
+
+ Output: [B, H, N]
"""
- return y_hat.squeeze(-1)
+
+ return y_hat
def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -3022,15 +3073,16 @@ def __call__(
**Returns:**
`accuracy`: tensor (single value).
"""
+
if mask is None:
mask = torch.ones_like(y_hat)
- measure = (y.unsqueeze(-1) == y_hat) * mask.unsqueeze(-1)
+ measure = (y == y_hat) * mask
accuracy = torch.mean(measure)
return accuracy
# %% ../../nbs/losses.pytorch.ipynb 122
-class sCRPS(torch.nn.Module):
+class sCRPS(BasePointLoss):
"""Scaled Continues Ranked Probability Score
Calculates a scaled variation of the CRPS, as proposed by Rangapuram (2021),
@@ -3070,8 +3122,9 @@ def __call__(
self,
y: torch.Tensor,
y_hat: torch.Tensor,
+ y_insample: torch.Tensor,
mask: Union[torch.Tensor, None] = None,
- ):
+ ) -> torch.Tensor:
"""
**Parameters:**
`y`: tensor, Actual values.
@@ -3081,7 +3134,7 @@ def __call__(
**Returns:**
`scrps`: tensor (single value).
"""
- mql = self.mql(y=y, y_hat=y_hat, mask=mask)
+ mql = self.mql(y=y, y_hat=y_hat, mask=mask, y_insample=y_insample)
norm = torch.sum(torch.abs(y))
unmean = torch.sum(mask)
scrps = 2 * mql * unmean / (norm + 1e-5)
diff --git a/neuralforecast/models/autoformer.py b/neuralforecast/models/autoformer.py
index 815e57bc2..d1a4c53b8 100644
--- a/neuralforecast/models/autoformer.py
+++ b/neuralforecast/models/autoformer.py
@@ -14,7 +14,7 @@
import torch.nn.functional as F
from ..common._modules import DataEmbedding, SeriesDecomp
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
@@ -394,7 +394,7 @@ def forward(self, x, cross, x_mask=None, cross_mask=None, trend=None):
return x, trend
# %% ../../nbs/models.autoformer.ipynb 10
-class Autoformer(BaseWindows):
+class Autoformer(BaseModel):
"""Autoformer
The Autoformer model tackles the challenge of finding reliable dependencies on intricate temporal patterns of long-horizon forecasting.
@@ -454,10 +454,13 @@ class Autoformer(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -631,13 +634,9 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
futr_exog = windows_batch["futr_exog"]
# Parse inputs
- insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]
if self.futr_exog_size > 0:
x_mark_enc = futr_exog[:, : self.input_size, :]
x_mark_dec = futr_exog[:, -(self.label_len + self.h) :, :]
@@ -670,5 +669,6 @@ def forward(self, windows_batch):
# final
dec_out = trend_part + seasonal_part
- forecast = self.loss.domain_map(dec_out[:, -self.h :])
+ forecast = dec_out[:, -self.h :]
+
return forecast
diff --git a/neuralforecast/models/bitcn.py b/neuralforecast/models/bitcn.py
index 53a775838..34566fd36 100644
--- a/neuralforecast/models/bitcn.py
+++ b/neuralforecast/models/bitcn.py
@@ -12,7 +12,7 @@
import numpy as np
from neuralforecast.losses.pytorch import MAE
-from neuralforecast.common._base_windows import BaseWindows
+from neuralforecast.common._base_model import BaseModel
# %% ../../nbs/models.bitcn.ipynb 8
class CustomConv1d(nn.Module):
@@ -84,7 +84,7 @@ def forward(self, x):
return (h_prev + h_next, out_prev + out_next)
# %% ../../nbs/models.bitcn.ipynb 10
-class BiTCN(BaseWindows):
+class BiTCN(BaseModel):
"""BiTCN
Bidirectional Temporal Convolutional Network (BiTCN) is a forecasting architecture based on two temporal convolutional networks (TCNs). The first network ('forward') encodes future covariates of the time series, whereas the second network ('backward') encodes past observations and covariates. This is a univariate model.
@@ -108,7 +108,7 @@ class BiTCN(BaseWindows):
`batch_size`: int=32, number of different series in each batch.
`valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
`windows_batch_size`: int=1024, number of windows to sample in each training batch, default uses all.
- `inference_windows_batch_size`: int=-1, number of windows to sample in each inference batch, -1 uses all.
+ `inference_windows_batch_size`: int=1024, number of windows to sample in each inference batch, -1 uses all.
`start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
@@ -129,10 +129,13 @@ class BiTCN(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -277,7 +280,7 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- x = windows_batch["insample_y"].unsqueeze(-1) # [B, L, 1]
+ x = windows_batch["insample_y"].contiguous() # [B, L, 1]
hist_exog = windows_batch["hist_exog"] # [B, L, X]
futr_exog = windows_batch["futr_exog"] # [B, L + h, F]
stat_exog = windows_batch["stat_exog"] # [B, S]
@@ -348,9 +351,6 @@ def forward(self, windows_batch):
# Output layer to create forecasts
x = x.permute(0, 2, 1) # [B, 3 * hidden_size, h] -> [B, h, 3 * hidden_size]
- x = self.output_lin(x) # [B, h, 3 * hidden_size] -> [B, h, n_outputs]
-
- # Map to output domain
- forecast = self.loss.domain_map(x)
+ forecast = self.output_lin(x) # [B, h, 3 * hidden_size] -> [B, h, n_outputs]
return forecast
diff --git a/neuralforecast/models/deepar.py b/neuralforecast/models/deepar.py
index 3d2a2fd94..8d3859a14 100644
--- a/neuralforecast/models/deepar.py
+++ b/neuralforecast/models/deepar.py
@@ -4,15 +4,13 @@
__all__ = ['Decoder', 'DeepAR']
# %% ../../nbs/models.deepar.ipynb 4
-import numpy as np
-
import torch
import torch.nn as nn
from typing import Optional
-from ..common._base_windows import BaseWindows
-from ..losses.pytorch import DistributionLoss, MQLoss
+from ..common._base_model import BaseModel
+from ..losses.pytorch import DistributionLoss, MAE
# %% ../../nbs/models.deepar.ipynb 7
class Decoder(nn.Module):
@@ -53,7 +51,7 @@ def forward(self, x):
return self.layers(x)
# %% ../../nbs/models.deepar.ipynb 8
-class DeepAR(BaseWindows):
+class DeepAR(BaseModel):
"""DeepAR
**Parameters:**
@@ -101,10 +99,11 @@ class DeepAR(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = False
EXOGENOUS_STAT = True
+ MULTIVARIATE = False
+ RECURRENT = True
def __init__(
self,
@@ -123,7 +122,7 @@ def __init__(
loss=DistributionLoss(
distribution="StudentT", level=[80, 90], return_params=False
),
- valid_loss=MQLoss(level=[80, 90]),
+ valid_loss=MAE(),
max_steps: int = 1000,
learning_rate: float = 1e-3,
num_lr_decays: int = 3,
@@ -150,19 +149,6 @@ def __init__(
if exclude_insample_y:
raise Exception("DeepAR has no possibility for excluding y.")
- if not loss.is_distribution_output:
- raise Exception("DeepAR only supports distributional outputs.")
-
- if str(type(valid_loss)) not in [
- ""
- ]:
- raise Exception("DeepAR only supports MQLoss as validation loss.")
-
- if loss.return_params:
- raise Exception(
- "DeepAR does not return distribution parameters due to Monte Carlo sampling."
- )
-
# Inherit BaseWindows class
super(DeepAR, self).__init__(
h=h,
@@ -196,8 +182,7 @@ def __init__(
**trainer_kwargs
)
- self.horizon_backup = self.h # Used because h=0 during training
- self.trajectory_samples = trajectory_samples
+ self.n_samples = trajectory_samples
# LSTM
self.encoder_n_layers = lstm_n_layers
@@ -208,6 +193,8 @@ def __init__(
input_encoder = 1 + self.futr_exog_size + self.stat_exog_size
# Instantiate model
+ self.rnn_state = None
+ self.maintain_state = False
self.hist_encoder = nn.LSTM(
input_size=input_encoder,
hidden_size=self.encoder_hidden_size,
@@ -224,206 +211,17 @@ def __init__(
hidden_layers=decoder_hidden_layers,
)
- # Override BaseWindows method
- def training_step(self, batch, batch_idx):
-
- # During training h=0
- self.h = 0
- y_idx = batch["y_idx"]
-
- # Create and normalize windows [Ws, L, C]
- windows = self._create_windows(batch, step="train")
- original_insample_y = windows["temporal"][
- :, :, y_idx
- ].clone() # windows: [B, L, Feature] -> [B, L]
- original_insample_y = original_insample_y[
- :, 1:
- ] # Remove first (shift in DeepAr, cell at t outputs t+1)
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- insample_y, insample_mask, _, _, _, futr_exog, stat_exog = self._parse_windows(
- batch, windows
- )
-
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L]
- insample_mask=insample_mask, # [Ws, L]
- futr_exog=futr_exog, # [Ws, L+H]
- hist_exog=None, # None
- stat_exog=stat_exog,
- y_idx=y_idx,
- ) # [Ws, 1]
-
- # Model Predictions
- output = self.train_forward(windows_batch)
-
- if self.loss.is_distribution_output:
- _, y_loc, y_scale = self._inv_normalization(
- y_hat=original_insample_y,
- temporal_cols=batch["temporal_cols"],
- y_idx=y_idx,
- )
- outsample_y = original_insample_y
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- mask = insample_mask[
- :, 1:
- ].clone() # Remove first (shift in DeepAr, cell at t outputs t+1)
- loss = self.loss(y=outsample_y, distr_args=distr_args, mask=mask)
- else:
- raise Exception("DeepAR only supports distributional outputs.")
-
- if torch.isnan(loss):
- print("Model Parameters", self.hparams)
- print("insample_y", torch.isnan(insample_y).sum())
- print("outsample_y", torch.isnan(outsample_y).sum())
- print("output", torch.isnan(output).sum())
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "train_loss",
- loss.item(),
- batch_size=outsample_y.size(0),
- prog_bar=True,
- on_epoch=True,
- )
- self.train_trajectories.append((self.global_step, loss.item()))
-
- self.h = self.horizon_backup # Restore horizon
- return loss
-
- def validation_step(self, batch, batch_idx):
-
- self.h == self.horizon_backup
-
- if self.val_size == 0:
- return np.nan
-
- # TODO: Hack to compute number of windows
- windows = self._create_windows(batch, step="val")
- n_windows = len(windows["temporal"])
- y_idx = batch["y_idx"]
-
- # Number of windows in batch
- windows_batch_size = self.inference_windows_batch_size
- if windows_batch_size < 0:
- windows_batch_size = n_windows
- n_batches = int(np.ceil(n_windows / windows_batch_size))
-
- valid_losses = []
- batch_sizes = []
- for i in range(n_batches):
- # Create and normalize windows [Ws, L+H, C]
- w_idxs = np.arange(
- i * windows_batch_size, min((i + 1) * windows_batch_size, n_windows)
- )
- windows = self._create_windows(batch, step="val", w_idxs=w_idxs)
- original_outsample_y = torch.clone(windows["temporal"][:, -self.h :, 0])
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- insample_y, insample_mask, _, outsample_mask, _, futr_exog, stat_exog = (
- self._parse_windows(batch, windows)
- )
- windows_batch = dict(
- insample_y=insample_y,
- insample_mask=insample_mask,
- futr_exog=futr_exog,
- hist_exog=None,
- stat_exog=stat_exog,
- temporal_cols=batch["temporal_cols"],
- y_idx=y_idx,
- )
-
- # Model Predictions
- output_batch = self(windows_batch)
- # Monte Carlo already returns y_hat with mean and quantiles
- output_batch = output_batch[:, :, 1:] # Remove mean
- valid_loss_batch = self.valid_loss(
- y=original_outsample_y, y_hat=output_batch, mask=outsample_mask
- )
- valid_losses.append(valid_loss_batch)
- batch_sizes.append(len(output_batch))
-
- valid_loss = torch.stack(valid_losses)
- batch_sizes = torch.tensor(batch_sizes, device=valid_loss.device)
- batch_size = torch.sum(batch_sizes)
- valid_loss = torch.sum(valid_loss * batch_sizes) / batch_size
-
- if torch.isnan(valid_loss):
- raise Exception("Loss is NaN, training stopped.")
-
- self.log(
- "valid_loss",
- valid_loss.item(),
- batch_size=batch_size,
- prog_bar=True,
- on_epoch=True,
- )
- self.validation_step_outputs.append(valid_loss)
- return valid_loss
-
- def predict_step(self, batch, batch_idx):
-
- self.h == self.horizon_backup
-
- # TODO: Hack to compute number of windows
- windows = self._create_windows(batch, step="predict")
- n_windows = len(windows["temporal"])
- y_idx = batch["y_idx"]
-
- # Number of windows in batch
- windows_batch_size = self.inference_windows_batch_size
- if windows_batch_size < 0:
- windows_batch_size = n_windows
- n_batches = int(np.ceil(n_windows / windows_batch_size))
-
- y_hats = []
- for i in range(n_batches):
- # Create and normalize windows [Ws, L+H, C]
- w_idxs = np.arange(
- i * windows_batch_size, min((i + 1) * windows_batch_size, n_windows)
- )
- windows = self._create_windows(batch, step="predict", w_idxs=w_idxs)
- windows = self._normalization(windows=windows, y_idx=y_idx)
-
- # Parse windows
- insample_y, insample_mask, _, _, _, futr_exog, stat_exog = (
- self._parse_windows(batch, windows)
- )
- windows_batch = dict(
- insample_y=insample_y, # [Ws, L]
- insample_mask=insample_mask, # [Ws, L]
- futr_exog=futr_exog, # [Ws, L+H]
- stat_exog=stat_exog,
- temporal_cols=batch["temporal_cols"],
- y_idx=y_idx,
- )
-
- # Model Predictions
- y_hat = self(windows_batch)
- # Monte Carlo already returns y_hat with mean and quantiles
- y_hats.append(y_hat)
- y_hat = torch.cat(y_hats, dim=0)
- return y_hat
-
- def train_forward(self, windows_batch):
+ def forward(self, windows_batch):
# Parse windows_batch
- encoder_input = windows_batch["insample_y"][:, :, None] # <- [B,T,1]
+ encoder_input = windows_batch["insample_y"] # <- [B, T, 1]
futr_exog = windows_batch["futr_exog"]
stat_exog = windows_batch["stat_exog"]
- # [B, input_size-1, X]
- encoder_input = encoder_input[
- :, :-1, :
- ] # Remove last (shift in DeepAr, cell at t outputs t+1)
_, input_size = encoder_input.shape[:2]
if self.futr_exog_size > 0:
- # Shift futr_exog (t predicts t+1, last output is outside insample_y)
- encoder_input = torch.cat((encoder_input, futr_exog[:, 1:, :]), dim=2)
+ encoder_input = torch.cat((encoder_input, futr_exog), dim=2)
+
if self.stat_exog_size > 0:
stat_exog = stat_exog.unsqueeze(1).repeat(
1, input_size, 1
@@ -431,114 +229,20 @@ def train_forward(self, windows_batch):
encoder_input = torch.cat((encoder_input, stat_exog), dim=2)
# RNN forward
- hidden_state, _ = self.hist_encoder(
- encoder_input
+ if self.maintain_state:
+ rnn_state = self.rnn_state
+ else:
+ rnn_state = None
+
+ hidden_state, rnn_state = self.hist_encoder(
+ encoder_input, rnn_state
) # [B, input_size-1, rnn_hidden_state]
+ if self.maintain_state:
+ self.rnn_state = rnn_state
+
# Decoder forward
output = self.decoder(hidden_state) # [B, input_size-1, output_size]
- output = self.loss.domain_map(output)
- return output
-
- def forward(self, windows_batch):
-
- # Parse windows_batch
- encoder_input = windows_batch["insample_y"][:, :, None] # <- [B,L,1]
- futr_exog = windows_batch["futr_exog"] # <- [B,L+H, n_f]
- stat_exog = windows_batch["stat_exog"]
- y_idx = windows_batch["y_idx"]
- # [B, seq_len, X]
- batch_size, input_size = encoder_input.shape[:2]
- if self.futr_exog_size > 0:
- futr_exog_input_window = futr_exog[
- :, 1 : input_size + 1, :
- ] # Align y_t with futr_exog_t+1
- encoder_input = torch.cat((encoder_input, futr_exog_input_window), dim=2)
- if self.stat_exog_size > 0:
- stat_exog_input_window = stat_exog.unsqueeze(1).repeat(
- 1, input_size, 1
- ) # [B, S] -> [B, input_size, S]
- encoder_input = torch.cat((encoder_input, stat_exog_input_window), dim=2)
-
- # Use input_size history to predict first h of the forecasting window
- _, h_c_tuple = self.hist_encoder(encoder_input)
- h_n = h_c_tuple[0] # [n_layers, B, lstm_hidden_state]
- c_n = h_c_tuple[1] # [n_layers, B, lstm_hidden_state]
-
- # Vectorizes trajectory samples in batch dimension [1]
- h_n = torch.repeat_interleave(
- h_n, self.trajectory_samples, 1
- ) # [n_layers, B*trajectory_samples, rnn_hidden_state]
- c_n = torch.repeat_interleave(
- c_n, self.trajectory_samples, 1
- ) # [n_layers, B*trajectory_samples, rnn_hidden_state]
-
- # Scales for inverse normalization
- y_scale = (
- self.scaler.x_scale[:, 0, [y_idx]].squeeze(-1).to(encoder_input.device)
- )
- y_loc = self.scaler.x_shift[:, 0, [y_idx]].squeeze(-1).to(encoder_input.device)
- y_scale = torch.repeat_interleave(y_scale, self.trajectory_samples, 0)
- y_loc = torch.repeat_interleave(y_loc, self.trajectory_samples, 0)
-
- # Recursive strategy prediction
- quantiles = self.loss.quantiles.to(encoder_input.device)
- y_hat = torch.zeros(
- batch_size, self.h, len(quantiles) + 1, device=encoder_input.device
- )
- for tau in range(self.h):
- # Decoder forward
- last_layer_h = h_n[-1] # [B*trajectory_samples, lstm_hidden_state]
- output = self.decoder(last_layer_h)
- output = self.loss.domain_map(output)
-
- # Inverse normalization
- distr_args = self.loss.scale_decouple(
- output=output, loc=y_loc, scale=y_scale
- )
- # Add horizon (1) dimension
- distr_args = list(distr_args)
- for i in range(len(distr_args)):
- distr_args[i] = distr_args[i].unsqueeze(-1)
- distr_args = tuple(distr_args)
- samples_tau, _, _ = self.loss.sample(distr_args=distr_args, num_samples=1)
- samples_tau = samples_tau.reshape(batch_size, self.trajectory_samples)
- sample_mean = torch.mean(samples_tau, dim=-1).to(encoder_input.device)
- quants = torch.quantile(input=samples_tau, q=quantiles, dim=-1).to(
- encoder_input.device
- )
- y_hat[:, tau, 0] = sample_mean
- y_hat[:, tau, 1:] = quants.permute((1, 0)) # [Q, B] -> [B, Q]
-
- # Stop if already in the last step (no need to predict next step)
- if tau + 1 == self.h:
- continue
- # Normalize to use as input
- encoder_input = self.scaler.scaler(
- samples_tau.flatten(), y_loc, y_scale
- ) # [B*n_samples]
- encoder_input = encoder_input[:, None, None] # [B*n_samples, 1, 1]
-
- # Update input
- if self.futr_exog_size > 0:
- futr_exog_tau = futr_exog[:, [input_size + tau + 1], :] # [B, 1, n_f]
- futr_exog_tau = torch.repeat_interleave(
- futr_exog_tau, self.trajectory_samples, 0
- ) # [B*n_samples, 1, n_f]
- encoder_input = torch.cat(
- (encoder_input, futr_exog_tau), dim=2
- ) # [B*n_samples, 1, 1+n_f]
- if self.stat_exog_size > 0:
- stat_exog_tau = torch.repeat_interleave(
- stat_exog, self.trajectory_samples, 0
- ) # [B*n_samples, n_s]
- encoder_input = torch.cat(
- (encoder_input, stat_exog_tau[:, None, :]), dim=2
- ) # [B*n_samples, 1, 1+n_f+n_s]
-
- _, h_c_tuple = self.hist_encoder(encoder_input, (h_n, c_n))
- h_n = h_c_tuple[0] # [n_layers, B, rnn_hidden_state]
- c_n = h_c_tuple[1] # [n_layers, B, rnn_hidden_state]
-
- return y_hat
+ # Return only horizon part
+ return output[:, -self.h :]
diff --git a/neuralforecast/models/deepnpts.py b/neuralforecast/models/deepnpts.py
index f958e71be..b8b168f5a 100644
--- a/neuralforecast/models/deepnpts.py
+++ b/neuralforecast/models/deepnpts.py
@@ -11,11 +11,11 @@
from typing import Optional
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
# %% ../../nbs/models.deepnpts.ipynb 6
-class DeepNPTS(BaseWindows):
+class DeepNPTS(BaseModel):
"""DeepNPTS
Deep Non-Parametric Time Series Forecaster (`DeepNPTS`) is a baseline model for time-series forecasting. This model generates predictions by (weighted) sampling from the empirical distribution according to a learnable strategy. The strategy is learned by exploiting the information across multiple related time series.
@@ -62,10 +62,13 @@ class DeepNPTS(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -107,12 +110,12 @@ def __init__(
if exclude_insample_y:
raise Exception("DeepNPTS has no possibility for excluding y.")
- if not isinstance(loss, losses.BasePointLoss):
+ if loss.outputsize_multiplier > 1:
raise Exception(
"DeepNPTS only supports point loss functions (MAE, MSE, etc) as loss function."
)
- if not isinstance(valid_loss, losses.BasePointLoss):
+ if valid_loss is not None and not isinstance(valid_loss, losses.BasePointLoss):
raise Exception(
"DeepNPTS only supports point loss functions (MAE, MSE, etc) as valid loss function."
)
@@ -175,13 +178,13 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- x = windows_batch["insample_y"].unsqueeze(-1) # [B, L, 1]
+ x = windows_batch["insample_y"] # [B, L, 1]
hist_exog = windows_batch["hist_exog"] # [B, L, X]
futr_exog = windows_batch["futr_exog"] # [B, L + h, F]
stat_exog = windows_batch["stat_exog"] # [B, S]
batch_size, seq_len = x.shape[:2] # B = batch_size, L = seq_len
- insample_y = windows_batch["insample_y"].unsqueeze(-1)
+ insample_y = windows_batch["insample_y"]
# Concatenate x_t with future exogenous of input
if self.futr_exog_size > 0:
@@ -223,8 +226,6 @@ def forward(self, windows_batch):
x = (
F.softmax(weights, dim=1) * insample_y
) # [B, L, h] * [B, L, 1] = [B, L, h]
- output = torch.sum(x, dim=1).unsqueeze(-1) # [B, L, h] -> [B, h, 1]
-
- forecast = self.loss.domain_map(output) # [B, h, 1] -> [B, h, 1]
+ forecast = torch.sum(x, dim=1).unsqueeze(-1) # [B, L, h] -> [B, h, 1]
return forecast
diff --git a/neuralforecast/models/dilated_rnn.py b/neuralforecast/models/dilated_rnn.py
index d56cc5f08..cbc8bb484 100644
--- a/neuralforecast/models/dilated_rnn.py
+++ b/neuralforecast/models/dilated_rnn.py
@@ -10,7 +10,7 @@
import torch.nn as nn
from ..losses.pytorch import MAE
-from ..common._base_recurrent import BaseRecurrent
+from ..common._base_model import BaseModel
from ..common._modules import MLP
# %% ../../nbs/models.dilated_rnn.ipynb 7
@@ -256,8 +256,8 @@ def _split_outputs(self, dilated_outputs, rate):
for i in range(rate)
]
- interleaved = torch.stack((blocks)).transpose(1, 0).contiguous()
- interleaved = interleaved.view(
+ interleaved = torch.stack((blocks)).transpose(1, 0)
+ interleaved = interleaved.reshape(
dilated_outputs.size(0) * rate, batchsize, dilated_outputs.size(2)
)
return interleaved
@@ -286,7 +286,7 @@ def _prepare_inputs(self, inputs, rate):
return dilated_inputs
# %% ../../nbs/models.dilated_rnn.ipynb 12
-class DilatedRNN(BaseRecurrent):
+class DilatedRNN(BaseModel):
"""DilatedRNN
**Parameters:**
@@ -326,25 +326,29 @@ class DilatedRNN(BaseRecurrent):
"""
# Class attributes
- SAMPLING_TYPE = "recurrent"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
h: int,
- input_size: int = -1,
+ input_size: int,
inference_input_size: int = -1,
cell_type: str = "LSTM",
dilations: List[List[int]] = [[1, 2], [4, 8]],
- encoder_hidden_size: int = 200,
+ encoder_hidden_size: int = 128,
context_size: int = 10,
- decoder_hidden_size: int = 200,
+ decoder_hidden_size: int = 128,
decoder_layers: int = 2,
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
loss=MAE(),
valid_loss=None,
max_steps: int = 1000,
@@ -354,6 +358,9 @@ def __init__(
val_check_steps: int = 100,
batch_size=32,
valid_batch_size: Optional[int] = None,
+ windows_batch_size=128,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "robust",
random_seed: int = 1,
@@ -369,7 +376,10 @@ def __init__(
super(DilatedRNN, self).__init__(
h=h,
input_size=input_size,
- inference_input_size=inference_input_size,
+ futr_exog_list=futr_exog_list,
+ hist_exog_list=hist_exog_list,
+ stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -379,13 +389,14 @@ def __init__(
val_check_steps=val_check_steps,
batch_size=batch_size,
valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
+ step_size=step_size,
scaler_type=scaler_type,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
+ random_seed=random_seed,
num_workers_loader=num_workers_loader,
drop_last_loader=drop_last_loader,
- random_seed=random_seed,
optimizer=optimizer,
optimizer_kwargs=optimizer_kwargs,
lr_scheduler=lr_scheduler,
@@ -407,14 +418,14 @@ def __init__(
self.decoder_layers = decoder_layers
# RNN input size (1 for target variable y)
- input_encoder = 1 + self.hist_exog_size + self.stat_exog_size
+ input_encoder = (
+ 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size
+ )
# Instantiate model
layers = []
for grp_num in range(len(self.dilations)):
- if grp_num == 0:
- input_encoder = 1 + self.hist_exog_size + self.stat_exog_size
- else:
+ if grp_num > 0:
input_encoder = self.encoder_hidden_size
layer = DRNN(
input_encoder,
@@ -428,14 +439,11 @@ def __init__(
self.rnn_stack = nn.Sequential(*layers)
# Context adapter
- self.context_adapter = nn.Linear(
- in_features=self.encoder_hidden_size + self.futr_exog_size * h,
- out_features=self.context_size * h,
- )
+ self.context_adapter = nn.Linear(in_features=self.input_size, out_features=h)
# Decoder MLP
self.mlp_decoder = MLP(
- in_features=self.context_size + self.futr_exog_size,
+ in_features=self.encoder_hidden_size + self.futr_exog_size,
out_features=self.loss.outputsize_multiplier,
hidden_size=self.decoder_hidden_size,
num_layers=self.decoder_layers,
@@ -446,26 +454,30 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- encoder_input = windows_batch["insample_y"] # [B, seq_len, 1]
- futr_exog = windows_batch["futr_exog"]
- hist_exog = windows_batch["hist_exog"]
- stat_exog = windows_batch["stat_exog"]
+ encoder_input = windows_batch["insample_y"] # [B, L, 1]
+ futr_exog = windows_batch["futr_exog"] # [B, L + h, F]
+ hist_exog = windows_batch["hist_exog"] # [B, L, X]
+ stat_exog = windows_batch["stat_exog"] # [B, S]
# Concatenate y, historic and static inputs
- # [B, C, seq_len, 1] -> [B, seq_len, C]
- # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]
batch_size, seq_len = encoder_input.shape[:2]
if self.hist_exog_size > 0:
- hist_exog = hist_exog.permute(0, 2, 1, 3).squeeze(
- -1
- ) # [B, X, seq_len, 1] -> [B, seq_len, X]
- encoder_input = torch.cat((encoder_input, hist_exog), dim=2)
+ encoder_input = torch.cat(
+ (encoder_input, hist_exog), dim=2
+ ) # [B, L, 1] + [B, L, X] -> [B, L, 1 + X]
if self.stat_exog_size > 0:
stat_exog = stat_exog.unsqueeze(1).repeat(
1, seq_len, 1
- ) # [B, S] -> [B, seq_len, S]
- encoder_input = torch.cat((encoder_input, stat_exog), dim=2)
+ ) # [B, S] -> [B, L, S]
+ encoder_input = torch.cat(
+ (encoder_input, stat_exog), dim=2
+ ) # [B, L, 1 + X] + [B, L, S] -> [B, L, 1 + X + S]
+
+ if self.futr_exog_size > 0:
+ encoder_input = torch.cat(
+ (encoder_input, futr_exog[:, :seq_len]), dim=2
+ ) # [B, L, 1 + X + S] + [B, L, F] -> [B, L, 1 + X + S + F]
# DilatedRNN forward
for layer_num in range(len(self.rnn_stack)):
@@ -475,24 +487,21 @@ def forward(self, windows_batch):
output += residual
encoder_input = output
- if self.futr_exog_size > 0:
- futr_exog = futr_exog.permute(0, 2, 3, 1)[
- :, :, 1:, :
- ] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]
- encoder_input = torch.cat(
- (encoder_input, futr_exog.reshape(batch_size, seq_len, -1)), dim=2
- )
-
# Context adapter
- context = self.context_adapter(encoder_input)
- context = context.reshape(batch_size, seq_len, self.h, self.context_size)
+ output = output.permute(0, 2, 1) # [B, L, C] -> [B, C, L]
+ context = self.context_adapter(output) # [B, C, L] -> [B, C, h]
# Residual connection with futr_exog
if self.futr_exog_size > 0:
- context = torch.cat((context, futr_exog), dim=-1)
+ futr_exog_futr = futr_exog[:, seq_len:].permute(
+ 0, 2, 1
+ ) # [B, h, F] -> [B, F, h]
+ context = torch.cat(
+ (context, futr_exog_futr), dim=1
+ ) # [B, C, h] + [B, F, h] = [B, C + F, h]
# Final forecast
- output = self.mlp_decoder(context)
- output = self.loss.domain_map(output)
+ context = context.permute(0, 2, 1) # [B, C + F, h] -> [B, h, C + F]
+ output = self.mlp_decoder(context) # [B, h, C + F] -> [B, h, n_output]
return output
diff --git a/neuralforecast/models/dlinear.py b/neuralforecast/models/dlinear.py
index 17965c869..79a8d75de 100644
--- a/neuralforecast/models/dlinear.py
+++ b/neuralforecast/models/dlinear.py
@@ -9,7 +9,7 @@
import torch
import torch.nn as nn
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
@@ -48,7 +48,7 @@ def forward(self, x):
return res, moving_mean
# %% ../../nbs/models.dlinear.ipynb 10
-class DLinear(BaseWindows):
+class DLinear(BaseModel):
"""DLinear
*Parameters:*
@@ -87,10 +87,13 @@ class DLinear(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -178,11 +181,7 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
- # futr_exog = windows_batch['futr_exog']
+ insample_y = windows_batch["insample_y"].squeeze(-1)
# Parse inputs
batch_size = len(insample_y)
@@ -194,5 +193,4 @@ def forward(self, windows_batch):
# Final
forecast = trend_part + seasonal_part
forecast = forecast.reshape(batch_size, self.h, self.loss.outputsize_multiplier)
- forecast = self.loss.domain_map(forecast)
return forecast
diff --git a/neuralforecast/models/fedformer.py b/neuralforecast/models/fedformer.py
index 89e2fe3ef..a91bae3a8 100644
--- a/neuralforecast/models/fedformer.py
+++ b/neuralforecast/models/fedformer.py
@@ -4,7 +4,7 @@
__all__ = ['LayerNorm', 'AutoCorrelationLayer', 'EncoderLayer', 'Encoder', 'DecoderLayer', 'Decoder', 'get_frequency_modes',
'FourierBlock', 'FourierCrossAttention', 'FEDformer']
-# %% ../../nbs/models.fedformer.ipynb 5
+# %% ../../nbs/models.fedformer.ipynb 6
import numpy as np
from typing import Optional
@@ -14,11 +14,11 @@
from ..common._modules import DataEmbedding
from ..common._modules import SeriesDecomp
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
-# %% ../../nbs/models.fedformer.ipynb 7
+# %% ../../nbs/models.fedformer.ipynb 8
class LayerNorm(nn.Module):
"""
Special designed layernorm for the seasonal part
@@ -66,7 +66,7 @@ def forward(self, queries, keys, values, attn_mask):
return self.out_projection(out), attn
-# %% ../../nbs/models.fedformer.ipynb 8
+# %% ../../nbs/models.fedformer.ipynb 9
class EncoderLayer(nn.Module):
"""
FEDformer encoder layer with the progressive decomposition architecture
@@ -234,7 +234,7 @@ def forward(self, x, cross, x_mask=None, cross_mask=None, trend=None):
x = self.projection(x)
return x, trend
-# %% ../../nbs/models.fedformer.ipynb 9
+# %% ../../nbs/models.fedformer.ipynb 10
def get_frequency_modes(seq_len, modes=64, mode_select_method="random"):
"""
Get modes on frequency domain:
@@ -390,8 +390,8 @@ def forward(self, q, k, v, mask):
)
return (out, None)
-# %% ../../nbs/models.fedformer.ipynb 11
-class FEDformer(BaseWindows):
+# %% ../../nbs/models.fedformer.ipynb 12
+class FEDformer(BaseModel):
"""FEDformer
The FEDformer model tackles the challenge of finding reliable dependencies on intricate temporal patterns of long-horizon forecasting.
@@ -450,10 +450,13 @@ class FEDformer(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -626,13 +629,9 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
futr_exog = windows_batch["futr_exog"]
# Parse inputs
- insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]
if self.futr_exog_size > 0:
x_mark_enc = futr_exog[:, : self.input_size, :]
x_mark_dec = futr_exog[:, -(self.label_len + self.h) :, :]
@@ -666,6 +665,6 @@ def forward(self, windows_batch):
)
# final
dec_out = trend_part + seasonal_part
+ forecast = dec_out[:, -self.h :]
- forecast = self.loss.domain_map(dec_out[:, -self.h :])
return forecast
diff --git a/neuralforecast/models/gru.py b/neuralforecast/models/gru.py
index 9a6d92325..f8061500e 100644
--- a/neuralforecast/models/gru.py
+++ b/neuralforecast/models/gru.py
@@ -9,13 +9,14 @@
import torch
import torch.nn as nn
+import warnings
from ..losses.pytorch import MAE
-from ..common._base_recurrent import BaseRecurrent
+from ..common._base_model import BaseModel
from ..common._modules import MLP
-# %% ../../nbs/models.gru.ipynb 8
-class GRU(BaseRecurrent):
+# %% ../../nbs/models.gru.ipynb 7
+class GRU(BaseModel):
"""GRU
Multi Layer Recurrent Network with Gated Units (GRU), and
@@ -23,7 +24,7 @@ class GRU(BaseRecurrent):
using ADAM stochastic gradient descent. The network accepts static, historic
and future exogenous data, flattens the inputs.
- **Parameters:**
+ **Parameters:**
`h`: int, forecast horizon.
`input_size`: int, maximum sequence length for truncated train backpropagation. Default -1 uses all history.
`inference_input_size`: int, maximum sequence length for truncated inference. Default -1 uses all history.
@@ -32,7 +33,7 @@ class GRU(BaseRecurrent):
`encoder_activation`: Optional[str]=None, Deprecated. Activation function in GRU is frozen in PyTorch.
`encoder_bias`: bool=True, whether or not to use biases b_ih, b_hh within GRU units.
`encoder_dropout`: float=0., dropout regularization applied to GRU outputs.
- `context_size`: int=10, size of context vector for each timestamp on the forecasting window.
+ `context_size`: deprecated.
`decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
`decoder_layers`: int=2, number of layers for the MLP decoder.
`futr_exog_list`: str list, future exogenous columns.
@@ -61,10 +62,13 @@ class GRU(BaseRecurrent):
"""
# Class attributes
- SAMPLING_TYPE = "recurrent"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ True # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -76,12 +80,14 @@ def __init__(
encoder_activation: Optional[str] = None,
encoder_bias: bool = True,
encoder_dropout: float = 0.0,
- context_size: int = 10,
- decoder_hidden_size: int = 200,
+ context_size: Optional[int] = None,
+ decoder_hidden_size: int = 128,
decoder_layers: int = 2,
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
+ recurrent=False,
loss=MAE(),
valid_loss=None,
max_steps: int = 1000,
@@ -91,6 +97,10 @@ def __init__(
val_check_steps: int = 100,
batch_size=32,
valid_batch_size: Optional[int] = None,
+ windows_batch_size=128,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
+ step_size: int = 1,
scaler_type: str = "robust",
random_seed=1,
num_workers_loader=0,
@@ -102,10 +112,16 @@ def __init__(
dataloader_kwargs=None,
**trainer_kwargs
):
+
+ self.RECURRENT = recurrent
+
super(GRU, self).__init__(
h=h,
input_size=input_size,
- inference_input_size=inference_input_size,
+ futr_exog_list=futr_exog_list,
+ hist_exog_list=hist_exog_list,
+ stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -115,13 +131,14 @@ def __init__(
val_check_steps=val_check_steps,
batch_size=batch_size,
valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
+ step_size=step_size,
scaler_type=scaler_type,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
+ random_seed=random_seed,
num_workers_loader=num_workers_loader,
drop_last_loader=drop_last_loader,
- random_seed=random_seed,
optimizer=optimizer,
optimizer_kwargs=optimizer_kwargs,
lr_scheduler=lr_scheduler,
@@ -145,16 +162,23 @@ def __init__(
self.encoder_dropout = encoder_dropout
# Context adapter
- self.context_size = context_size
+ if context_size is not None:
+ warnings.warn(
+ "context_size is deprecated and will be removed in future versions."
+ )
# MLP decoder
self.decoder_hidden_size = decoder_hidden_size
self.decoder_layers = decoder_layers
# RNN input size (1 for target variable y)
- input_encoder = 1 + self.hist_exog_size + self.stat_exog_size
+ input_encoder = (
+ 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size
+ )
# Instantiate model
+ self.rnn_state = None
+ self.maintain_state = False
self.hist_encoder = nn.GRU(
input_size=input_encoder,
hidden_size=self.encoder_hidden_size,
@@ -164,69 +188,80 @@ def __init__(
batch_first=True,
)
- # Context adapter
- self.context_adapter = nn.Linear(
- in_features=self.encoder_hidden_size + self.futr_exog_size * h,
- out_features=self.context_size * h,
- )
-
# Decoder MLP
- self.mlp_decoder = MLP(
- in_features=self.context_size + self.futr_exog_size,
- out_features=self.loss.outputsize_multiplier,
- hidden_size=self.decoder_hidden_size,
- num_layers=self.decoder_layers,
- activation="ReLU",
- dropout=0.0,
- )
+ if self.RECURRENT:
+ self.proj = nn.Linear(
+ self.encoder_hidden_size, self.loss.outputsize_multiplier
+ )
+ else:
+ self.mlp_decoder = MLP(
+ in_features=self.encoder_hidden_size + self.futr_exog_size,
+ out_features=self.loss.outputsize_multiplier,
+ hidden_size=self.decoder_hidden_size,
+ num_layers=self.decoder_layers,
+ activation="ReLU",
+ dropout=0.0,
+ )
def forward(self, windows_batch):
# Parse windows_batch
encoder_input = windows_batch["insample_y"] # [B, seq_len, 1]
- futr_exog = windows_batch["futr_exog"]
- hist_exog = windows_batch["hist_exog"]
- stat_exog = windows_batch["stat_exog"]
+ futr_exog = windows_batch["futr_exog"] # [B, seq_len, F]
+ hist_exog = windows_batch["hist_exog"] # [B, seq_len, X]
+ stat_exog = windows_batch["stat_exog"] # [B, S]
# Concatenate y, historic and static inputs
- # [B, C, seq_len, 1] -> [B, seq_len, C]
- # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]
batch_size, seq_len = encoder_input.shape[:2]
if self.hist_exog_size > 0:
- hist_exog = hist_exog.permute(0, 2, 1, 3).squeeze(
- -1
- ) # [B, X, seq_len, 1] -> [B, seq_len, X]
- encoder_input = torch.cat((encoder_input, hist_exog), dim=2)
+ encoder_input = torch.cat(
+ (encoder_input, hist_exog), dim=2
+ ) # [B, seq_len, 1] + [B, seq_len, X] -> [B, seq_len, 1 + X]
if self.stat_exog_size > 0:
+ # print(encoder_input.shape)
stat_exog = stat_exog.unsqueeze(1).repeat(
1, seq_len, 1
) # [B, S] -> [B, seq_len, S]
- encoder_input = torch.cat((encoder_input, stat_exog), dim=2)
-
- # RNN forward
- hidden_state, _ = self.hist_encoder(
- encoder_input
- ) # [B, seq_len, rnn_hidden_state]
+ encoder_input = torch.cat(
+ (encoder_input, stat_exog), dim=2
+ ) # [B, seq_len, 1 + X] + [B, seq_len, S] -> [B, seq_len, 1 + X + S]
if self.futr_exog_size > 0:
- futr_exog = futr_exog.permute(0, 2, 3, 1)[
- :, :, 1:, :
- ] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]
- hidden_state = torch.cat(
- (hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2
- )
+ encoder_input = torch.cat(
+ (encoder_input, futr_exog[:, :seq_len]), dim=2
+ ) # [B, seq_len, 1 + X + S] + [B, seq_len, F] -> [B, seq_len, 1 + X + S + F]
- # Context adapter
- context = self.context_adapter(hidden_state)
- context = context.reshape(batch_size, seq_len, self.h, self.context_size)
+ if self.RECURRENT:
+ if self.maintain_state:
+ rnn_state = self.rnn_state
+ else:
+ rnn_state = None
- # Residual connection with futr_exog
- if self.futr_exog_size > 0:
- context = torch.cat((context, futr_exog), dim=-1)
+ output, rnn_state = self.hist_encoder(
+ encoder_input, rnn_state
+ ) # [B, seq_len, rnn_hidden_state]
+ output = self.proj(
+ output
+ ) # [B, seq_len, rnn_hidden_state] -> [B, seq_len, n_output]
+ if self.maintain_state:
+ self.rnn_state = rnn_state
+ else:
+ hidden_state, _ = self.hist_encoder(
+ encoder_input, None
+ ) # [B, seq_len, rnn_hidden_state]
+ hidden_state = hidden_state[
+ :, -self.h :
+ ] # [B, seq_len, rnn_hidden_state] -> [B, h, rnn_hidden_state]
+
+ if self.futr_exog_size > 0:
+ futr_exog_futr = futr_exog[:, -self.h :] # [B, h, F]
+ hidden_state = torch.cat(
+ (hidden_state, futr_exog_futr), dim=-1
+ ) # [B, h, rnn_hidden_state] + [B, h, F] -> [B, h, rnn_hidden_state + F]
- # Final forecast
- output = self.mlp_decoder(context)
- output = self.loss.domain_map(output)
+ output = self.mlp_decoder(
+ hidden_state
+ ) # [B, h, rnn_hidden_state + F] -> [B, seq_len, n_output]
- return output
+ return output[:, -self.h :]
diff --git a/neuralforecast/models/informer.py b/neuralforecast/models/informer.py
index 8b115cebd..f775e31e7 100644
--- a/neuralforecast/models/informer.py
+++ b/neuralforecast/models/informer.py
@@ -19,7 +19,7 @@
DataEmbedding,
AttentionLayer,
)
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
@@ -179,7 +179,7 @@ def forward(self, queries, keys, values, attn_mask):
return context.contiguous(), attn
# %% ../../nbs/models.informer.ipynb 11
-class Informer(BaseWindows):
+class Informer(BaseModel):
"""Informer
The Informer model tackles the vanilla Transformer computational complexity challenges for long-horizon forecasting.
@@ -238,10 +238,11 @@ class Informer(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False
+ RECURRENT = False
def __init__(
self,
@@ -414,14 +415,8 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
-
futr_exog = windows_batch["futr_exog"]
- insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]
-
if self.futr_exog_size > 0:
x_mark_enc = futr_exog[:, : self.input_size, :]
x_mark_dec = futr_exog[:, -(self.label_len + self.h) :, :]
@@ -438,5 +433,5 @@ def forward(self, windows_batch):
dec_out = self.dec_embedding(x_dec, x_mark_dec)
dec_out = self.decoder(dec_out, enc_out, x_mask=None, cross_mask=None)
- forecast = self.loss.domain_map(dec_out[:, -self.h :])
+ forecast = dec_out[:, -self.h :]
return forecast
diff --git a/neuralforecast/models/itransformer.py b/neuralforecast/models/itransformer.py
index 9e577a71d..fb67ee20f 100644
--- a/neuralforecast/models/itransformer.py
+++ b/neuralforecast/models/itransformer.py
@@ -11,9 +11,9 @@
import numpy as np
from math import sqrt
-
+from typing import Optional
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
from neuralforecast.common._modules import (
TransEncoder,
@@ -102,7 +102,7 @@ def forward(self, x, x_mark):
return self.dropout(x)
# %% ../../nbs/models.itransformer.ipynb 13
-class iTransformer(BaseMultivariate):
+class iTransformer(BaseModel):
"""iTransformer
**Parameters:**
@@ -128,6 +128,10 @@ class iTransformer(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=128, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=128, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -146,10 +150,11 @@ class iTransformer(BaseMultivariate):
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = True
+ RECURRENT = False
def __init__(
self,
@@ -159,6 +164,7 @@ def __init__(
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
hidden_size: int = 512,
n_heads: int = 8,
e_layers: int = 2,
@@ -175,6 +181,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=128,
+ inference_windows_batch_size=128,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -195,6 +205,7 @@ def __init__(
stat_exog_list=None,
futr_exog_list=None,
hist_exog_list=None,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -203,6 +214,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
random_seed=random_seed,
@@ -253,7 +268,9 @@ def __init__(
norm_layer=torch.nn.LayerNorm(self.hidden_size),
)
- self.projector = nn.Linear(self.hidden_size, h, bias=True)
+ self.projector = nn.Linear(
+ self.hidden_size, h * self.loss.outputsize_multiplier, bias=True
+ )
def forecast(self, x_enc):
if self.use_norm:
@@ -287,8 +304,16 @@ def forecast(self, x_enc):
if self.use_norm:
# De-Normalization from Non-stationary Transformer
- dec_out = dec_out * (stdev[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))
- dec_out = dec_out + (means[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))
+ dec_out = dec_out * (
+ stdev[:, 0, :]
+ .unsqueeze(1)
+ .repeat(1, self.h * self.loss.outputsize_multiplier, 1)
+ )
+ dec_out = dec_out + (
+ means[:, 0, :]
+ .unsqueeze(1)
+ .repeat(1, self.h * self.loss.outputsize_multiplier, 1)
+ )
return dec_out
@@ -296,11 +321,6 @@ def forward(self, windows_batch):
insample_y = windows_batch["insample_y"]
y_pred = self.forecast(insample_y)
- y_pred = y_pred[:, -self.h :, :]
- y_pred = self.loss.domain_map(y_pred)
+ y_pred = y_pred.reshape(insample_y.shape[0], self.h, -1)
- # domain_map might have squeezed the last dimension in case n_series == 1
- if y_pred.ndim == 2:
- return y_pred.unsqueeze(-1)
- else:
- return y_pred
+ return y_pred
diff --git a/neuralforecast/models/kan.py b/neuralforecast/models/kan.py
index 29d7b1d00..924f53136 100644
--- a/neuralforecast/models/kan.py
+++ b/neuralforecast/models/kan.py
@@ -12,7 +12,7 @@
import torch.nn.functional as F
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.kan.ipynb 8
class KANLinear(torch.nn.Module):
@@ -240,7 +240,7 @@ def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0)
)
# %% ../../nbs/models.kan.ipynb 9
-class KAN(BaseWindows):
+class KAN(BaseModel):
"""KAN
Simple Kolmogorov-Arnold Network (KAN).
@@ -294,10 +294,13 @@ class KAN(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -436,7 +439,7 @@ def regularization_loss(self, regularize_activation=1.0, regularize_entropy=1.0)
def forward(self, windows_batch, update_grid=False):
- insample_y = windows_batch["insample_y"]
+ insample_y = windows_batch["insample_y"].squeeze(-1)
futr_exog = windows_batch["futr_exog"]
hist_exog = windows_batch["hist_exog"]
stat_exog = windows_batch["stat_exog"]
@@ -466,5 +469,4 @@ def forward(self, windows_batch, update_grid=False):
y_pred = layer(y_pred)
y_pred = y_pred.reshape(batch_size, self.h, self.loss.outputsize_multiplier)
- y_pred = self.loss.domain_map(y_pred)
return y_pred
diff --git a/neuralforecast/models/lstm.py b/neuralforecast/models/lstm.py
index e89db3628..8ad263cc1 100644
--- a/neuralforecast/models/lstm.py
+++ b/neuralforecast/models/lstm.py
@@ -8,13 +8,14 @@
import torch
import torch.nn as nn
+import warnings
from ..losses.pytorch import MAE
-from ..common._base_recurrent import BaseRecurrent
+from ..common._base_model import BaseModel
from ..common._modules import MLP
# %% ../../nbs/models.lstm.ipynb 7
-class LSTM(BaseRecurrent):
+class LSTM(BaseModel):
"""LSTM
LSTM encoder, with MLP decoder.
@@ -30,7 +31,7 @@ class LSTM(BaseRecurrent):
`encoder_hidden_size`: int=200, units for the LSTM's hidden state size.
`encoder_bias`: bool=True, whether or not to use biases b_ih, b_hh within LSTM units.
`encoder_dropout`: float=0., dropout regularization applied to LSTM outputs.
- `context_size`: int=10, size of context vector for each timestamp on the forecasting window.
+ `context_size`: deprecated.
`decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
`decoder_layers`: int=2, number of layers for the MLP decoder.
`futr_exog_list`: str list, future exogenous columns.
@@ -59,26 +60,30 @@ class LSTM(BaseRecurrent):
"""
# Class attributes
- SAMPLING_TYPE = "recurrent"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ True # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
h: int,
- input_size: int = -1,
- inference_input_size: int = -1,
+ input_size: int,
encoder_n_layers: int = 2,
- encoder_hidden_size: int = 200,
+ encoder_hidden_size: int = 128,
encoder_bias: bool = True,
encoder_dropout: float = 0.0,
- context_size: int = 10,
- decoder_hidden_size: int = 200,
+ context_size: Optional[int] = None,
+ decoder_hidden_size: int = 128,
decoder_layers: int = 2,
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
+ recurrent=False,
loss=MAE(),
valid_loss=None,
max_steps: int = 1000,
@@ -88,6 +93,10 @@ def __init__(
val_check_steps: int = 100,
batch_size=32,
valid_batch_size: Optional[int] = None,
+ windows_batch_size=128,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
+ step_size: int = 1,
scaler_type: str = "robust",
random_seed=1,
num_workers_loader=0,
@@ -99,10 +108,16 @@ def __init__(
dataloader_kwargs=None,
**trainer_kwargs
):
+
+ self.RECURRENT = recurrent
+
super(LSTM, self).__init__(
h=h,
input_size=input_size,
- inference_input_size=inference_input_size,
+ futr_exog_list=futr_exog_list,
+ hist_exog_list=hist_exog_list,
+ stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -112,13 +127,14 @@ def __init__(
val_check_steps=val_check_steps,
batch_size=batch_size,
valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
+ step_size=step_size,
scaler_type=scaler_type,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
+ random_seed=random_seed,
num_workers_loader=num_workers_loader,
drop_last_loader=drop_last_loader,
- random_seed=random_seed,
optimizer=optimizer,
optimizer_kwargs=optimizer_kwargs,
lr_scheduler=lr_scheduler,
@@ -134,16 +150,23 @@ def __init__(
self.encoder_dropout = encoder_dropout
# Context adapter
- self.context_size = context_size
+ if context_size is not None:
+ warnings.warn(
+ "context_size is deprecated and will be removed in future versions."
+ )
# MLP decoder
self.decoder_hidden_size = decoder_hidden_size
self.decoder_layers = decoder_layers
# LSTM input size (1 for target variable y)
- input_encoder = 1 + self.hist_exog_size + self.stat_exog_size
+ input_encoder = (
+ 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size
+ )
# Instantiate model
+ self.rnn_state = None
+ self.maintain_state = False
self.hist_encoder = nn.LSTM(
input_size=input_encoder,
hidden_size=self.encoder_hidden_size,
@@ -151,71 +174,76 @@ def __init__(
bias=self.encoder_bias,
dropout=self.encoder_dropout,
batch_first=True,
- )
-
- # Context adapter
- self.context_adapter = nn.Linear(
- in_features=self.encoder_hidden_size + self.futr_exog_size * h,
- out_features=self.context_size * h,
+ proj_size=self.loss.outputsize_multiplier if self.RECURRENT else 0,
)
# Decoder MLP
- self.mlp_decoder = MLP(
- in_features=self.context_size + self.futr_exog_size,
- out_features=self.loss.outputsize_multiplier,
- hidden_size=self.decoder_hidden_size,
- num_layers=self.decoder_layers,
- activation="ReLU",
- dropout=0.0,
- )
+ if not self.RECURRENT:
+ self.mlp_decoder = MLP(
+ in_features=self.encoder_hidden_size + self.futr_exog_size,
+ out_features=self.loss.outputsize_multiplier,
+ hidden_size=self.decoder_hidden_size,
+ num_layers=self.decoder_layers,
+ activation="ReLU",
+ dropout=0.0,
+ )
def forward(self, windows_batch):
# Parse windows_batch
encoder_input = windows_batch["insample_y"] # [B, seq_len, 1]
- futr_exog = windows_batch["futr_exog"]
- hist_exog = windows_batch["hist_exog"]
- stat_exog = windows_batch["stat_exog"]
+ futr_exog = windows_batch["futr_exog"] # [B, seq_len, F]
+ hist_exog = windows_batch["hist_exog"] # [B, seq_len, X]
+ stat_exog = windows_batch["stat_exog"] # [B, S]
# Concatenate y, historic and static inputs
- # [B, C, seq_len, 1] -> [B, seq_len, C]
- # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]
batch_size, seq_len = encoder_input.shape[:2]
if self.hist_exog_size > 0:
- hist_exog = hist_exog.permute(0, 2, 1, 3).squeeze(
- -1
- ) # [B, X, seq_len, 1] -> [B, seq_len, X]
- encoder_input = torch.cat((encoder_input, hist_exog), dim=2)
+ encoder_input = torch.cat(
+ (encoder_input, hist_exog), dim=2
+ ) # [B, seq_len, 1] + [B, seq_len, X] -> [B, seq_len, 1 + X]
if self.stat_exog_size > 0:
+ # print(encoder_input.shape)
stat_exog = stat_exog.unsqueeze(1).repeat(
1, seq_len, 1
) # [B, S] -> [B, seq_len, S]
- encoder_input = torch.cat((encoder_input, stat_exog), dim=2)
-
- # RNN forward
- hidden_state, _ = self.hist_encoder(
- encoder_input
- ) # [B, seq_len, rnn_hidden_state]
-
- if self.futr_exog_size > 0:
- futr_exog = futr_exog.permute(0, 2, 3, 1)[
- :, :, 1:, :
- ] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]
- hidden_state = torch.cat(
- (hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2
- )
-
- # Context adapter
- context = self.context_adapter(hidden_state)
- context = context.reshape(batch_size, seq_len, self.h, self.context_size)
+ encoder_input = torch.cat(
+ (encoder_input, stat_exog), dim=2
+ ) # [B, seq_len, 1 + X] + [B, seq_len, S] -> [B, seq_len, 1 + X + S]
- # Residual connection with futr_exog
if self.futr_exog_size > 0:
- context = torch.cat((context, futr_exog), dim=-1)
-
- # Final forecast
- output = self.mlp_decoder(context)
- output = self.loss.domain_map(output)
-
- return output
+ encoder_input = torch.cat(
+ (encoder_input, futr_exog[:, :seq_len]), dim=2
+ ) # [B, seq_len, 1 + X + S] + [B, seq_len, F] -> [B, seq_len, 1 + X + S + F]
+
+ if self.RECURRENT:
+ if self.maintain_state:
+ rnn_state = self.rnn_state
+ else:
+ rnn_state = None
+
+ output, rnn_state = self.hist_encoder(
+ encoder_input, rnn_state
+ ) # [B, seq_len, n_output]
+ if self.maintain_state:
+ self.rnn_state = rnn_state
+ else:
+ hidden_state, _ = self.hist_encoder(
+ encoder_input, None
+ ) # [B, seq_len, rnn_hidden_state]
+ hidden_state = hidden_state[
+ :, -self.h :
+ ] # [B, seq_len, rnn_hidden_state] -> [B, h, rnn_hidden_state]
+
+ if self.futr_exog_size > 0:
+ futr_exog_futr = futr_exog[:, -self.h :] # [B, h, F]
+ hidden_state = torch.cat(
+ (hidden_state, futr_exog_futr), dim=-1
+ ) # [B, h, rnn_hidden_state] + [B, h, F] -> [B, h, rnn_hidden_state + F]
+
+ output = self.mlp_decoder(
+ hidden_state
+ ) # [B, h, rnn_hidden_state + F] -> [B, seq_len, n_output]
+
+ return output[:, -self.h :]
diff --git a/neuralforecast/models/mlp.py b/neuralforecast/models/mlp.py
index 0794ac7c3..63e2a5409 100644
--- a/neuralforecast/models/mlp.py
+++ b/neuralforecast/models/mlp.py
@@ -10,10 +10,10 @@
import torch.nn as nn
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.mlp.ipynb 6
-class MLP(BaseWindows):
+class MLP(BaseModel):
"""MLP
Simple Multi Layer Perceptron architecture (MLP).
@@ -58,10 +58,13 @@ class MLP(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -158,7 +161,7 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
+ insample_y = windows_batch["insample_y"].squeeze(-1)
futr_exog = windows_batch["futr_exog"]
hist_exog = windows_batch["hist_exog"]
stat_exog = windows_batch["stat_exog"]
@@ -187,5 +190,4 @@ def forward(self, windows_batch):
y_pred = self.out(y_pred)
y_pred = y_pred.reshape(batch_size, self.h, self.loss.outputsize_multiplier)
- y_pred = self.loss.domain_map(y_pred)
return y_pred
diff --git a/neuralforecast/models/mlpmultivariate.py b/neuralforecast/models/mlpmultivariate.py
index 7554bb44d..89124c47f 100644
--- a/neuralforecast/models/mlpmultivariate.py
+++ b/neuralforecast/models/mlpmultivariate.py
@@ -7,11 +7,12 @@
import torch
import torch.nn as nn
+from typing import Optional
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
# %% ../../nbs/models.mlpmultivariate.ipynb 6
-class MLPMultivariate(BaseMultivariate):
+class MLPMultivariate(BaseModel):
"""MLPMultivariate
Simple Multi Layer Perceptron architecture (MLP) for multivariate forecasting.
@@ -37,6 +38,10 @@ class MLPMultivariate(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -52,10 +57,13 @@ class MLPMultivariate(BaseMultivariate):
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -65,6 +73,7 @@ def __init__(
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
num_layers=2,
hidden_size=1024,
loss=MAE(),
@@ -75,6 +84,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=256,
+ inference_windows_batch_size=256,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -96,6 +109,7 @@ def __init__(
futr_exog_list=futr_exog_list,
hist_exog_list=hist_exog_list,
stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -104,6 +118,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
num_workers_loader=num_workers_loader,
@@ -173,12 +191,6 @@ def forward(self, windows_batch):
x = torch.relu(layer(x))
x = self.out(x)
- x = x.reshape(batch_size, self.h, -1)
- forecast = self.loss.domain_map(x)
+ forecast = x.reshape(batch_size, self.h, -1)
- # domain_map might have squeezed the last dimension in case n_series == 1
- # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.
- if forecast.ndim == 2:
- return forecast.unsqueeze(-1)
- else:
- return forecast
+ return forecast
diff --git a/neuralforecast/models/nbeats.py b/neuralforecast/models/nbeats.py
index 02280fb79..8cf178583 100644
--- a/neuralforecast/models/nbeats.py
+++ b/neuralforecast/models/nbeats.py
@@ -11,7 +11,7 @@
import torch.nn as nn
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.nbeats.ipynb 7
class IdentityBasis(nn.Module):
@@ -189,7 +189,7 @@ def forward(self, insample_y: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]
return backcast, forecast
# %% ../../nbs/models.nbeats.ipynb 9
-class NBEATS(BaseWindows):
+class NBEATS(BaseModel):
"""NBEATS
The Neural Basis Expansion Analysis for Time Series (NBEATS), is a simple and yet
@@ -241,10 +241,13 @@ class NBEATS(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -406,8 +409,8 @@ def create_stack(
def forward(self, windows_batch):
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
- insample_mask = windows_batch["insample_mask"]
+ insample_y = windows_batch["insample_y"].squeeze(-1)
+ insample_mask = windows_batch["insample_mask"].squeeze(-1)
# NBEATS' forward
residuals = insample_y.flip(dims=(-1,)) # backcast init
@@ -423,9 +426,6 @@ def forward(self, windows_batch):
if self.decompose_forecast:
block_forecasts.append(block_forecast)
- # Adapting output's domain
- forecast = self.loss.domain_map(forecast)
-
if self.decompose_forecast:
# (n_batch, n_blocks, h, out_features)
block_forecasts = torch.stack(block_forecasts)
diff --git a/neuralforecast/models/nbeatsx.py b/neuralforecast/models/nbeatsx.py
index 811392a66..403ca53c0 100644
--- a/neuralforecast/models/nbeatsx.py
+++ b/neuralforecast/models/nbeatsx.py
@@ -11,7 +11,7 @@
import torch.nn as nn
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.nbeatsx.ipynb 8
class IdentityBasis(nn.Module):
@@ -274,7 +274,7 @@ def forward(
return backcast, forecast
# %% ../../nbs/models.nbeatsx.ipynb 10
-class NBEATSx(BaseWindows):
+class NBEATSx(BaseModel):
"""NBEATSx
The Neural Basis Expansion Analysis with Exogenous variables (NBEATSx) is a simple
@@ -328,10 +328,13 @@ class NBEATSx(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -513,8 +516,8 @@ def create_stack(
def forward(self, windows_batch):
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
- insample_mask = windows_batch["insample_mask"]
+ insample_y = windows_batch["insample_y"].squeeze(-1)
+ insample_mask = windows_batch["insample_mask"].squeeze(-1)
futr_exog = windows_batch["futr_exog"]
hist_exog = windows_batch["hist_exog"]
stat_exog = windows_batch["stat_exog"]
@@ -538,9 +541,6 @@ def forward(self, windows_batch):
if self.decompose_forecast:
block_forecasts.append(block_forecast)
- # Adapting output's domain
- forecast = self.loss.domain_map(forecast)
-
if self.decompose_forecast:
# (n_batch, n_blocks, h)
block_forecasts = torch.stack(block_forecasts)
diff --git a/neuralforecast/models/nhits.py b/neuralforecast/models/nhits.py
index ce5caeaaa..20833a4d7 100644
--- a/neuralforecast/models/nhits.py
+++ b/neuralforecast/models/nhits.py
@@ -12,7 +12,7 @@
import torch.nn.functional as F
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.nhits.ipynb 8
class _IdentityBasis(nn.Module):
@@ -184,7 +184,7 @@ def forward(
return backcast, forecast
# %% ../../nbs/models.nhits.ipynb 10
-class NHITS(BaseWindows):
+class NHITS(BaseModel):
"""NHITS
The Neural Hierarchical Interpolation for Time Series (NHITS), is an MLP-based deep
@@ -240,10 +240,13 @@ class NHITS(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -398,8 +401,8 @@ def create_stack(
def forward(self, windows_batch):
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
- insample_mask = windows_batch["insample_mask"]
+ insample_y = windows_batch["insample_y"].squeeze(-1).contiguous()
+ insample_mask = windows_batch["insample_mask"].squeeze(-1).contiguous()
futr_exog = windows_batch["futr_exog"]
hist_exog = windows_batch["hist_exog"]
stat_exog = windows_batch["stat_exog"]
@@ -423,9 +426,6 @@ def forward(self, windows_batch):
if self.decompose_forecast:
block_forecasts.append(block_forecast)
- # Adapting output's domain
- forecast = self.loss.domain_map(forecast)
-
if self.decompose_forecast:
# (n_batch, n_blocks, h, output_size)
block_forecasts = torch.stack(block_forecasts)
diff --git a/neuralforecast/models/nlinear.py b/neuralforecast/models/nlinear.py
index 4909ddbd3..e0db42273 100644
--- a/neuralforecast/models/nlinear.py
+++ b/neuralforecast/models/nlinear.py
@@ -8,12 +8,12 @@
import torch.nn as nn
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
# %% ../../nbs/models.nlinear.ipynb 7
-class NLinear(BaseWindows):
+class NLinear(BaseModel):
"""NLinear
*Parameters:*
@@ -51,10 +51,13 @@ class NLinear(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -132,11 +135,7 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
- # futr_exog = windows_batch['futr_exog']
+ insample_y = windows_batch["insample_y"].squeeze(-1)
# Parse inputs
batch_size = len(insample_y)
@@ -148,5 +147,4 @@ def forward(self, windows_batch):
# Final
forecast = self.linear(norm_insample_y) + last_value
forecast = forecast.reshape(batch_size, self.h, self.loss.outputsize_multiplier)
- forecast = self.loss.domain_map(forecast)
return forecast
diff --git a/neuralforecast/models/patchtst.py b/neuralforecast/models/patchtst.py
index 0b2029fd4..9472b8e86 100644
--- a/neuralforecast/models/patchtst.py
+++ b/neuralforecast/models/patchtst.py
@@ -14,7 +14,7 @@
import torch.nn as nn
import torch.nn.functional as F
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..common._modules import RevIN
from ..losses.pytorch import MAE
@@ -785,7 +785,7 @@ def forward(
return output, attn_weights
# %% ../../nbs/models.patchtst.ipynb 15
-class PatchTST(BaseWindows):
+class PatchTST(BaseModel):
"""PatchTST
The PatchTST model is an efficient Transformer-based model for multivariate time series forecasting.
@@ -848,10 +848,13 @@ class PatchTST(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -995,20 +998,10 @@ def __init__(
def forward(self, windows_batch): # x: [batch, input_size]
# Parse windows_batch
- insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
- # futr_exog = windows_batch['futr_exog']
-
- # Add dimension for channel
- x = insample_y.unsqueeze(-1) # [Ws,L,1]
+ x = windows_batch["insample_y"]
x = x.permute(0, 2, 1) # x: [Batch, 1, input_size]
x = self.model(x)
- x = x.reshape(x.shape[0], self.h, -1) # x: [Batch, h, c_out]
-
- # Domain map
- forecast = self.loss.domain_map(x)
+ forecast = x.reshape(x.shape[0], self.h, -1) # x: [Batch, h, c_out]
return forecast
diff --git a/neuralforecast/models/rmok.py b/neuralforecast/models/rmok.py
index 35db80aca..f91d589a5 100644
--- a/neuralforecast/models/rmok.py
+++ b/neuralforecast/models/rmok.py
@@ -11,8 +11,9 @@
import torch.nn.functional as F
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
-from ..common._modules import RevIN
+from ..common._base_model import BaseModel
+from ..common._modules import RevINMultivariate
+from typing import Optional
# %% ../../nbs/models.rmok.ipynb 8
class WaveKANLayer(nn.Module):
@@ -256,9 +257,11 @@ def forward(self, x):
return y
# %% ../../nbs/models.rmok.ipynb 14
-class RMoK(BaseMultivariate):
+class RMoK(BaseModel):
"""Reversible Mixture of KAN
- **Parameters**
+
+
+ **Parameters:**
`h`: int, Forecast horizon.
`input_size`: int, autorregresive inputs size, y=[1,2,3,4] input_size=2 -> y_[t-2:t]=[1,2].
`n_series`: int, number of time-series.
@@ -278,6 +281,10 @@ class RMoK(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=1024, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=1024, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -291,21 +298,24 @@ class RMoK(BaseMultivariate):
`dataloader_kwargs`: dict, optional, list of parameters passed into the PyTorch Lightning dataloader by the `TimeSeriesDataLoader`.
`**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
- Reference
- [Xiao Han, Xinfeng Zhang, Yiling Wu, Zhenduo Zhang, Zhe Wu."KAN4TSF: Are KAN and KAN-based models Effective for Time Series Forecasting?"](https://arxiv.org/abs/2408.11306)
+ **References**
+ - [Xiao Han, Xinfeng Zhang, Yiling Wu, Zhenduo Zhang, Zhe Wu."KAN4TSF: Are KAN and KAN-based models Effective for Time Series Forecasting?". arXiv.](https://arxiv.org/abs/2408.11306)
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
h,
input_size,
- n_series,
+ n_series: int,
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
@@ -322,6 +332,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=1024,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -350,6 +364,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
random_seed=random_seed,
@@ -376,25 +394,34 @@ def __init__(
self.experts = nn.ModuleList(
[
TaylorKANLayer(
- self.input_size, self.h, order=self.taylor_order, addbias=True
+ self.input_size,
+ self.h * self.loss.outputsize_multiplier,
+ order=self.taylor_order,
+ addbias=True,
+ ),
+ JacobiKANLayer(
+ self.input_size,
+ self.h * self.loss.outputsize_multiplier,
+ degree=self.jacobi_degree,
),
- JacobiKANLayer(self.input_size, self.h, degree=self.jacobi_degree),
WaveKANLayer(
- self.input_size, self.h, wavelet_type=self.wavelet_function
+ self.input_size,
+ self.h * self.loss.outputsize_multiplier,
+ wavelet_type=self.wavelet_function,
),
- nn.Linear(self.input_size, self.h),
+ nn.Linear(self.input_size, self.h * self.loss.outputsize_multiplier),
]
)
self.num_experts = len(self.experts)
self.gate = nn.Linear(self.input_size, self.num_experts)
self.softmax = nn.Softmax(dim=-1)
- self.rev = RevIN(self.n_series, affine=self.revin_affine)
+ self.rev = RevINMultivariate(self.n_series, affine=self.revin_affine)
def forward(self, windows_batch):
insample_y = windows_batch["insample_y"]
B, L, N = insample_y.shape
- x = self.rev(insample_y, "norm") if self.rev else insample_y
+ x = self.rev(insample_y, "norm")
x = self.dropout(x).transpose(1, 2).reshape(B * N, L)
score = F.softmax(self.gate(x), dim=-1)
@@ -403,15 +430,11 @@ def forward(self, windows_batch):
)
y_pred = (
- torch.einsum("BLE,BE->BL", expert_outputs, score)
- .reshape(B, N, -1)
+ torch.einsum("BLE, BE -> BL", expert_outputs, score)
+ .reshape(B, N, self.h * self.loss.outputsize_multiplier)
.permute(0, 2, 1)
)
y_pred = self.rev(y_pred, "denorm")
- y_pred = self.loss.domain_map(y_pred)
+ y_pred = y_pred.reshape(B, self.h, -1)
- # domain_map might have squeezed the last dimension in case n_series == 1
- if y_pred.ndim == 2:
- return y_pred.unsqueeze(-1)
- else:
- return y_pred
+ return y_pred
diff --git a/neuralforecast/models/rnn.py b/neuralforecast/models/rnn.py
index f5d60f42a..30c53c15c 100644
--- a/neuralforecast/models/rnn.py
+++ b/neuralforecast/models/rnn.py
@@ -8,13 +8,14 @@
import torch
import torch.nn as nn
+import warnings
from ..losses.pytorch import MAE
-from ..common._base_recurrent import BaseRecurrent
+from ..common._base_model import BaseModel
from ..common._modules import MLP
# %% ../../nbs/models.rnn.ipynb 7
-class RNN(BaseRecurrent):
+class RNN(BaseModel):
"""RNN
Multi Layer Elman RNN (RNN), with MLP decoder.
@@ -31,7 +32,7 @@ class RNN(BaseRecurrent):
`encoder_activation`: str=`tanh`, type of RNN activation from `tanh` or `relu`.
`encoder_bias`: bool=True, whether or not to use biases b_ih, b_hh within RNN units.
`encoder_dropout`: float=0., dropout regularization applied to RNN outputs.
- `context_size`: int=10, size of context vector for each timestamp on the forecasting window.
+ `context_size`: deprecated.
`decoder_hidden_size`: int=200, size of hidden layer for the MLP decoder.
`decoder_layers`: int=2, number of layers for the MLP decoder.
`futr_exog_list`: str list, future exogenous columns.
@@ -61,10 +62,13 @@ class RNN(BaseRecurrent):
"""
# Class attributes
- SAMPLING_TYPE = "recurrent"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ True # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -72,16 +76,18 @@ def __init__(
input_size: int = -1,
inference_input_size: int = -1,
encoder_n_layers: int = 2,
- encoder_hidden_size: int = 200,
+ encoder_hidden_size: int = 128,
encoder_activation: str = "tanh",
encoder_bias: bool = True,
encoder_dropout: float = 0.0,
- context_size: int = 10,
- decoder_hidden_size: int = 200,
+ context_size: Optional[int] = None,
+ decoder_hidden_size: int = 128,
decoder_layers: int = 2,
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
+ recurrent=False,
loss=MAE(),
valid_loss=None,
max_steps: int = 1000,
@@ -91,6 +97,10 @@ def __init__(
val_check_steps: int = 100,
batch_size=32,
valid_batch_size: Optional[int] = None,
+ windows_batch_size=128,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
+ step_size: int = 1,
scaler_type: str = "robust",
random_seed=1,
num_workers_loader=0,
@@ -102,10 +112,16 @@ def __init__(
dataloader_kwargs=None,
**trainer_kwargs
):
+
+ self.RECURRENT = recurrent
+
super(RNN, self).__init__(
h=h,
input_size=input_size,
- inference_input_size=inference_input_size,
+ futr_exog_list=futr_exog_list,
+ hist_exog_list=hist_exog_list,
+ stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -115,13 +131,14 @@ def __init__(
val_check_steps=val_check_steps,
batch_size=batch_size,
valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
+ step_size=step_size,
scaler_type=scaler_type,
- futr_exog_list=futr_exog_list,
- hist_exog_list=hist_exog_list,
- stat_exog_list=stat_exog_list,
+ random_seed=random_seed,
num_workers_loader=num_workers_loader,
drop_last_loader=drop_last_loader,
- random_seed=random_seed,
optimizer=optimizer,
optimizer_kwargs=optimizer_kwargs,
lr_scheduler=lr_scheduler,
@@ -137,6 +154,12 @@ def __init__(
self.encoder_bias = encoder_bias
self.encoder_dropout = encoder_dropout
+ # Context adapter
+ if context_size is not None:
+ warnings.warn(
+ "context_size is deprecated and will be removed in future versions."
+ )
+
# Context adapter
self.context_size = context_size
@@ -145,82 +168,96 @@ def __init__(
self.decoder_layers = decoder_layers
# RNN input size (1 for target variable y)
- input_encoder = 1 + self.hist_exog_size + self.stat_exog_size
+ input_encoder = (
+ 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size
+ )
# Instantiate model
+ self.rnn_state = None
+ self.maintain_state = False
self.hist_encoder = nn.RNN(
input_size=input_encoder,
hidden_size=self.encoder_hidden_size,
num_layers=self.encoder_n_layers,
- nonlinearity=self.encoder_activation,
bias=self.encoder_bias,
dropout=self.encoder_dropout,
batch_first=True,
)
- # Context adapter
- self.context_adapter = nn.Linear(
- in_features=self.encoder_hidden_size + self.futr_exog_size * h,
- out_features=self.context_size * h,
- )
-
# Decoder MLP
- self.mlp_decoder = MLP(
- in_features=self.context_size + self.futr_exog_size,
- out_features=self.loss.outputsize_multiplier,
- hidden_size=self.decoder_hidden_size,
- num_layers=self.decoder_layers,
- activation="ReLU",
- dropout=0.0,
- )
+ if self.RECURRENT:
+ self.proj = nn.Linear(
+ self.encoder_hidden_size, self.loss.outputsize_multiplier
+ )
+ else:
+ self.mlp_decoder = MLP(
+ in_features=self.encoder_hidden_size + self.futr_exog_size,
+ out_features=self.loss.outputsize_multiplier,
+ hidden_size=self.decoder_hidden_size,
+ num_layers=self.decoder_layers,
+ activation="ReLU",
+ dropout=0.0,
+ )
def forward(self, windows_batch):
# Parse windows_batch
encoder_input = windows_batch["insample_y"] # [B, seq_len, 1]
- futr_exog = windows_batch["futr_exog"]
- hist_exog = windows_batch["hist_exog"]
- stat_exog = windows_batch["stat_exog"]
+ futr_exog = windows_batch["futr_exog"] # [B, seq_len, F]
+ hist_exog = windows_batch["hist_exog"] # [B, seq_len, X]
+ stat_exog = windows_batch["stat_exog"] # [B, S]
# Concatenate y, historic and static inputs
- # [B, C, seq_len, 1] -> [B, seq_len, C]
- # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]
batch_size, seq_len = encoder_input.shape[:2]
if self.hist_exog_size > 0:
- hist_exog = hist_exog.permute(0, 2, 1, 3).squeeze(
- -1
- ) # [B, X, seq_len, 1] -> [B, seq_len, X]
- encoder_input = torch.cat((encoder_input, hist_exog), dim=2)
+ encoder_input = torch.cat(
+ (encoder_input, hist_exog), dim=2
+ ) # [B, seq_len, 1] + [B, seq_len, X] -> [B, seq_len, 1 + X]
if self.stat_exog_size > 0:
+ # print(encoder_input.shape)
stat_exog = stat_exog.unsqueeze(1).repeat(
1, seq_len, 1
) # [B, S] -> [B, seq_len, S]
- encoder_input = torch.cat((encoder_input, stat_exog), dim=2)
-
- # RNN forward
- hidden_state, _ = self.hist_encoder(
- encoder_input
- ) # [B, seq_len, rnn_hidden_state]
+ encoder_input = torch.cat(
+ (encoder_input, stat_exog), dim=2
+ ) # [B, seq_len, 1 + X] + [B, seq_len, S] -> [B, seq_len, 1 + X + S]
if self.futr_exog_size > 0:
- futr_exog = futr_exog.permute(0, 2, 3, 1)[
- :, :, 1:, :
- ] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]
- hidden_state = torch.cat(
- (hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2
- )
+ encoder_input = torch.cat(
+ (encoder_input, futr_exog[:, :seq_len]), dim=2
+ ) # [B, seq_len, 1 + X + S] + [B, seq_len, F] -> [B, seq_len, 1 + X + S + F]
- # Context adapter
- context = self.context_adapter(hidden_state)
- context = context.reshape(batch_size, seq_len, self.h, self.context_size)
+ if self.RECURRENT:
+ if self.maintain_state:
+ rnn_state = self.rnn_state
+ else:
+ rnn_state = None
- # Residual connection with futr_exog
- if self.futr_exog_size > 0:
- context = torch.cat((context, futr_exog), dim=-1)
+ output, rnn_state = self.hist_encoder(
+ encoder_input, rnn_state
+ ) # [B, seq_len, rnn_hidden_state]
+ output = self.proj(
+ output
+ ) # [B, seq_len, rnn_hidden_state] -> [B, seq_len, n_output]
+ if self.maintain_state:
+ self.rnn_state = rnn_state
+ else:
+ hidden_state, _ = self.hist_encoder(
+ encoder_input, None
+ ) # [B, seq_len, rnn_hidden_state]
+ hidden_state = hidden_state[
+ :, -self.h :
+ ] # [B, seq_len, rnn_hidden_state] -> [B, h, rnn_hidden_state]
+
+ if self.futr_exog_size > 0:
+ futr_exog_futr = futr_exog[:, -self.h :] # [B, h, F]
+ hidden_state = torch.cat(
+ (hidden_state, futr_exog_futr), dim=-1
+ ) # [B, h, rnn_hidden_state] + [B, h, F] -> [B, h, rnn_hidden_state + F]
- # Final forecast
- output = self.mlp_decoder(context)
- output = self.loss.domain_map(output)
+ output = self.mlp_decoder(
+ hidden_state
+ ) # [B, h, rnn_hidden_state + F] -> [B, seq_len, n_output]
- return output
+ return output[:, -self.h :]
diff --git a/neuralforecast/models/softs.py b/neuralforecast/models/softs.py
index cb425200a..c7a1a2a7c 100644
--- a/neuralforecast/models/softs.py
+++ b/neuralforecast/models/softs.py
@@ -8,8 +8,9 @@
import torch.nn as nn
import torch.nn.functional as F
+from typing import Optional
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
from ..common._modules import TransEncoder, TransEncoderLayer
# %% ../../nbs/models.softs.ipynb 6
@@ -57,7 +58,7 @@ def forward(self, input, *args, **kwargs):
# stochastic pooling
if self.training:
- ratio = F.softmax(combined_mean, dim=1)
+ ratio = F.softmax(torch.nan_to_num(combined_mean), dim=1)
ratio = ratio.permute(0, 2, 1)
ratio = ratio.reshape(-1, channels)
indices = torch.multinomial(ratio, 1)
@@ -79,7 +80,7 @@ def forward(self, input, *args, **kwargs):
return output, None
# %% ../../nbs/models.softs.ipynb 10
-class SOFTS(BaseMultivariate):
+class SOFTS(BaseModel):
"""SOFTS
**Parameters:**
@@ -103,6 +104,10 @@ class SOFTS(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -121,10 +126,11 @@ class SOFTS(BaseMultivariate):
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = True
+ RECURRENT = False
def __init__(
self,
@@ -134,6 +140,7 @@ def __init__(
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
hidden_size: int = 512,
d_core: int = 512,
e_layers: int = 2,
@@ -148,6 +155,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=256,
+ inference_windows_batch_size=256,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -168,6 +179,7 @@ def __init__(
stat_exog_list=None,
futr_exog_list=None,
hist_exog_list=None,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -176,6 +188,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
random_seed=random_seed,
@@ -211,7 +227,9 @@ def __init__(
]
)
- self.projection = nn.Linear(hidden_size, self.h, bias=True)
+ self.projection = nn.Linear(
+ hidden_size, self.h * self.loss.outputsize_multiplier, bias=True
+ )
def forecast(self, x_enc):
# Normalization from Non-stationary Transformer
@@ -230,19 +248,22 @@ def forecast(self, x_enc):
# De-Normalization from Non-stationary Transformer
if self.use_norm:
- dec_out = dec_out * (stdev[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))
- dec_out = dec_out + (means[:, 0, :].unsqueeze(1).repeat(1, self.h, 1))
+ dec_out = dec_out * (
+ stdev[:, 0, :]
+ .unsqueeze(1)
+ .repeat(1, self.h * self.loss.outputsize_multiplier, 1)
+ )
+ dec_out = dec_out + (
+ means[:, 0, :]
+ .unsqueeze(1)
+ .repeat(1, self.h * self.loss.outputsize_multiplier, 1)
+ )
return dec_out
def forward(self, windows_batch):
insample_y = windows_batch["insample_y"]
y_pred = self.forecast(insample_y)
- y_pred = y_pred[:, -self.h :, :]
- y_pred = self.loss.domain_map(y_pred)
+ y_pred = y_pred.reshape(insample_y.shape[0], self.h, -1)
- # domain_map might have squeezed the last dimension in case n_series == 1
- if y_pred.ndim == 2:
- return y_pred.unsqueeze(-1)
- else:
- return y_pred
+ return y_pred
diff --git a/neuralforecast/models/stemgnn.py b/neuralforecast/models/stemgnn.py
index 85a014e65..b242ad2ce 100644
--- a/neuralforecast/models/stemgnn.py
+++ b/neuralforecast/models/stemgnn.py
@@ -8,8 +8,9 @@
import torch.nn as nn
import torch.nn.functional as F
+from typing import Optional
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
# %% ../../nbs/models.stemgnn.ipynb 7
class GLU(nn.Module):
@@ -136,7 +137,7 @@ def forward(self, x, mul_L):
return forecast, backcast_source
# %% ../../nbs/models.stemgnn.ipynb 9
-class StemGNN(BaseMultivariate):
+class StemGNN(BaseModel):
"""StemGNN
The Spectral Temporal Graph Neural Network (`StemGNN`) is a Graph-based multivariate
@@ -163,6 +164,10 @@ class StemGNN(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int, number of windows in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=1024, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=1024, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='robust', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int, random_seed for pytorch initializer and numpy generators.
@@ -178,10 +183,13 @@ class StemGNN(BaseMultivariate):
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -191,6 +199,7 @@ def __init__(
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
n_stacks=2,
multi_layer: int = 5,
dropout_rate: float = 0.5,
@@ -203,6 +212,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=1024,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "robust",
random_seed: int = 1,
@@ -224,6 +237,7 @@ def __init__(
futr_exog_list=futr_exog_list,
hist_exog_list=hist_exog_list,
stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -232,6 +246,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
num_workers_loader=num_workers_loader,
@@ -370,11 +388,5 @@ def forward(self, windows_batch):
forecast = forecast.reshape(
batch_size, self.h, self.loss.outputsize_multiplier * self.n_series
)
- forecast = self.loss.domain_map(forecast)
- # domain_map might have squeezed the last dimension in case n_series == 1
- # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.
- if forecast.ndim == 2:
- return forecast.unsqueeze(-1)
- else:
- return forecast
+ return forecast
diff --git a/neuralforecast/models/tcn.py b/neuralforecast/models/tcn.py
index fd900512c..8ac791ef7 100644
--- a/neuralforecast/models/tcn.py
+++ b/neuralforecast/models/tcn.py
@@ -10,11 +10,11 @@
import torch.nn as nn
from ..losses.pytorch import MAE
-from ..common._base_recurrent import BaseRecurrent
+from ..common._base_model import BaseModel
from ..common._modules import MLP, TemporalConvolutionEncoder
# %% ../../nbs/models.tcn.ipynb 7
-class TCN(BaseRecurrent):
+class TCN(BaseModel):
"""TCN
Temporal Convolution Network (TCN), with MLP decoder.
@@ -56,10 +56,13 @@ class TCN(BaseRecurrent):
"""
# Class attributes
- SAMPLING_TYPE = "recurrent"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -68,10 +71,10 @@ def __init__(
inference_input_size: int = -1,
kernel_size: int = 2,
dilations: List[int] = [1, 2, 4, 8, 16],
- encoder_hidden_size: int = 200,
+ encoder_hidden_size: int = 128,
encoder_activation: str = "ReLU",
context_size: int = 10,
- decoder_hidden_size: int = 200,
+ decoder_hidden_size: int = 128,
decoder_layers: int = 2,
futr_exog_list=None,
hist_exog_list=None,
@@ -85,6 +88,10 @@ def __init__(
val_check_steps: int = 100,
batch_size: int = 32,
valid_batch_size: Optional[int] = None,
+ windows_batch_size=128,
+ inference_windows_batch_size=1024,
+ start_padding_enabled=False,
+ step_size: int = 1,
scaler_type: str = "robust",
random_seed: int = 1,
num_workers_loader=0,
@@ -109,6 +116,10 @@ def __init__(
val_check_steps=val_check_steps,
batch_size=batch_size,
valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
+ step_size=step_size,
scaler_type=scaler_type,
futr_exog_list=futr_exog_list,
hist_exog_list=hist_exog_list,
@@ -139,7 +150,9 @@ def __init__(
self.decoder_layers = decoder_layers
# TCN input size (1 for target variable y)
- input_encoder = 1 + self.hist_exog_size + self.stat_exog_size
+ input_encoder = (
+ 1 + self.hist_exog_size + self.stat_exog_size + self.futr_exog_size
+ )
# ---------------------------------- Instantiate Model -----------------------------------#
# Instantiate historic encoder
@@ -152,14 +165,11 @@ def __init__(
)
# Context adapter
- self.context_adapter = nn.Linear(
- in_features=self.encoder_hidden_size + self.futr_exog_size * h,
- out_features=self.context_size * h,
- )
+ self.context_adapter = nn.Linear(in_features=self.input_size, out_features=h)
# Decoder MLP
self.mlp_decoder = MLP(
- in_features=self.context_size + self.futr_exog_size,
+ in_features=self.encoder_hidden_size + self.futr_exog_size,
out_features=self.loss.outputsize_multiplier,
hidden_size=self.decoder_hidden_size,
num_layers=self.decoder_layers,
@@ -170,50 +180,51 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- encoder_input = windows_batch["insample_y"] # [B, seq_len, 1]
- futr_exog = windows_batch["futr_exog"]
- hist_exog = windows_batch["hist_exog"]
- stat_exog = windows_batch["stat_exog"]
+ encoder_input = windows_batch["insample_y"] # [B, L, 1]
+ futr_exog = windows_batch["futr_exog"] # [B, L + h, F]
+ hist_exog = windows_batch["hist_exog"] # [B, L, X]
+ stat_exog = windows_batch["stat_exog"] # [B, S]
# Concatenate y, historic and static inputs
- # [B, C, seq_len, 1] -> [B, seq_len, C]
- # Contatenate [ Y_t, | X_{t-L},..., X_{t} | S ]
- batch_size, seq_len = encoder_input.shape[:2]
+ batch_size, input_size = encoder_input.shape[:2]
if self.hist_exog_size > 0:
- hist_exog = hist_exog.permute(0, 2, 1, 3).squeeze(
- -1
- ) # [B, X, seq_len, 1] -> [B, seq_len, X]
- encoder_input = torch.cat((encoder_input, hist_exog), dim=2)
+ encoder_input = torch.cat(
+ (encoder_input, hist_exog), dim=2
+ ) # [B, L, 1] + [B, L, X] -> [B, L, 1 + X]
if self.stat_exog_size > 0:
+ # print(encoder_input.shape)
stat_exog = stat_exog.unsqueeze(1).repeat(
- 1, seq_len, 1
- ) # [B, S] -> [B, seq_len, S]
- encoder_input = torch.cat((encoder_input, stat_exog), dim=2)
-
- # TCN forward
- hidden_state = self.hist_encoder(
- encoder_input
- ) # [B, seq_len, tcn_hidden_state]
+ 1, input_size, 1
+ ) # [B, S] -> [B, L, S]
+ encoder_input = torch.cat(
+ (encoder_input, stat_exog), dim=2
+ ) # [B, L, 1 + X] + [B, L, S] -> [B, L, 1 + X + S]
if self.futr_exog_size > 0:
- futr_exog = futr_exog.permute(0, 2, 3, 1)[
- :, :, 1:, :
- ] # [B, F, seq_len, 1+H] -> [B, seq_len, H, F]
- hidden_state = torch.cat(
- (hidden_state, futr_exog.reshape(batch_size, seq_len, -1)), dim=2
- )
+ encoder_input = torch.cat(
+ (encoder_input, futr_exog[:, :input_size]), dim=2
+ ) # [B, L, 1 + X + S] + [B, L, F] -> [B, L, 1 + X + S + F]
+
+ # TCN forward
+ hidden_state = self.hist_encoder(encoder_input) # [B, L, C]
# Context adapter
- context = self.context_adapter(hidden_state)
- context = context.reshape(batch_size, seq_len, self.h, self.context_size)
+ hidden_state = hidden_state.permute(0, 2, 1) # [B, L, C] -> [B, C, L]
+ context = self.context_adapter(hidden_state) # [B, C, L] -> [B, C, h]
# Residual connection with futr_exog
if self.futr_exog_size > 0:
- context = torch.cat((context, futr_exog), dim=-1)
+ futr_exog_futr = futr_exog[:, input_size:].swapaxes(
+ 1, 2
+ ) # [B, L + h, F] -> [B, F, h]
+ context = torch.cat(
+ (context, futr_exog_futr), dim=1
+ ) # [B, C, h] + [B, F, h] = [B, C + F, h]
+
+ context = context.swapaxes(1, 2) # [B, C + F, h] -> [B, h, C + F]
# Final forecast
- output = self.mlp_decoder(context)
- output = self.loss.domain_map(output)
+ output = self.mlp_decoder(context) # [B, h, C + F] -> [B, h, n_output]
return output
diff --git a/neuralforecast/models/tft.py b/neuralforecast/models/tft.py
index f96d5646b..8214fbe15 100644
--- a/neuralforecast/models/tft.py
+++ b/neuralforecast/models/tft.py
@@ -13,7 +13,7 @@
from torch.nn import LayerNorm
import pandas as pd
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.tft.ipynb 11
def get_activation_fn(activation_str: str) -> Callable:
@@ -419,7 +419,7 @@ def forward(self, temporal_features, ce):
return x, atten_vect
# %% ../../nbs/models.tft.ipynb 24
-class TFT(BaseWindows):
+class TFT(BaseModel):
"""TFT
The Temporal Fusion Transformer architecture (TFT) is an Sequence-to-Sequence
@@ -470,10 +470,13 @@ class TFT(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -595,7 +598,7 @@ def __init__(
def forward(self, windows_batch):
# Parsiw windows_batch
- y_insample = windows_batch["insample_y"][:, :, None] # <- [B,T,1]
+ y_insample = windows_batch["insample_y"] # <- [B,T,1]
futr_exog = windows_batch["futr_exog"]
hist_exog = windows_batch["hist_exog"]
stat_exog = windows_batch["stat_exog"]
@@ -665,7 +668,6 @@ def forward(self, windows_batch):
# Adapt output to loss
y_hat = self.output_adapter(temporal_features)
- y_hat = self.loss.domain_map(y_hat)
return y_hat
diff --git a/neuralforecast/models/tide.py b/neuralforecast/models/tide.py
index ec98c2b13..b5a6f9144 100644
--- a/neuralforecast/models/tide.py
+++ b/neuralforecast/models/tide.py
@@ -11,7 +11,7 @@
import torch.nn.functional as F
from ..losses.pytorch import MAE
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
# %% ../../nbs/models.tide.ipynb 8
class MLPResidual(nn.Module):
@@ -48,7 +48,7 @@ def forward(self, input):
return x
# %% ../../nbs/models.tide.ipynb 10
-class TiDE(BaseWindows):
+class TiDE(BaseModel):
"""TiDE
Time-series Dense Encoder (`TiDE`) is a MLP-based univariate time-series forecasting model. `TiDE` uses Multi-layer Perceptrons (MLPs) in an encoder-decoder model for long-term time-series forecasting.
@@ -94,10 +94,13 @@ class TiDE(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -243,7 +246,7 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
- x = windows_batch["insample_y"].unsqueeze(-1) # [B, L, 1]
+ x = windows_batch["insample_y"] # [B, L, 1]
hist_exog = windows_batch["hist_exog"] # [B, L, X]
futr_exog = windows_batch["futr_exog"] # [B, L + h, F]
stat_exog = windows_batch["stat_exog"] # [B, S]
@@ -313,7 +316,6 @@ def forward(self, windows_batch):
x
) # [B, h, temporal_width + decoder_output_dim] -> [B, h, n_outputs]
- # Map to output domain
- forecast = self.loss.domain_map(x + x_skip)
+ forecast = x + x_skip
return forecast
diff --git a/neuralforecast/models/timellm.py b/neuralforecast/models/timellm.py
index aa9276f72..3468db9b6 100644
--- a/neuralforecast/models/timellm.py
+++ b/neuralforecast/models/timellm.py
@@ -7,12 +7,12 @@
import math
from typing import Optional
+import neuralforecast.losses.pytorch as losses
import torch
import torch.nn as nn
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..common._modules import RevIN
-
from ..losses.pytorch import MAE
try:
@@ -165,7 +165,7 @@ def reprogramming(self, target_embedding, source_embedding, value_embedding):
return reprogramming_embedding
# %% ../../nbs/models.timellm.ipynb 11
-class TimeLLM(BaseWindows):
+class TimeLLM(BaseModel):
"""TimeLLM
Time-LLM is a reprogramming framework to repurpose an off-the-shelf LLM for time series forecasting.
@@ -226,10 +226,13 @@ class TimeLLM(BaseWindows):
"""
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -309,6 +312,15 @@ def __init__(
dataloader_kwargs=dataloader_kwargs,
**trainer_kwargs,
)
+ if loss.outputsize_multiplier > 1:
+ raise Exception(
+ "TimeLLM only supports point loss functions (MAE, MSE, etc) as loss function."
+ )
+
+ if valid_loss is not None and not isinstance(valid_loss, losses.BasePointLoss):
+ raise Exception(
+ "TimeLLM only supports point loss functions (MAE, MSE, etc) as valid loss function."
+ )
# Architecture
self.patch_len = patch_len
@@ -468,12 +480,9 @@ def calcute_lags(self, x_enc):
return lags
def forward(self, windows_batch):
- insample_y = windows_batch["insample_y"]
-
- x = insample_y.unsqueeze(-1)
+ x = windows_batch["insample_y"]
y_pred = self.forecast(x)
y_pred = y_pred[:, -self.h :, :]
- y_pred = self.loss.domain_map(y_pred)
return y_pred
diff --git a/neuralforecast/models/timemixer.py b/neuralforecast/models/timemixer.py
index 5585539bd..10dd07222 100644
--- a/neuralforecast/models/timemixer.py
+++ b/neuralforecast/models/timemixer.py
@@ -11,7 +11,7 @@
import torch
import torch.nn as nn
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
from neuralforecast.common._modules import (
PositionalEmbedding,
TokenEmbedding,
@@ -19,8 +19,8 @@
SeriesDecomp,
RevIN,
)
-
from ..losses.pytorch import MAE
+from typing import Optional
# %% ../../nbs/models.timemixer.ipynb 6
class DataEmbedding_wo_pos(nn.Module):
@@ -249,7 +249,7 @@ def forward(self, x_list):
return out_list
# %% ../../nbs/models.timemixer.ipynb 12
-class TimeMixer(BaseMultivariate):
+class TimeMixer(BaseModel):
"""TimeMixer
**Parameters**
`h`: int, Forecast horizon.
@@ -279,6 +279,10 @@ class TimeMixer(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -293,14 +297,17 @@ class TimeMixer(BaseMultivariate):
`**trainer_kwargs`: int, keyword trainer arguments inherited from [PyTorch Lighning's trainer](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=trainer).
**References**
- [Shiyu Wang, Haixu Wu, Xiaoming Shi, Tengge Hu, Huakun Luo, Lintao Ma, James Y. Zhang, Jun Zhou."TimeMixer: Decomposable Multiscale Mixing For Time Series Forecasting"](https://openreview.net/pdf?id=7oLshfEIC2)
+ [Shiyu Wang, Haixu Wu, Xiaoming Shi, Tengge Hu, Huakun Luo, Lintao Ma, James Y. Zhang, Jun Zhou."TimeMixer: Decomposable Multiscale Mixing For Time Series Forecasting"](https://openreview.net/pdf?id=7oLshfEIC2)
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -331,6 +338,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=256,
+ inference_windows_batch_size=256,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -359,6 +370,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
random_seed=random_seed,
@@ -474,6 +489,11 @@ def __init__(
]
)
+ if self.loss.outputsize_multiplier > 1:
+ self.distr_output = nn.Linear(
+ self.n_series, self.n_series * self.loss.outputsize_multiplier
+ )
+
def out_projection(self, dec_out, i, out_res):
dec_out = self.projection_layer(dec_out)
out_res = out_res.permute(0, 2, 1)
@@ -647,10 +667,7 @@ def forward(self, windows_batch):
y_pred = self.forecast(insample_y, x_mark_enc, x_mark_dec)
y_pred = y_pred[:, -self.h :, :]
- y_pred = self.loss.domain_map(y_pred)
+ if self.loss.outputsize_multiplier > 1:
+ y_pred = self.distr_output(y_pred)
- # domain_map might have squeezed the last dimension in case n_series == 1
- if y_pred.ndim == 2:
- return y_pred.unsqueeze(-1)
- else:
- return y_pred
+ return y_pred
diff --git a/neuralforecast/models/timesnet.py b/neuralforecast/models/timesnet.py
index aab548382..24dde3ecd 100644
--- a/neuralforecast/models/timesnet.py
+++ b/neuralforecast/models/timesnet.py
@@ -12,7 +12,7 @@
import torch.fft
from ..common._modules import DataEmbedding
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
@@ -119,7 +119,7 @@ def forward(self, x):
return res
# %% ../../nbs/models.timesnet.ipynb 10
-class TimesNet(BaseWindows):
+class TimesNet(BaseModel):
"""TimesNet
The TimesNet univariate model tackles the challenge of modeling multiple intraperiod and interperiod temporal variations.
@@ -199,10 +199,13 @@ class TimesNet(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -309,13 +312,9 @@ def forward(self, windows_batch):
# Parse windows_batch
insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
futr_exog = windows_batch["futr_exog"]
# Parse inputs
- insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]
if self.futr_exog_size > 0:
x_mark_enc = futr_exog[:, : self.input_size, :]
else:
@@ -332,5 +331,5 @@ def forward(self, windows_batch):
# porject back
dec_out = self.projection(enc_out)
- forecast = self.loss.domain_map(dec_out[:, -self.h :])
+ forecast = dec_out[:, -self.h :]
return forecast
diff --git a/neuralforecast/models/tsmixer.py b/neuralforecast/models/tsmixer.py
index 0d68e1e4c..46dd6d908 100644
--- a/neuralforecast/models/tsmixer.py
+++ b/neuralforecast/models/tsmixer.py
@@ -1,15 +1,16 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/models.tsmixer.ipynb.
# %% auto 0
-__all__ = ['TemporalMixing', 'FeatureMixing', 'MixingLayer', 'ReversibleInstanceNorm1d', 'TSMixer']
+__all__ = ['TemporalMixing', 'FeatureMixing', 'MixingLayer', 'TSMixer']
# %% ../../nbs/models.tsmixer.ipynb 5
-import torch
import torch.nn as nn
import torch.nn.functional as F
+from typing import Optional
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
+from ..common._modules import RevINMultivariate
# %% ../../nbs/models.tsmixer.ipynb 8
class TemporalMixing(nn.Module):
@@ -93,44 +94,7 @@ def forward(self, input):
return x
# %% ../../nbs/models.tsmixer.ipynb 10
-class ReversibleInstanceNorm1d(nn.Module):
- """
- ReversibleInstanceNorm1d
- """
-
- def __init__(self, n_series, eps=1e-5):
- super().__init__()
- self.weight = nn.Parameter(torch.ones((1, 1, n_series)))
- self.bias = nn.Parameter(torch.zeros((1, 1, n_series)))
-
- self.eps = eps
-
- def forward(self, x):
- # Batch statistics
- self.batch_mean = torch.mean(x, axis=1, keepdim=True).detach()
- self.batch_std = torch.sqrt(
- torch.var(x, axis=1, keepdim=True, unbiased=False) + self.eps
- ).detach()
-
- # Instance normalization
- x = x - self.batch_mean
- x = x / self.batch_std
- x = x * self.weight
- x = x + self.bias
-
- return x
-
- def reverse(self, x):
- # Reverse the normalization
- x = x - self.bias
- x = x / self.weight
- x = x * self.batch_std
- x = x + self.batch_mean
-
- return x
-
-# %% ../../nbs/models.tsmixer.ipynb 12
-class TSMixer(BaseMultivariate):
+class TSMixer(BaseModel):
"""TSMixer
Time-Series Mixer (`TSMixer`) is a MLP-based multivariate time-series forecasting model. `TSMixer` jointly learns temporal and cross-sectional representations of the time-series by repeatedly combining time- and feature information using stacked mixing layers. A mixing layer consists of a sequential time- and feature Multi Layer Perceptron (`MLP`).
@@ -154,6 +118,10 @@ class TSMixer(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -173,10 +141,13 @@ class TSMixer(BaseMultivariate):
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = False
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -186,6 +157,7 @@ def __init__(
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
n_block=2,
ff_dim=64,
dropout=0.9,
@@ -198,6 +170,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=256,
+ inference_windows_batch_size=256,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -219,6 +195,7 @@ def __init__(
futr_exog_list=futr_exog_list,
hist_exog_list=hist_exog_list,
stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -227,6 +204,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
random_seed=random_seed,
@@ -243,7 +224,7 @@ def __init__(
# Reversible InstanceNormalization layer
self.revin = revin
if self.revin:
- self.norm = ReversibleInstanceNorm1d(n_series=n_series)
+ self.norm = RevINMultivariate(num_features=n_series, affine=True)
# Mixing layers
mixing_layers = [
@@ -266,22 +247,16 @@ def forward(self, windows_batch):
# TSMixer: InstanceNorm + Mixing layers + Dense output layer + ReverseInstanceNorm
if self.revin:
- x = self.norm(x)
+ x = self.norm(x, "norm")
x = self.mixing_layers(x)
x = x.permute(0, 2, 1)
x = self.out(x)
x = x.permute(0, 2, 1)
if self.revin:
- x = self.norm.reverse(x)
+ x = self.norm(x, "denorm")
x = x.reshape(
batch_size, self.h, self.loss.outputsize_multiplier * self.n_series
)
- forecast = self.loss.domain_map(x)
-
- # domain_map might have squeezed the last dimension in case n_series == 1
- # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.
- if forecast.ndim == 2:
- return forecast.unsqueeze(-1)
- else:
- return forecast
+
+ return x
diff --git a/neuralforecast/models/tsmixerx.py b/neuralforecast/models/tsmixerx.py
index 24897d442..61eb55e68 100644
--- a/neuralforecast/models/tsmixerx.py
+++ b/neuralforecast/models/tsmixerx.py
@@ -8,8 +8,10 @@
import torch.nn as nn
import torch.nn.functional as F
+from typing import Optional
from ..losses.pytorch import MAE
-from ..common._base_multivariate import BaseMultivariate
+from ..common._base_model import BaseModel
+from ..common._modules import RevINMultivariate
# %% ../../nbs/models.tsmixerx.ipynb 8
class TemporalMixing(nn.Module):
@@ -158,7 +160,7 @@ def reverse(self, x):
return x
# %% ../../nbs/models.tsmixerx.ipynb 12
-class TSMixerx(BaseMultivariate):
+class TSMixerx(BaseModel):
"""TSMixerx
Time-Series Mixer exogenous (`TSMixerx`) is a MLP-based multivariate time-series forecasting model, with capability for additional exogenous inputs. `TSMixerx` jointly learns temporal and cross-sectional representations of the time-series by repeatedly combining time- and feature information using stacked mixing layers. A mixing layer consists of a sequential time- and feature Multi Layer Perceptron (`MLP`).
@@ -182,6 +184,10 @@ class TSMixerx(BaseMultivariate):
`early_stop_patience_steps`: int=-1, Number of validation iterations before early stopping.
`val_check_steps`: int=100, Number of training steps between every validation loss check.
`batch_size`: int=32, number of different series in each batch.
+ `valid_batch_size`: int=None, number of different series in each validation and test batch, if None uses batch_size.
+ `windows_batch_size`: int=256, number of windows to sample in each training batch, default uses all.
+ `inference_windows_batch_size`: int=256, number of windows to sample in each inference batch, -1 uses all.
+ `start_padding_enabled`: bool=False, if True, the model will pad the time series with zeros at the beginning, by input size.
`step_size`: int=1, step size between each window of temporal data.
`scaler_type`: str='identity', type of scaler for temporal inputs normalization see [temporal scalers](https://nixtla.github.io/neuralforecast/common.scalers.html).
`random_seed`: int=1, random_seed for pytorch initializer and numpy generators.
@@ -201,10 +207,13 @@ class TSMixerx(BaseMultivariate):
"""
# Class attributes
- SAMPLING_TYPE = "multivariate"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = True
EXOGENOUS_STAT = True
+ MULTIVARIATE = True # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -214,6 +223,7 @@ def __init__(
futr_exog_list=None,
hist_exog_list=None,
stat_exog_list=None,
+ exclude_insample_y=False,
n_block=2,
ff_dim=64,
dropout=0.0,
@@ -226,6 +236,10 @@ def __init__(
early_stop_patience_steps: int = -1,
val_check_steps: int = 100,
batch_size: int = 32,
+ valid_batch_size: Optional[int] = None,
+ windows_batch_size=256,
+ inference_windows_batch_size=256,
+ start_padding_enabled=False,
step_size: int = 1,
scaler_type: str = "identity",
random_seed: int = 1,
@@ -247,6 +261,7 @@ def __init__(
futr_exog_list=futr_exog_list,
hist_exog_list=hist_exog_list,
stat_exog_list=stat_exog_list,
+ exclude_insample_y=exclude_insample_y,
loss=loss,
valid_loss=valid_loss,
max_steps=max_steps,
@@ -255,6 +270,10 @@ def __init__(
early_stop_patience_steps=early_stop_patience_steps,
val_check_steps=val_check_steps,
batch_size=batch_size,
+ valid_batch_size=valid_batch_size,
+ windows_batch_size=windows_batch_size,
+ inference_windows_batch_size=inference_windows_batch_size,
+ start_padding_enabled=start_padding_enabled,
step_size=step_size,
scaler_type=scaler_type,
random_seed=random_seed,
@@ -270,7 +289,7 @@ def __init__(
# Reversible InstanceNormalization layer
self.revin = revin
if self.revin:
- self.norm = ReversibleInstanceNorm1d(n_series=n_series)
+ self.norm = RevINMultivariate(num_features=n_series, affine=True)
# Forecast horizon
self.h = h
@@ -358,12 +377,12 @@ def forward(self, windows_batch):
stat_exog = windows_batch["stat_exog"] # [N, stat_exog_size (S)]
batch_size, input_size = x.shape[:2]
- # Add channel dimension to x
- x = x.unsqueeze(1) # [B, L, N] -> [B, 1, L, N]
-
# Apply revin to x
if self.revin:
- x = self.norm(x) # [B, 1, L, N] -> [B, 1, L, N]
+ x = self.norm(x, mode="norm") # [B, L, N] -> [B, L, N]
+
+ # Add channel dimension to x
+ x = x.unsqueeze(1) # [B, L, N] -> [B, 1, L, N]
# Concatenate x with historical exogenous
if self.hist_exog_size > 0:
@@ -430,24 +449,16 @@ def forward(self, windows_batch):
x = self.mixing_block(x) # [B, h, ff_dim] -> [B, h, ff_dim]
# Fully connected output layer
- x = self.out(x) # [B, h, ff_dim] -> [B, h, N * n_outputs]
+ forecast = self.out(x) # [B, h, ff_dim] -> [B, h, N * n_outputs]
# Reverse Instance Normalization on output
if self.revin:
- x = x.reshape(
- batch_size, self.h, self.loss.outputsize_multiplier, -1
- ) # [B, h, N * n_outputs] -> [B, h, n_outputs, N]
- x = self.norm.reverse(x)
- x = x.reshape(
+ forecast = forecast.reshape(
+ batch_size, self.h * self.loss.outputsize_multiplier, -1
+ ) # [B, h, N * n_outputs] -> [B, h * n_outputs, N]
+ forecast = self.norm(forecast, "denorm")
+ forecast = forecast.reshape(
batch_size, self.h, -1
- ) # [B, h, n_outputs, N] -> [B, h, n_outputs * N]
+ ) # [B, h * n_outputs, N] -> [B, h, n_outputs * N]
- # Map to loss domain
- forecast = self.loss.domain_map(x)
-
- # domain_map might have squeezed the last dimension in case n_series == 1
- # Note that this fails in case of a tuple loss, but Multivariate does not support tuple losses yet.
- if forecast.ndim == 2:
- return forecast.unsqueeze(-1)
- else:
- return forecast
+ return forecast
diff --git a/neuralforecast/models/vanillatransformer.py b/neuralforecast/models/vanillatransformer.py
index 69fcc9c4d..7cf9ec714 100644
--- a/neuralforecast/models/vanillatransformer.py
+++ b/neuralforecast/models/vanillatransformer.py
@@ -19,7 +19,7 @@
DataEmbedding,
AttentionLayer,
)
-from ..common._base_windows import BaseWindows
+from ..common._base_model import BaseModel
from ..losses.pytorch import MAE
@@ -73,7 +73,7 @@ def forward(self, queries, keys, values, attn_mask):
return (V.contiguous(), None)
# %% ../../nbs/models.vanillatransformer.ipynb 10
-class VanillaTransformer(BaseWindows):
+class VanillaTransformer(BaseModel):
"""VanillaTransformer
Vanilla Transformer, following implementation of the Informer paper, used as baseline.
@@ -129,10 +129,13 @@ class VanillaTransformer(BaseWindows):
"""
# Class attributes
- SAMPLING_TYPE = "windows"
EXOGENOUS_FUTR = True
EXOGENOUS_HIST = False
EXOGENOUS_STAT = False
+ MULTIVARIATE = False # If the model produces multivariate forecasts (True) or univariate (False)
+ RECURRENT = (
+ False # If the model produces forecasts recursively (True) or direct (False)
+ )
def __init__(
self,
@@ -293,14 +296,8 @@ def __init__(
def forward(self, windows_batch):
# Parse windows_batch
insample_y = windows_batch["insample_y"]
- # insample_mask = windows_batch['insample_mask']
- # hist_exog = windows_batch['hist_exog']
- # stat_exog = windows_batch['stat_exog']
-
futr_exog = windows_batch["futr_exog"]
- insample_y = insample_y.unsqueeze(-1) # [Ws,L,1]
-
if self.futr_exog_size > 0:
x_mark_enc = futr_exog[:, : self.input_size, :]
x_mark_dec = futr_exog[:, -(self.label_len + self.h) :, :]
@@ -317,5 +314,5 @@ def forward(self, windows_batch):
dec_out = self.dec_embedding(x_dec, x_mark_dec)
dec_out = self.decoder(dec_out, enc_out, x_mask=None, cross_mask=None)
- forecast = self.loss.domain_map(dec_out[:, -self.h :])
+ forecast = dec_out[:, -self.h :]
return forecast
diff --git a/neuralforecast/utils.py b/neuralforecast/utils.py
index 4a272dfcb..ab3ff1d5e 100644
--- a/neuralforecast/utils.py
+++ b/neuralforecast/utils.py
@@ -6,17 +6,16 @@
'HourOfDay', 'DayOfWeek', 'DayOfMonth', 'DayOfYear', 'MonthOfYear', 'WeekOfYear',
'time_features_from_frequency_str', 'augment_calendar_df', 'get_indexer_raise_missing',
'PredictionIntervals', 'add_conformal_distribution_intervals', 'add_conformal_error_intervals',
- 'get_prediction_interval_method']
+ 'get_prediction_interval_method', 'level_to_quantiles', 'quantiles_to_level']
# %% ../nbs/utils.ipynb 3
import random
from itertools import chain
-from typing import List, Union
+from typing import List, Union, Optional, Tuple
from utilsforecast.compat import DFType
import numpy as np
import pandas as pd
-import utilsforecast.processing as ufp
# %% ../nbs/utils.ipynb 6
def generate_series(
@@ -484,77 +483,113 @@ def __repr__(self):
# %% ../nbs/utils.ipynb 32
def add_conformal_distribution_intervals(
- fcst_df: DFType,
+ model_fcsts: np.array,
cs_df: DFType,
- model_names: List[str],
- level: List[Union[int, float]],
+ model: str,
cs_n_windows: int,
n_series: int,
horizon: int,
-) -> DFType:
+ level: Optional[List[Union[int, float]]] = None,
+ quantiles: Optional[List[float]] = None,
+) -> Tuple[np.array, List[str]]:
"""
Adds conformal intervals to a `fcst_df` based on conformal scores `cs_df`.
`level` should be already sorted. This strategy creates forecasts paths
based on errors and calculate quantiles using those paths.
"""
- fcst_df = ufp.copy_if_pandas(fcst_df, deep=False)
- alphas = [100 - lv for lv in level]
- cuts = [alpha / 200 for alpha in reversed(alphas)]
- cuts.extend(1 - alpha / 200 for alpha in alphas)
- for model in model_names:
- scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)
- scores = scores.transpose(1, 0, 2)
- # restrict scores to horizon
- scores = scores[:, :, :horizon]
- mean = fcst_df[model].to_numpy().reshape(1, n_series, -1)
- scores = np.vstack([mean - scores, mean + scores])
- quantiles = np.quantile(
- scores,
- cuts,
- axis=0,
- )
- quantiles = quantiles.reshape(len(cuts), -1).T
+ assert (
+ level is not None or quantiles is not None
+ ), "Either level or quantiles must be provided"
+
+ if quantiles is None and level is not None:
+ alphas = [100 - lv for lv in level]
+ cuts = [alpha / 200 for alpha in reversed(alphas)]
+ cuts.extend(1 - alpha / 200 for alpha in alphas)
+ elif quantiles is not None:
+ cuts = quantiles
+
+ scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)
+ scores = scores.transpose(1, 0, 2)
+ # restrict scores to horizon
+ scores = scores[:, :, :horizon]
+ mean = model_fcsts.reshape(1, n_series, -1)
+ scores = np.vstack([mean - scores, mean + scores])
+ scores_quantiles = np.quantile(
+ scores,
+ cuts,
+ axis=0,
+ )
+ scores_quantiles = scores_quantiles.reshape(len(cuts), -1).T
+ if quantiles is None and level is not None:
lo_cols = [f"{model}-lo-{lv}" for lv in reversed(level)]
hi_cols = [f"{model}-hi-{lv}" for lv in level]
out_cols = lo_cols + hi_cols
- fcst_df = ufp.assign_columns(fcst_df, out_cols, quantiles)
- return fcst_df
+ elif quantiles is not None:
+ out_cols = [f"{model}-ql{q}" for q in quantiles]
+
+ fcsts_with_intervals = np.hstack([model_fcsts, scores_quantiles])
+
+ return fcsts_with_intervals, out_cols
# %% ../nbs/utils.ipynb 33
def add_conformal_error_intervals(
- fcst_df: DFType,
+ model_fcsts: np.array,
cs_df: DFType,
- model_names: List[str],
- level: List[Union[int, float]],
+ model: str,
cs_n_windows: int,
n_series: int,
horizon: int,
-) -> DFType:
+ level: Optional[List[Union[int, float]]] = None,
+ quantiles: Optional[List[float]] = None,
+) -> Tuple[np.array, List[str]]:
"""
Adds conformal intervals to a `fcst_df` based on conformal scores `cs_df`.
`level` should be already sorted. This startegy creates prediction intervals
based on the absolute errors.
"""
- fcst_df = ufp.copy_if_pandas(fcst_df, deep=False)
- cuts = [lv / 100 for lv in level]
- for model in model_names:
- mean = fcst_df[model].to_numpy().ravel()
- scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)
- scores = scores.transpose(1, 0, 2)
- # restrict scores to horizon
- scores = scores[:, :, :horizon]
- quantiles = np.quantile(
- scores,
- cuts,
- axis=0,
- )
- quantiles = quantiles.reshape(len(cuts), -1)
+ assert (
+ level is not None or quantiles is not None
+ ), "Either level or quantiles must be provided"
+
+ if quantiles is None and level is not None:
+ cuts = [lv / 100 for lv in level]
+ elif quantiles is not None:
+ cuts = quantiles
+
+ mean = model_fcsts.ravel()
+ scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)
+ scores = scores.transpose(1, 0, 2)
+ # restrict scores to horizon
+ scores = scores[:, :, :horizon]
+ scores_quantiles = np.quantile(
+ scores,
+ cuts,
+ axis=0,
+ )
+ scores_quantiles = scores_quantiles.reshape(len(cuts), -1)
+ if quantiles is None and level is not None:
lo_cols = [f"{model}-lo-{lv}" for lv in reversed(level)]
hi_cols = [f"{model}-hi-{lv}" for lv in level]
- quantiles = np.vstack([mean - quantiles[::-1], mean + quantiles]).T
- columns = lo_cols + hi_cols
- fcst_df = ufp.assign_columns(fcst_df, columns, quantiles)
- return fcst_df
+ out_cols = lo_cols + hi_cols
+ scores_quantiles = np.vstack(
+ [mean - scores_quantiles[::-1], mean + scores_quantiles]
+ ).T
+ elif quantiles is not None:
+ out_cols = []
+ scores_quantiles_ls = []
+ for i, q in enumerate(quantiles):
+ out_cols.append(f"{model}-ql{q}")
+ if q < 0.5:
+ scores_quantiles_ls.append(mean - scores_quantiles[::-1][i])
+ elif q > 0.5:
+ scores_quantiles_ls.append(mean + scores_quantiles[i])
+ else:
+ scores_quantiles_ls.append(mean)
+ scores_quantiles = np.vstack(scores_quantiles_ls).T
+
+ fcsts_with_intervals = np.hstack([model_fcsts, scores_quantiles])
+
+ return fcsts_with_intervals, out_cols
# %% ../nbs/utils.ipynb 34
def get_prediction_interval_method(method: str):
@@ -568,3 +603,30 @@ def get_prediction_interval_method(method: str):
f'please choose one of {", ".join(available_methods.keys())}'
)
return available_methods[method]
+
+# %% ../nbs/utils.ipynb 35
+def level_to_quantiles(level: List[Union[int, float]]) -> List[float]:
+ """
+ Converts a list of levels to a list of quantiles.
+ """
+ level_set = set(level)
+ return sorted(
+ list(
+ set(sum([[(50 - l / 2) / 100, (50 + l / 2) / 100] for l in level_set], []))
+ )
+ )
+
+
+def quantiles_to_level(quantiles: List[float]) -> List[Union[int, float]]:
+ """
+ Converts a list of quantiles to a list of levels.
+ """
+ quantiles_set = set(quantiles)
+ return sorted(
+ set(
+ [
+ int(round(100 - 200 * (q * (q < 0.5) + (1 - q) * (q >= 0.5)), 2))
+ for q in quantiles_set
+ ]
+ )
+ )