From 047264b37550ae8faddf5f02b8346a9b0819170d Mon Sep 17 00:00:00 2001 From: Parth Nobel Date: Sat, 14 Sep 2024 19:57:38 -0700 Subject: [PATCH] Merges in Draft code (#1) * Begin new RandALO API * Adds examples with API design * Adds the modeling layer * Work on reductions * Environment setup * Better imports * Begin ALO impl * First pass at ALO impl * Adds precommit * Fixes examples * starts work on jacobian * Bug fixes for RandALO * Adds first part of the Jacobian expressions * Adds Jacobian operator * Added truncnorm tests * Added utils.py tests * Basic randalo.py test * Fixes `diag` property -> method * More randalo.py tests * Fixed test rng * Updated to Ruff formatter * Fixes up the Jacobian * Loss functions now reduce themselves * + cvxpylayers dep * Begins sklearn->model impl * scikit-learn integration for regression * scikit-learn example * Adds demo to README * Added logistic regression support * Added logistic regression example * Fix ABC * Cut at cleaning up the code * Refactor * Added generic Jacobian example to README * Adds workflow file * Misc finishing work --------- Co-authored-by: Daniel LeJeune <16905520+dlej@users.noreply.github.com> --- .github/workflows/build.yml | 49 ++++ .pre-commit-config.yaml | 7 + README.md | 75 +++++- alogcv/alo.py | 72 ------ alogcv/diagonal.py | 13 -- alogcv/gcv.py | 72 ------ examples/cvxpy-example.ipynb | 208 +++++++++++++++++ examples/randalo_modeling.py | 21 ++ examples/scikit-learn.ipynb | 388 +++++++++++++++++++++++++++++++ randalo/__init__.py | 9 + randalo/modeling_layer.py | 252 ++++++++++++++++++++ randalo/randalo.py | 344 +++++++++++++++++++++++++++ randalo/reductions.py | 139 +++++++++++ randalo/sklearn_integration.py | 76 ++++++ randalo/truncnorm.py | 107 +++++++++ randalo/utils.py | 180 ++++++++++++++ requirements.txt | 11 + setup.py | 30 +-- {alogcv => test}/__init__.py | 0 test/test_randalo.py | 173 ++++++++++++++ test/test_reductions.py | 48 ++++ test/test_sklearn_integration.py | 261 +++++++++++++++++++++ test/test_truncnorm.py | 81 +++++++ test/test_utils.py | 97 ++++++++ 24 files changed, 2531 insertions(+), 182 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .pre-commit-config.yaml delete mode 100644 alogcv/alo.py delete mode 100644 alogcv/diagonal.py delete mode 100644 alogcv/gcv.py create mode 100644 examples/cvxpy-example.ipynb create mode 100644 examples/randalo_modeling.py create mode 100644 examples/scikit-learn.ipynb create mode 100644 randalo/__init__.py create mode 100644 randalo/modeling_layer.py create mode 100644 randalo/randalo.py create mode 100644 randalo/reductions.py create mode 100644 randalo/sklearn_integration.py create mode 100644 randalo/truncnorm.py create mode 100644 randalo/utils.py create mode 100644 requirements.txt rename {alogcv => test}/__init__.py (100%) create mode 100644 test/test_randalo.py create mode 100644 test/test_reductions.py create mode 100644 test/test_sklearn_integration.py create mode 100644 test/test_truncnorm.py create mode 100644 test/test_utils.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..bb594b2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,49 @@ +name: Publish Python 🐍 distribution 📦 to PyPI + +on: push + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/randalo + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..bdc1b47 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: 'v0.1.11' + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] \ No newline at end of file diff --git a/README.md b/README.md index ba34280..2d364d9 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,73 @@ -# ALO Library +# RandALO: fast randomized risk estimation for high-dimensional data + +This repository contains a software package implementing RandALO, a fast randomized method for risk estimation of machine learning models, as described in the paper, + +P. T. Nobel, D. LeJeune, E. J. Candès. RandALO: Out-of-sample risk estimation in no time flat. 2024. ## Installation In a folder run the following: + +```bash +git clone git@github.com:cvxgrp/randalo.git +cd randalo + +# create a new environment with Python >= 3.10 (could also use venv or similar) +conda create -n randalo python=3.12 + +# install requirements and randalo +pip install -r requirements.txt ``` -git clone git@github.com:cvxgrp/alo.git -cd alo -# create a new environment with torch & friends (could also use conda or similar) -python -m venv venv -. venv/bin/activate +## Usage -pip install wheel -pip install torch numpy scipy matplotlib +### Scikit-learn -pip install git+ssh://git@github.com/cvxgrp/SURE-CR.git@xtrace -pip install git+ssh://git@github.com/cvxgrp/torch_linops.git -pip install -e . +The simplest way to use RandALO is with linear models from scikit-learn. See a longer demonstration in a notebook [here](examples/scikit-learn.ipynb). + +```python +from torch import nn +from sklearn.linear_model import Lasso +from randalo import RandALO + +X, y = ... # load data as np.ndarrays as usual + +model = Lasso(1.0).fit(X, y) # fit the model +alo = RandALO.from_sklearn(model, X, y) # set up the Jacobian +mse_estimate = alo.evaluate(nn.MSELoss()) # estimate risk ``` + +We currently support the following models: + +- `LinearRegression` +- `Ridge` +- `Lasso` +- `LassoLars` +- `ElasticNet` +- `LogisticRegression` + +### Linear models with any solver + +If you prefer to use other solvers for fitting your models than scikit-learn, or if you wish to extend to other models than the ones listed above, you can still use RandALO by instantiating the Jacobian yourself. You need only be careful to ensure that you scale the regularizer correctly for your problem formulation. + +```python +from torch import nn +from sklearn.linear_model import Lasso +from randalo import RandALO, MSELoss, L1Regularizer, Jacobian + +X, y = ... # load data as np.ndarrays as usual + +model = Lasso(1.0).fit(X, y) # fit the model + +# instantiate RandALO by creating a Jacobian object +loss = MSELoss() +reg = 2.0 * model.alpha * L1Regularizer() # scale the regularizer appropriately +y_hat = model.predict(X) +solution_func = lambda: model.coef_ +jac = Jacobian(y, X, solution_func, loss, reg) +alo = RandALO(loss, jac, y, y_hat) + +mse_estimate = alo.evaluate(nn.MSELoss()) # estimate risk +``` + +Please refer to our [scikit-learn integration](randalo/sklearn_integration.py) source code for more examples. diff --git a/alogcv/alo.py b/alogcv/alo.py deleted file mode 100644 index 8d0e5fb..0000000 --- a/alogcv/alo.py +++ /dev/null @@ -1,72 +0,0 @@ -import torch -#torch.autograd.set_detect_anomaly(True) -from typing import Literal -from surecr.solver import Solver -import alo.diagonal as diag_lib -import time - -class GCV: - def __init__(self, - solver: Solver): - self._solver = solver - self._divergence_strategy: \ - Literal['default', 'exact', 'xdiag'] = 'default' - self._solution = None - - @property - def solution(self): - """ - Returns solver.solve(y) from the last compute call. - """ - return self._solution - - def runtimes(self): - """ - Returns how long it took for the solver to run and how long it took - the divergence estimator to run during the last compute call. - """ - return {'solver': - self._t_f_solver - self._t_i_solver, - 'divergence': - self._t_f_div - self._t_i_div, - } - - def compute(self, data: torch.Tensor, divergence_parameters={}) -> torch.Tensor: - """ - Computes and returns SURE for the estimator computed by the solver - at the point y. - - Currently, divergence_parameters can contain the key "m" to indicate - how many samples to use during the divergence estimation (which - dominates the runtime at high dimensions). The default is for m to be - 102. - - In the future we may switch to A-Hutch++ and may change what options - the divergence_parameters specifies. - """ - - shape = data.shape - shape_ereased_data = data.reshape((-1,)) - shape_ereased_data.requires_grad_(True) - - def mu_hat(self, y): - y = y.reshape(shape) - self._solution = self._solver.solve(y) - return self._solver.estimate(self._solution).reshape((-1,)) - - dim_var_term = -data.numel() * self._variance - - self._t_i_solver = time.monotonic() - mu_hat_evaled = mu_hat(self, shape_ereased_data) - self._t_f_solver = time.monotonic() - - self._t_i_div = time.monotonic() - diagonal = diag_lib.diagonal(mu_hat_evaled, - shape_ereased_data, - self._divergence_strategy, - divergence_parameters) - self._t_f_div = time.monotonic() - self._divergence = divergence - r = shape_ereased_data.detach() - mu_hat_evaled.deatch() - - return ((r / (1 - diagonal))**2).sum() diff --git a/alogcv/diagonal.py b/alogcv/diagonal.py deleted file mode 100644 index 684570d..0000000 --- a/alogcv/diagonal.py +++ /dev/null @@ -1,13 +0,0 @@ -import torch -import linops as lo -import linops.diag as lod - -def diagonal(solution, input_vec, strategy, parameters): - J = lo.VectorJacobianOperator(solution, input_vec) - - if strategy == 'exact': - return lod.exact_diag(J, **parameters)[0] - elif strategy == 'default' or strategy == 'xdiag': - return lod.xdiag(J, **parameters)[0] - else: - raise RuntimeError("Unknown divergence strategy.") diff --git a/alogcv/gcv.py b/alogcv/gcv.py deleted file mode 100644 index 415ef52..0000000 --- a/alogcv/gcv.py +++ /dev/null @@ -1,72 +0,0 @@ -import torch -#torch.autograd.set_detect_anomaly(True) -from typing import Literal -from surecr.solver import Solver -import surecr.divergence as div_lib -import time - -class GCV: - def __init__(self, - solver: Solver): - self._solver = solver - self._divergence_strategy: \ - Literal['default', 'exact', 'xtrace', 'hutchinson', 'hutch++'] = 'default' - self._solution = None - - @property - def solution(self): - """ - Returns solver.solve(y) from the last compute call. - """ - return self._solution - - def runtimes(self): - """ - Returns how long it took for the solver to run and how long it took - the divergence estimator to run during the last compute call. - """ - return {'solver': - self._t_f_solver - self._t_i_solver, - 'divergence': - self._t_f_div - self._t_i_div, - } - - def compute(self, data: torch.Tensor, divergence_parameters={}) -> torch.Tensor: - """ - Computes and returns SURE for the estimator computed by the solver - at the point y. - - Currently, divergence_parameters can contain the key "m" to indicate - how many samples to use during the divergence estimation (which - dominates the runtime at high dimensions). The default is for m to be - 102. - - In the future we may switch to A-Hutch++ and may change what options - the divergence_parameters specifies. - """ - - shape = data.shape - shape_ereased_data = data.reshape((-1,)) - shape_ereased_data.requires_grad_(True) - - def mu_hat(self, y): - y = y.reshape(shape) - self._solution = self._solver.solve(y) - return self._solver.estimate(self._solution).reshape((-1,)) - - dim_var_term = -data.numel() * self._variance - - self._t_i_solver = time.monotonic() - mu_hat_evaled = mu_hat(self, shape_ereased_data) - self._t_f_solver = time.monotonic() - - self._t_i_div = time.monotonic() - divergence = div_lib.divergence(mu_hat_evaled, - shape_ereased_data, - self._divergence_strategy, - divergence_parameters) - self._t_f_div = time.monotonic() - self._divergence = divergence - r = shape_ereased_data.detach() - mu_hat_evaled.deatch() - - return ((r / (1 - divergence / data.numel()))**2).sum() diff --git a/examples/cvxpy-example.ipynb b/examples/cvxpy-example.ipynb new file mode 100644 index 0000000..b87c6f7 --- /dev/null +++ b/examples/cvxpy-example.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "55d17ccd-1abc-4c82-a62d-2530d2937c0a", + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'randalo'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[2], line 10\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mcvxpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mcp\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m#import torch\u001b[39;00m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m#from torch.nn import functional as F\u001b[39;00m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;66;03m#from tqdm import tqdm\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mrandalo\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RandALO\n\u001b[1;32m 12\u001b[0m np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mseed(\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m 13\u001b[0m \u001b[38;5;66;03m#torch.manual_seed(0)\u001b[39;00m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'randalo'" + ] + } + ], + "source": [ + "import time\n", + "\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import cvxpy as cp\n", + "#import torch\n", + "#from torch.nn import functional as F\n", + "#from tqdm import tqdm\n", + "\n", + "import randalo\n", + "\n", + "np.random.seed(0)\n", + "#torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "id": "a2d8471f-feba-4e26-b700-c9b7518bf18d", + "metadata": {}, + "source": [ + "# Using RandALO with CVXPY\n", + "\n", + "Since the CVXPY modeling language is significantly more general than the settings where RandALO is appropriate and CVXPYlayers (the differentiation library for CVXPY scales poorly to large problems), we provide a simple modeling language to describe a loss function and a regularizer. We also provide helper methods to transform these loss and regularizers into a CVXPY problem and a linear operator representing the Jacobian." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0761a852-bf1f-4eac-afeb-598e33905015", + "metadata": {}, + "outputs": [], + "source": [ + "n, p = 100, 30\n", + "X = np.random.randn(n, p)\n", + "beta = np.zeros(p)\n", + "p[0:3] = 1.0\n", + "p[9:12] = 2.0\n", + "y = X @ beta + 0.1 * np.random.randn(n)" + ] + }, + { + "cell_type": "markdown", + "id": "74eba39c-c246-42b9-bc68-87bfc28df0b0", + "metadata": {}, + "source": [ + "## Forming the model\n", + "\n", + "We provide two loss functions: `randalo.LogisticLoss` and `randalo.MLELoss`. Custom loss functions can be written by subclassing `randalo.Loss`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "76bc4a11-8983-49bf-b652-7366476f7e38", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'randalo' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[43mrandalo\u001b[49m\u001b[38;5;241m.\u001b[39mMLELoss()\n", + "\u001b[0;31mNameError\u001b[0m: name 'randalo' is not defined" + ] + } + ], + "source": [ + "loss = randalo.MLELoss()" + ] + }, + { + "cell_type": "markdown", + "id": "558eb322-9cfd-4745-8cda-4d0fe42d5bb2", + "metadata": {}, + "source": [ + "Additionally, we support a number of regularizers: `randalo.L2Regularizer`, `randalo.L1Regularizer`, and `SquareRegularizer`. Each of the default regularizers takes an optional matrix (or indices of the variable it should operate on). Custom regularizers may be written by subclassing `Regularizer`. Hyperparameters can be implemented with `randalo.HyperParameter`.\n", + "\n", + "In this notebook we implement group lasso." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4512444a-0033-45ec-ba34-c6bada1701c1", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'randalo' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m alpha \u001b[38;5;241m=\u001b[39m \u001b[43mrandalo\u001b[49m\u001b[38;5;241m.\u001b[39mHyperParameter()\n\u001b[1;32m 3\u001b[0m regularizer \u001b[38;5;241m=\u001b[39m alpha \u001b[38;5;241m*\u001b[39m \u001b[38;5;28msum\u001b[39m(randalo\u001b[38;5;241m.\u001b[39mL2Regularizer(\u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mrange\u001b[39m(i, i \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m3\u001b[39m)) \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, p, \u001b[38;5;241m3\u001b[39m)))\n", + "\u001b[0;31mNameError\u001b[0m: name 'randalo' is not defined" + ] + } + ], + "source": [ + "alpha = randalo.HyperParameter()\n", + "\n", + "regularizer = alpha * sum(randalo.L2Regularizer(list(range(i, i + 3)) for i in range(0, p, 3)))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e578a365-4ea8-448d-afbc-527ab3baf703", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'randalo' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m b \u001b[38;5;241m=\u001b[39m cp\u001b[38;5;241m.\u001b[39mVariable(p)\n\u001b[0;32m----> 3\u001b[0m prob, J \u001b[38;5;241m=\u001b[39m \u001b[43mrandalo\u001b[49m\u001b[38;5;241m.\u001b[39mgen_cvxpy_jacobian(loss, regularizer, X, b, y) \n", + "\u001b[0;31mNameError\u001b[0m: name 'randalo' is not defined" + ] + } + ], + "source": [ + "b = cp.Variable(p)\n", + "\n", + "prob, J = randalo.gen_cvxpy_jacobian(loss, regularizer, X, b, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "77f085f3-d364-483d-8538-7d8c3594eb39", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'alpha' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43malpha\u001b[49m\u001b[38;5;241m.\u001b[39mvalue \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.1\u001b[39m\n\u001b[1;32m 2\u001b[0m prob\u001b[38;5;241m.\u001b[39msolve()\n\u001b[1;32m 4\u001b[0m alo \u001b[38;5;241m=\u001b[39m randalo\u001b[38;5;241m.\u001b[39mRandALO(loss, J, y, b\u001b[38;5;241m.\u001b[39mvalue)\n", + "\u001b[0;31mNameError\u001b[0m: name 'alpha' is not defined" + ] + } + ], + "source": [ + "alpha.value = 0.1\n", + "prob.solve()\n", + "\n", + "alo = randalo.RandALO(loss, J, y, b.value)\n", + "alo.evaluate(loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ccd15851-7844-4455-a58c-d803152d0cad", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/randalo_modeling.py b/examples/randalo_modeling.py new file mode 100644 index 0000000..654bfd3 --- /dev/null +++ b/examples/randalo_modeling.py @@ -0,0 +1,21 @@ +import cvxpy as cp +import numpy as np + +import randalo as ra + +X, y = ... + +beta = cp.Variable(X.shape[1]) +lamda = ra.HyperParameter() +gamma = ra.HyperParameter() + +loss = ra.LogisticLoss() +regularizer = lamda * ra.SquareRegularizer() + gamma * ra.L1Regularizer(np.diff(np.eye(X.shape[1]))) + +prob, J = ra.gen_cvxpy_jacobian(loss, regularizer, X, beta, y) +lamda.value = 10 +gamma.value = 1 +prob.solve() + +alo = ra.RandALO(y_hat=X @ beta.value, y=y, loss=loss, J=J).evaluate(loss) + diff --git a/examples/scikit-learn.ipynb b/examples/scikit-learn.ipynb new file mode 100644 index 0000000..7bcad8e --- /dev/null +++ b/examples/scikit-learn.ipynb @@ -0,0 +1,388 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import torch\n", + "from torch.nn import functional as F\n", + "from tqdm import tqdm\n", + "\n", + "from sklearn.linear_model import Lasso, LogisticRegression\n", + "from sklearn.model_selection import cross_validate\n", + "\n", + "from randalo import RandALO\n", + "\n", + "np.random.seed(0)\n", + "torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using RandALO with scikit-learn models\n", + "\n", + "RandALO can be easily applied to a wide variety of linear models from scikit-learn, provided they were trained with twice-differentiable losses Unfortunately, this does rule out models with non-differentiable losses such as SVMs, but it does include logistic regression. Non-differentiable regularizers, on the other hand, are perfectly fine for RandALO, so we can use sparsity-inducing penalties such as the $\\ell_1$ norm or elastic net penalties.\n", + "\n", + "## Example 1: Lasso\n", + "\n", + "To get started with RandALO, we need a large dataset with plenty of randomness, such that ALO provides a good estimate of risk and cross-validation takes longer than we'd like. We'll generate sparse ground truth coefficients and apply the lasso to solve the problem." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# generate a large problem\n", + "n = 10000\n", + "p = 5000\n", + "s = 500\n", + "sigma = 1.0\n", + "\n", + "# we can generate our data as numpy arrays as usual\n", + "rng = np.random.default_rng(0)\n", + "X = rng.integers(0, 2, (n, p)) * 2 - 1\n", + "beta = np.zeros(p)\n", + "beta[:s] = rng.normal(0, 1 / np.sqrt(s), (s,))\n", + "y = X @ beta + rng.normal(0, sigma, (n,))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's now apply and compare\n", + "- standard 5-fold cross-validation as implemented in `cross_validate` from scikit-learn\n", + "- training the model from scikit-learn on the whole dataset and applying RandALO\n", + "\n", + "We'll compare on the basis of both time and risk estimation.\n", + "\n", + "For this problem, the risk metric we care about is mean squared error, which we will need to provide to the `RandALO` object. While everything else that we need to apply RandALO can be the ordinary `numpy` arrays that we normally use with scikit-learn, the risk metric should be a Pytorch function. Specifically, the risk function should accept `torch.Tensor` objects `y` and `y_hat` having shape `(n,)` and return a `float` value of the mean risk over `y` and `z`. We'll simply borrow an implementation of mean squared error from `torch.nn.functional` for this example." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Performing 5-fold CV with scikit-learn\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 20/20 [00:47<00:00, 2.39s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5-fold CV computed in 47.759 seconds\n", + "Applying RandALO and computing true risks\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 20/20 [00:27<00:00, 1.37s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RandALO computed in 27.439 seconds\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# define our risk function using torch.nn.functional\n", + "def mse_risk_fun(y, y_hat):\n", + " return F.mse_loss(y, y_hat, reduction=\"mean\").item()\n", + "\n", + "\n", + "# select a meaningful range of hyperparameters\n", + "alphas = np.linspace(0.1, 1.0, 20) / np.sqrt(s)\n", + "\n", + "# first compute cross-validation\n", + "# ==============================\n", + "print(\"Performing 5-fold CV with scikit-learn\")\n", + "cv_risks = []\n", + "tic = time.monotonic()\n", + "\n", + "for alpha in tqdm(alphas):\n", + " lasso = Lasso(alpha)\n", + " cv_results = cross_validate(lasso, X, y, cv=5, scoring=\"neg_mean_squared_error\")\n", + " cv_risks.append(-np.mean(cv_results[\"test_score\"]))\n", + "\n", + "toc = time.monotonic()\n", + "cv_time = toc - tic\n", + "print(f\"5-fold CV computed in {cv_time:01.3f} seconds\")\n", + "\n", + "# next, use RandALO, and compute the true conditional risks while we're at it\n", + "# ===========================================================================\n", + "print(\"Applying RandALO and computing true risks\")\n", + "alo_risks = []\n", + "true_risks = []\n", + "tic = time.monotonic()\n", + "\n", + "for alpha in tqdm(alphas):\n", + " lasso = Lasso(alpha)\n", + " lasso.fit(X, y)\n", + " alo = RandALO.from_sklearn(lasso, X, y)\n", + " alo_risks.append(alo.evaluate(mse_risk_fun))\n", + " true_risks.append(np.linalg.norm(lasso.coef_ - beta) ** 2 + sigma**2)\n", + "\n", + "toc = time.monotonic()\n", + "alo_time = toc - tic\n", + "print(f\"RandALO computed in {alo_time:01.3f} seconds\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "RandALO was able to compute the risk estimate in just over half of the time of 5-fold CV, with minimal additional code. What's more, the risk estimate is actually closer to the true risk, while 5-fold CV is biased:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGzCAYAAADHdKgcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACRcklEQVR4nOzdd3yN5//H8dcZ2TuyhCD23sTealWr2trEKlVUqzpoi2rL91eKtihqxCiKolU1au8t9gohRpaQnZycnHP//jhEU0JCkpPxeT4e59Gc+77u+/4c1Hm77uu+LpWiKApCCCGEEIWI2twFCCGEEELkNglAQgghhCh0JAAJIYQQotCRACSEEEKIQkcCkBBCCCEKHQlAQgghhCh0JAAJIYQQotDRmruAvMhoNHL37l0cHBxQqVTmLkcIIYQQmaAoCnFxcXh7e6NWP7uPRwLQU9y9excfHx9zlyGEEEKIF3Dr1i2KFy/+zDYSgJ7CwcEBMP0COjo6mrkaIYQQQmRGbGwsPj4+ad/jzyIB6Cke3fZydHSUACSEEELkM5kZviKDoIUQQghR6EgAEkIIIUShIwFICCGEEIWOjAF6CQaDAb1eb+4yRBZZWFig0WjMXYYQQggzkgD0AhRFISwsjOjoaHOXIl6Qs7MzXl5eMs+TEEIUUhKAXsCj8OPh4YGtra18ieYjiqKQmJhIREQEAEWLFjVzRUIIIcxBAlAWGQyGtPBTpEgRc5cjXoCNjQ0AEREReHh4yO0wIYQohGQQdBY9GvNja2tr5krEy3j0+ydjuIQQonCSAPSC5LZX/ia/f0IIUbhJABJCCCFEoSMBSAghhBCFjgQg8VyXLl2iQYMGWFtbU7NmzUwd079/f7p06fLMNi1atOCDDz546fqEEEKIrJIAVIhMnDgRlUqV7lWxYsXnHjdhwgTs7Oy4fPkyO3bsyIVKH0tJSeG7776jRo0a2Nra4ubmRuPGjVm8eDF6vZ7OnTvTvn37px67b98+VCoVZ86cydWahRBCPMeVrWBINWsJ8hh8IVOlShW2b9+e9l6rff4fgWvXrtGpUydKliyZk6U9ISUlhXbt2nH69Gm+/vprGjdujKOjI4cPH2batGnUqlWLQYMG8eabb3L79m2KFy+e7vjFixdTt25dqlevnqt1CyGEyNj1DZMpHfh/UO1teGM+qM3TFyM9QNkoMSU1y69UgzHt+FSDkcSUVJL1huee90VptVq8vLzSXm5ubs9sr1KpOHHiBJMmTUKlUjFx4kQAzp49S6tWrbCxsaFIkSIMGTKE+Pj4DM+TkJBAv379sLe3p2jRonz//ffPrXXmzJns3buXHTt2MHz4cGrWrEnp0qXp1asXR44coVy5crz66qu4u7sTEBCQ7tj4+HjWrFnDoEGDnnsdIYQQuePS6vGm8AMk2vmAGZ/IlR6gbFR5/NYsHzO7V206VTfNRrz1fDjDV5zEz9eV34Y2TGvT5P92cT8hJd1xN/7X6YVqvHr1Kt7e3lhbW9OwYUOmTJlCiRIlMmwfGhpKmzZtaN++PWPGjMHe3p6EhATatWtHw4YNOXbsGBEREQwePJgRI0Y8EUQe+fjjj9mzZw9//PEHHh4ejBs3jpMnTz5zTNGvv/5KmzZtqFWr1hP7LCwssLCwAKBfv34EBATw+eefpz3evmbNGgwGAz179sz8L44QQoicoSicW/EZVa/OBWBTkYG80vZLswYg6QEqRPz8/AgICGDLli38/PPPBAcH07RpU+Li4jI8xsvLC61Wi729PV5eXtjb27NixQqSk5NZunQpVatWpVWrVsyaNYtly5YRHh7+xDni4+NZuHAh06ZNo3Xr1lSrVo0lS5aQmvrsnqyrV69maozSwIEDuXbtGnv27EnbtnjxYt58802cnJyee7wQQogcpCgEBnyYFn6Wefej2ZBvsNCYN4JID1A2ujCpXZaPsfzXH4B2VTy5MKkd6v8k4v2ftnzp2gA6dOiQ9nP16tXx8/OjZMmSrF69mkGDBvHuu++yfPnytDYZ3dK6ePEiNWrUwM7OLm1b48aNMRqNXL58GU9Pz3Ttr127RkpKCn5+fmnbXF1dqVChwjPrVRQlU5+rYsWKNGrUiEWLFtGiRQuCgoLYt28fkyZNytTxQgghcoZiNHJiwXDq3l0BwK8lBrHQ7iTbd77Hz21+xtbCfKsqSADKRraWL/fLqdWo0T4lEb/seTPi7OxM+fLlCQoKAmDSpEmMGTMmR671IsqXL8+lS5cy1XbQoEGMHDmS2bNns3jxYsqUKUPz5s1zuEIhhBAZMRqMHJ/7DvUj1wLwW5lhLLA6QlRSFG42bqQYUswagOQWWCEWHx/PtWvX0lZE9/DwoGzZsmmvjFSqVInTp0+TkJCQtu3AgQOo1eqn9uqUKVMGCwsLjhw5krbtwYMHXLly5Zn19erVi+3bt3Pq1Kkn9un1+nTX79atG2q1mhUrVrB06VIGDhwoy10IIYSZpKamcmSWP/Uj12JUVKyr+D5zrY5wL+ke5V3K88srv+Bs7WzWGiUAFSJjxoxhz5493Lhxg4MHD/LGG2+g0WiyPFC4d+/eWFtb4+/vz7lz59i1axcjR46kb9++T9z+ArC3t2fQoEF8/PHH7Ny5k3PnztG/f3/Uz3n08YMPPqBx48a0bt2a2bNnc/r0aa5fv87q1atp0KABV69eTXeN7t27M3bsWEJDQ+nfv3+WPpMQQojsoUtJ4egPvWn44E+MioqTtb7mYinrtPCz4JUFuFi7mLtMuQVWmNy+fZuePXsSFRWFu7s7TZo04fDhw7i7u2fpPLa2tmzdupVRo0ZRr149bG1tefPNN5k+fXqGx0ydOpX4+Hg6d+6Mg4MDH330ETExMc+8jpWVFf/88w8zZsxg3rx5jBkzBltbWypVqsT7779P1apV07UfNGgQCxcupGPHjnh7e2fpMwkhhMgGhlQuze1Do7itGBQV5/ymUrfjO9Q0puJg6UDfyn3zRPgBUCmZHWlaiMTGxuLk5ERMTAyOjo7p9iUnJxMcHIyvry/W1tZmqlC8LPl9FEKIbGbQw7ohcH4dBtQcbvgNDV8ZhlqVezebnvX9/V9yC0wIIYQQL0WnS4K1A+D8OlBbcPO17xkfs46JBydiVIzPP4EZSAASQgghxAsLjYrmxNTX4OJG0Fhy87UZvBP0KxFJEZy9d5Z4fcarBJiTBCAhhBBCvBh9EinLe9Io9Sg6LAnqPJOBVwKISIqgrHNZFrZbiKPls29FmYsEICGEEEJkXUoirOxByQcH0autudDpB4ZeWZQWfha8sgBXa1dzV5khCUBCCCGEyJLLt0IxLn8Lru8GCzvuvvUzY24sTBd+itgUMXeZzySPwQshhBAi045cDMZiVXfUqssolg6o+vzOTbWO+0n3KeNUJl+EH5AAJIQQQohM2h14Fef1PaipCiJBZY+m1+9Yl/CjGTCr9SwquFbIF+EHJAAJIYQQIhP+Pnoen796U00dTLzagfDuv2BVpBjFH+5vXKyxWevLKhkDJHKFSqViw4YN5i5DCCHEC1i79xSl/uppCj8aJyJ6LWbome8ZuHUgt+Num7u8FyIBqBDp378/KpUKlUqFhYUFvr6+fPLJJyQnJ5utpqFDh6LRaFizZs0T+yZOnEjNmjUzPNZgMDBjxgyqVauGtbU1Li4udOjQgQMHDuRgxUIIUbgs+eco1bb3obL6JnFaV+73XsLQwO8ITwzHWmuNtTZ/zqYvAaiQad++PaGhoVy/fj1tja0JEyaYpZbExERWrVrFJ598wqJFi7J0rKIo9OjRg0mTJjFq1CguXrzI7t278fHxoUWLFtLbJIQQL0lRFOZs3Efjff5UUN8mzsKd6L5LGHRyMmEJYfg6+bKo3SLcbNzMXeoLkQBUyFhZWeHl5YWPjw9dunShTZs2/PPPPwBERUXRs2dPihUrhq2tLdWqVWPlypXpjm/RogXvv/8+n3zyCa6urnh5eTFx4sR0ba5evUqzZs2wtramcuXKaef/rzVr1lC5cmU+++wz9u7dy61btzL9OVavXs3atWtZunQpgwcPxtfXlxo1ajB//nxee+01Bg8eTEJCQtZ+cYQQQgBgNCpMX7uTDscGUVZ9lzgrL2L6LWXQsW/Sws/CVxbm2/ADZg5Ae/fupXPnznh7e2dqjMju3bvTbuH8+xUWFpau3ezZsylVqhTW1tb4+flx9OjRHPwUgKJASkLuv15yHdtz585x8OBBLC0tAdMCoXXq1GHTpk2cO3eOIUOG0Ldv3yd+/ZYsWYKdnR1Hjhzhu+++Y9KkSWkhx2g00rVrVywtLTly5Ahz587l008/fer1Fy5cSJ8+fXBycqJDhw4EBARkuvYVK1ZQvnx5Onfu/MS+jz76iKioqAyDlxBCiIylGox8u2ILb58diq86nHgbb2L7LWXgkYmEJoRSyrEUC19ZiLutu7lLfSlmfQosISGBGjVqMHDgQLp27Zrp4y5fvpxulVcPD4+0n3/77TdGjx7N3Llz8fPzY+bMmbRr147Lly+na5et9Ikw2Ttnzv0s4+6CpV2WDvnrr7+wt7cnNTUVnU6HWq1m1qxZABQrVowxY8aktR05ciRbt25l9erV1K9fP2179erV026blStXjlmzZrFjxw7atm3L9u3buXTpElu3bsXb2/RrMnnyZDp06JCujqtXr3L48GHWrVsHQJ8+fRg9ejRffPEFKpXquZ/jypUrVKpU6an7Hm2/cuVKZn9ZhBBCPPT9mu0MuDqC4up7xNv5YP/OZlKs7bC3tKeUphSL2i3K9+EHzByAOnTo8MQXY2Z4eHjg7Oz81H3Tp0/nnXfeYcCAAQDMnTuXTZs2sWjRIj777LOnHqPT6dDpdGnvY2Njs1xTftGyZUt+/vlnEhISmDFjBlqtljfffBMwDSqePHkyq1ev5s6dO6SkpKDT6bC1tU13jurVq6d7X7RoUSIiIgC4ePEiPj4+aeEHoGHDhk/UsWjRItq1a4ebm6n7tGPHjgwaNIidO3fSunXrTH0W5SV7wIQQQvxHdAgf3v4QS9U9EuxLYT9kMzh64woseGUBBqOhQIQfyKfzANWsWROdTkfVqlWZOHEijRub5h5ISUnhxIkTjB07Nq2tWq2mTZs2HDp0KMPzTZkyha+++urFC7KwNfXG5DYL2+e3+Q87OzvKli0LmEJIjRo1WLhwIYMGDWLq1Kn88MMPzJw5k2rVqmFnZ8cHH3xASkpK+staWKR7r1KpMBqNma7BYDCwZMkSwsLC0Gq16bYvWrQoUwGofPnyXLx48an7Hm0vX758pmsSQohCL+Y2BLyKZfxtjC5liO62kF33TvKqo+kftHl5Xa8Xka8CUNGiRZk7dy5169ZFp9OxYMECWrRowZEjR6hduzb37t3DYDDg6emZ7jhPT08uXbqU4XnHjh3L6NGj097Hxsbi4+OT+cJUqizfisoL1Go148aNY/To0fTq1YsDBw7w+uuv06dPH8A0nufKlStUrlw50+esVKkSt27dIjQ0lKJFiwJw+PDhdG3+/vtv4uLiOHXqFBqNJm37uXPnGDBgANHR0Rn28D3So0cPevXqxcaNG58YB/T9999TpEgR2rZtm+m6hRCisNKlGpi8cjvjIj7GKu4muPgS2m0BAw98xt2Eu6hQ0al0J3OXme3yVQCqUKECFSpUSHvfqFEjrl27xowZM1i2bNkLn9fKygorK6vsKDHfefvtt/n444+ZPXs25cqVY+3atRw8eBAXFxemT59OeHh4lgJQmzZtKF++PP7+/kydOpXY2Fg+//zzdG0WLlxIp06dqFGjRrrtlStX5sMPP+TXX39l+PDhACQlJREYGJiunYODAz169GDNmjVp12ndujWxsbHMnj2bP//8kzVr1mBnl/9CqRBC5LaALYfwv/o+VuowjM4lCe+2iEEHx3E34S4lHUtS17OuuUvMEfn+Mfj69esTFBQEgJubGxqNhvDw8HRtwsPD8fLyMkd5eZ5Wq2XEiBF89913fPTRR9SuXZt27drRokULvLy86NKlS5bOp1arWb9+PUlJSdSvX5/Bgwfz7bffpu0PDw9n06ZNaeOO/nvsG2+8wcKFC9O2XblyhVq1aqV7DR06FJVKxerVqxk3bhwzZsygQoUKNG3alJs3b7J79+4s1y2EEIVSXBjvXB9FaXUYyXbFiewewKBDX3An/g4lHEqw8JWFeNp5Pv88+ZBKySMjSVUqFevXr8/yF1fbtm1xcHBIe5rIz8+P+vXr89NPPwGm2zglSpRgxIgRGQ6C/q/Y2FicnJyIiYlJ97QZmB4VDw4OxtfXF2vr/Dn7pZDfRyFE4aY3GLFIugcBr8K9yyhOxYnq8SsDDn/JjdgbFLcvzuL2i/Gyy1+dB8/6/v4vs94Ci4+PT+u9AQgODiYwMBBXV1dKlCjB2LFjuXPnDkuXLgVg5syZ+Pr6UqVKFZKTk1mwYAE7d+5k27ZtaecYPXo0/v7+1K1bl/r16zNz5kwSEhLSngoTQgghCrPElFRGL9rON7FjcUu8Do7FSOq9hsEHx3Ej9gZF7YqysN3CfBd+ssqsAej48eO0bNky7f2jgcj+/v4EBAQQGhpKSEhI2v6UlBQ++ugj7ty5g62tLdWrV2f79u3pztG9e3ciIyMZP348YWFh1KxZky1btjwxMFoIIYQobBJ0qYxatJ2PQsfgpr6Fwc4Ljf9GbFxL07pka+KC4lj4ykK87c0wt10uyzO3wPISuQVW8MnvoxCisInXpTJy4Q4+DvuYyuqbpNh4YDloM7iVTWsTnRyNs7Wz+Yp8SVm5BZbvB0ELIYQQ4tlik/UM+2U7H4V9QmX1TfQ27qT0XcPU4PUkpSaltcvP4Ser8tVj8EIIIYTImpgkPcMW7OTTyLFUVd9Ab+2Gvu8a3js9nVMRpwhNCGV6i+nmLjPXSQ+QEEIIUUBFJ6YwZP5OPokcSw31dVKtXUntt5YRZ3/iVMQpHCwdGFxtsLnLNAsJQEIIIUQB9CAhhcG/7OLTqHHUVF8j1coZQ9/fGXXuZ46FHcPOwo55beZRuUjmJ7stSCQACSGEEAVMVLyOgfN38mnUF9RWB2GwcsLYbx0fXlzAodBD2GhtmNtmLtXcq5m7VLORACSEEEIUIPfidQyYv4fPHkygnvoKBksnNP5/8lXwOvbd2Ye1xprZrWdT06OmuUs1KwlAIk+4ceMGKpXqiXW/XratEEIUNieu3mFs9AT81JcwWDqi8V8P3jXpVbEXHjYe/NjqR+p51TN3mWYnAaiQUKlUz3xNnDjRrPX5+PgQGhpK1apVzVqHEELkaymJtDsziobqCxgt7NH0Ww/F6gBQxa0Km7puoqF3QzMXmTfIY/CFRGhoaNrPv/32G+PHj+fy5ctp2+zt7dN+VhQFg8GAVps7fzxSUlKwtLSUBWuFEOIFhcUkozYk4fHXAAjeC5b20HstU+78Q0crC2q41wDAWisTvz4iPUCFhJeXV9rLyckJlUqV9v7SpUs4ODiwefNm6tSpg5WVFfv376d///5PLE77wQcf0KJFi7T3RqORKVOm4Ovri42NDTVq1GDt2rXPrKVUqVJ8/fXX9OvXD0dHR4YMGfLEba0HDx7Qu3dv3N3dsbGxoVy5cixevPip5zMYDAwcOJCKFSumWzpFCCEKg7vRSfSdt4cbs7vC9V1gYYfSaw1f393GiksreG/7e8SmxJq7zDxHeoCyUaI+McN9GrUGK41VptqqVep0Kf1pbW0tbF+wyox99tlnTJs2jdKlS+Pi4pKpY6ZMmcLy5cuZO3cu5cqVY+/evfTp0wd3d3eaN2+e4XHTpk1j/PjxTJgw4an7v/zySy5cuMDmzZtxc3MjKCiIpKSkJ9rpdDp69uzJjRs32LdvH+7u7pn7sEIIUUDoU5KYmPQ/6isnMWptUPVazZTw3ay9sha1Ss3nfp/jaPnsZSEKIwlA2chvhV+G+5oWa8qcNnPS3rdY3SLd9OP/VtezLovbP+7taP97ex7oHqRrc9b/7EtW+6RJkybRtm3bTLfX6XRMnjyZ7du307Ch6Z5y6dKl2b9/P/PmzXtmAGrVqhUfffRR2vsbN26k2x8SEkKtWrWoW7cuYOo1+q/4+Hg6deqETqdj165dODk5Zbp2IYQoEFJTKLn9PUr+K/x8f+8wKy+tRIWKSY0m0bF0R3NXmSdJABJpHoWNzAoKCiIxMfGJ0JSSkkKtWrVe6lrDhg3jzTff5OTJk7zyyit06dKFRo0apWvTs2dPihcvzs6dO7GxsclS7UIIkZ/djErgZmQMzQI/hiubQWuNqudKfow+xZILSwAY33A8r5d93cyV5l0SgLLRkV5HMtynUWvSvd/dbXeGbdWq9EOztry55aXqyiw7O7v0dajVKIqSbpter0/7OT4+HoBNmzZRrFixdO2srKx4lv9e6786dOjAzZs3+fvvv/nnn39o3bo1w4cPZ9q0aWltOnbsyPLlyzl06BCtWrV65vmEEKKgCIlKpPe8/XyRPA3UR0FjBT1W8BfxLDi7AIBxfuN4q/xbZq40b5MAlI2yMi4np9pmJ3d3d86dO5duW2BgIBYWFgBUrlwZKysrQkJCnnm762Wu7+/vj7+/P02bNuXjjz9OF4CGDRtG1apVee2119i0aVOO1CCEEHnJ7QeJ9J5/gE+TptNecxRFY4mqxwoo25pXDDr+Dv6bhkUb0rNiT3OXmudJABIZatWqFVOnTmXp0qU0bNiQ5cuXc+7cubTbWw4ODowZM4YPP/wQo9FIkyZNiImJ4cCBAzg6OuLv7//C1x4/fjx16tShSpUq6HQ6/vrrLypVqvREu5EjR2IwGHj11VfZvHkzTZo0eeFrCiFEXnY3Oole8w8yImEWnbWHUdQWqLovh3JtALDSWDGr1awn7jiIp5MAJDLUrl07vvzySz755BOSk5MZOHAg/fr14+zZxwOwv/76a9zd3ZkyZQrXr1/H2dmZ2rVrM27cuJe6tqWlJWPHjuXGjRvY2NjQtGlTVq1a9dS2H3zwAUajkY4dO7Jly5YnxgoJIUR+FxaTTK/5h/CPW0B37W4UlRrVWwtZabxP6PHpfFjnQ1QqlYSfLFAp/x3kIYiNjcXJyYmYmBgcHdM/OpicnExwcDC+vr5YW8uEUvmV/D4KIfKLiLhkesw/zGsPlvCBdp1pY5efWWNrwaRDkwCY3Xo2zYo3M2OVecOzvr//SyZCFEIIIfKoe/E6ev9yhJb3Vz8OPx2m8oe9HV8f+hoA/8r+NC3W1IxV5k9yC0wIIYTIg+4npNBnwRFqR23kS4tfTRtbfcGOomUYv3s0Cgq9Kvbio7ofoVKpzFtsPiQBSAghhMhjohNN4adsxDamWJoebafxKI6Xb8En/7yLUTHSpWwXPqv/mYSfFyQBSAghhMhDEnSp9Ft0FM/wPcy0nIMaBeoOJKbpaN5f35EUYwotfFowoeEECT8vQcYAvSAZO56/ye+fECKvsrHQ0NnxGnMtZ6LFANW6QcfvcbJ2ZpzfOPy8/JjabCpatfRhvAz51cuiR5MAJiYmyvIL+VhiommB2Ue/n0IIkVeo755g8O1xqNBDhU7QZQ6oTf0Vr5Z+lU6+naTnJxtIAMoijUaDs7MzERERANja2sofxHxEURQSExOJiIjA2dkZjUbmzBBCmF9iSiqL9gcztGIyFsvfRJUSD77NiXvtRyYfHM/oOqNxt3UHkO+cbCIB6AV4eXkBpIUgkf84Ozun/T4KIYQ5KYrC8F9PEnzlDP0OfIuFIRqK10f39mJG7fuYY2HHCIkLYXmH5RJ+spEEoBegUqkoWrQoHh4e6RYHFfmDhYWF9PwIIfIMlUrF4OqWlL45BUfDffCshqHnSj478g3Hwo5hZ2HH536fS/jJZhKAXoJGo5EvUiGEEC8nPpLGBwcB96BIWZQ+6/jmzGy2h2zHQm3BDy1/oHKRyuaussCRp8CEEEKIXKY3GPliw1lCbt+BZW9AVBA4+UC/P5gdtJq1V9aiQsX/mv4Pv6J+5i63QJIeICGEECIXpRqMjFp1it1nb9DjzP+B8TLYeUC/P1gXcZR5Z+YB8EWDL3il1CtmrrbgkgAkhBBC5BKDUeHD1afZcTaExZbTqWq8DNbO0G8DFClDQytbfJ186eDbgW4Vupm73AJNApAQQgiRCwxGhY/XnGbz6RB+tpxFI/U5sLSHPr+DZxUAitoXZWWnldhqbc1cbcFn1jFAe/fupXPnznh7e6NSqdiwYUOmjz1w4ABarZaaNWum2z5x4kRUKlW6V8WKFbO3cCGEECILjEaFz34/w/pTt5hmMY+26uOgsYKeqzhnbc0/N/9Ja2tnYSdPfOUCswaghIQEatSowezZs7N0XHR0NP369aN169ZP3V+lShVCQ0PTXvv378+OcoUQQogsUxSFzzecY82JW3xjsZgumgOg1kK3pVx3Lcaw7cP4aPdH7ArZZe5SCxWz3gLr0KEDHTp0yPJx7777Lr169UKj0Ty110ir1WZpkjudTodOp0t7Hxsbm+WahBBCiP9SFIUJf55n5dGbjNWuordmB6CCrvMJK16Tdzf3I1oXTZUiVahftL65yy1U8t1j8IsXL+b69etMmDAhwzZXr17F29ub0qVL07t3b0JCQp55zilTpuDk5JT28vHxye6yhRBCFDKKovD1XxdZeugmw7V/MFS70bSj8w/ElGvDsO3DCE0IpZRjKea0mYOdhZ15Cy5k8lUAunr1Kp999hnLly9Hq31655Wfnx8BAQFs2bKFn3/+meDgYJo2bUpcXFyG5x07diwxMTFpr1u3buXURxBCCFEIKIrC/zZfYtGBYPpptvKxdrVpxyvfklSjGyN2jCAoOggPGw/mtp2Lq7WreQsuhPLNU2AGg4FevXrx1VdfUb58+Qzb/fuWWvXq1fHz86NkyZKsXr2aQYMGPfUYKysrrKyssr1mIYQQhdOKoyHM23udN9V7mWSxxLSx+aekNniXMbtGERgZiIOlA3PbzqWYfTHzFltI5ZsAFBcXx/Hjxzl16hQjRowAwGg0oigKWq2Wbdu20apVqyeOc3Z2pnz58gQFBeV2yUIIIQqpzjW8uXvwNz6KmW/a0OA9aDEWNQolHEpgpbFiduvZlHMpZ95CC7F8E4AcHR05e/Zsum1z5sxh586drF27Fl9f36ceFx8fz7Vr1+jbt29ulCmEEELgeGcfY+K+Q4URavWBdpNBpUKNik/qfUKPij0o6VjS3GUWamYNQPHx8el6ZoKDgwkMDMTV1ZUSJUowduxY7ty5w9KlS1Gr1VStWjXd8R4eHlhbW6fbPmbMGDp37kzJkiW5e/cuEyZMQKPR0LNnz1z7XEIIIQqfJQdvoEs1MKSCDlb7ozLqoXIX6Pwju27tpnGxxlhqLFGpVBJ+8gCzBqDjx4/TsmXLtPejR48GwN/fn4CAAEJDQ5/7BNd/3b59m549exIVFYW7uztNmjTh8OHDuLu7Z2vtQgghxCNnbkcz4c/zuBFDvyPfYK2LhZKNoet8NlzfyJcHvsSvqB8/t/4ZC42FucsVgEpRFMXcReQ1sbGxODk5ERMTg6Ojo7nLEUIIkQ/M2XaW108PpVjCeXAtA4O3szvqDB/s+gCDYmBAlQGMrjva3GUWaFn5/s43Y4CEEEKIvEZRFNOyFUYj70VPg4TzYOMCvddwMu4GY/aMwaAYeK3Ma3xY50Nzlyv+JV/NAySEEELkFVvPh9Fn4RHidamw6xu4sAHUFtB9OVfUBkbsHIHOoKN58eZMbDRR1vfKY6QHSAghhMiiXZciGLHiJHqDwuHff6TNle9NO177ibtupRn2d1/iUuKo5VGLqc2nYqGWcT95jfQACSGEEFlwIOgeQ5efQG9Q+KBsOK2DJpt2NPsYavYkIjGCpNQkyjqX5adWP2GjtTFvweKppAdICCGEyKQj16MYtOQYKalG+pRNYdS9r0yPu1fpCi3GAVDToyYBHQJwtHTEycrJzBWLjEgPkBBCCJEJJ0MeMDDgGMl6I53KWjEpYRKq5GgoXg/9az9yK/5OWtvyLuXxsvMyX7HiuSQACSGEEM9x9nYM/ouOkpBioHkZB35STUP94Do4l0Dp/iuTjn9H903dORF+wtylikySW2BCCCHEM1wMjaXvoiPEJadSv6QLC10DUJ89BFaO0Gs1P139jQ1BG1Cr1MTqYs1drsgk6QESQgghMhAUEUefBUeITtRT08eZZRX2oT37G6g08HYAK6JO8cvZXwAY32A8LUu0fM4ZRV4hAUgIIYR4iuB7CfT65QhRCSlULebIioZ3sNr78ImvTtPYqk3lf0f/B8DwmsN5s/ybZqxWZJUEICGEEOI/FEXhg1WniIjTUdHLgRXt1NhuGmHa2XAEx4pXY+y+sSgodK/QnaHVh5q3YJFlEoCEEEKI/1CpVHzfrSaNyxbh17e8cNzQDww6qNAR2k5i+YXl6I162pRow9j6Y2WW53xIFkN9ClkMVQghCiejUUGt/leYSYqGha/AvcvgVR0GbAYre1IMKSw+t5j+VftjpbEyW70ivax8f0sPkBBCCAHci9fRZc4B9l2NNG0w6GFNf1P4cfAmuftSsLIHwFJjydAaQyX85GMSgIQQQgjg593XOHM7hi83nEOfaoC/P4bru8DCjsRuAQw8+DlTj03FqBjNXarIBjIPkBBCCAF82r4iiSkG3mnqi8XRn+HEYkCFvut8xlxewtl7ZwmJC6Fv5b4yy3MBID1AQgghCq1kvYFHQ2EttWqmdK1G6ag9sO0LAJRXvmVi1CH23dmHtcaaWa1mSfgpICQACSGEKJSSUgz4LzrKxD/PYzQ+fB7obiD8PhhQoO4gfrTS8+e1P9GoNExrPo2aHjXNWLHIThKAhBBCFDrJegNDlh3nSPB91p28w+0HSRBzB1b2AH0ilGnFr761WHBuIQDjG46nuU9zM1ctspMEICGEEIWK3mBkxIqT7Lt6D1tLDQED61HC3ggrukNcKLhX4laHb5l64nsARtYaSddyXc1ctchuMghaCCFEoWEwKoxefZrtFyOw0qpZ6F+POj5OsKoXhJ8FO3fo9Rs+LiX5X7P/cSbyDO9Ue8fcZYscIAFICCFEoaAoCp+vP8vG03ex0KiY26cODcsUgS1j4coW0FpDj5XgUhKA9qXa075UezNXLXKK3AITQghR4CmKwtd/XWTVsVuoVfBDj1q0rOgBR3+Bw3MAuN1hMkMu/kJYQpiZqxW5QQKQEEKIAm/GP1dYdCAYgO/eqkHHakXh6nbY/CkA95t/zLsh6zkUeoivD39tzlJFLpEAJIQQokCbt+caP+4MAmDS61V4q05xCDtnWuZCMZBYvTsjEs9zM/YmRe2KMqHhBPMWLHKFBCAhhBAF1rLDN5my+RIAn7SvQL+GpeD+dVj2BqTEoS/ZmI8c1Jy9dw5nK2fmtp2Lh62HeYsWuUICkBBCiALpbnQSX2+8AMDwlmV4r0VZiAuDpV0gIQLFswoTfauw/+4B0yzPrWdR2qm0eYsWuUaeAhNCCFEgeTvbMLdvbQ5fv8+YVypA0gNY1hWib4KLLwvrdePP8wvRqDR83+J7arjXMHfJIhdJABJCCFGgGIwKGrUKgFYVPWlV0RNSEkwTHUacB3sv6LeBzpY2/H13L30r9aVZ8WZmrlrkNrkFJoQQosA4cj2K9jP3cjMq4fHG1BRY3Q9uHQFrJ+i7DlxK4WnnyW+dfuONcm+Yr2BhNhKAhBBCFAiKovDNpotcjYhn9i7TU18YjbBhGARtB60Nxzp8w+bEkLRjLDQWZqpWmJvcAhNCCFEgqFQqFvavyw/br/Llq5VBUWDLp3BuLai1XHn1O0adn0ucPg5rjTUtS7Q0d8nCjMzaA7R37146d+6Mt7c3KpWKDRs2ZPrYAwcOoNVqqVmz5hP7Zs+eTalSpbC2tsbPz4+jR49mX9FCCCHylGS9Ie1nDwdrvn2jGtYWGtj9Pzg6H1AR1uk7hl1dQpw+jloetWjo3dB8BYs8wawBKCEhgRo1ajB79uwsHRcdHU2/fv1o3br1E/t+++03Ro8ezYQJEzh58iQ1atSgXbt2REREZFfZQggh8og70Um0nbGHtSdup99xZB7s+R8AMe2+Ztjtv4hIjMDXyZefWv2EtdbaDNWKvESlKIpi7iLA1HW5fv16unTp8ty2PXr0oFy5cmg0GjZs2EBgYGDaPj8/P+rVq8esWbMAMBqN+Pj4MHLkSD777LNM1RIbG4uTkxMxMTE4Ojq+yMcRQgiRwyLikuk29xA3ohIp427H36OaYqXVwJnVsM60gruu+acM1V3hRPgJPGw8WNZxGd723mauXOSUrHx/57tB0IsXL+b69etMmPDkVOUpKSmcOHGCNm3apG1Tq9W0adOGQ4cOZXhOnU5HbGxsupcQQoi860FCCn0XHOVGVCLFXWxYPtjPFH6ubDMNegaM9YcyVnWPE+EnsLewZ06bORJ+RJp8FYCuXr3KZ599xvLly9Fqnxy/fe/ePQwGA56enum2e3p6EhaW8eq+U6ZMwcnJKe3l4+OT7bULIYTIHnHJevwXH+VyeBweDlb8OtiPok42cPMQrO4LxlSo1g3aTaa4Q3G0ai0zW86kgmsFc5cu8pB8E4AMBgO9evXiq6++onz58tl67rFjxxITE5P2unXrVraeXwghRPZISjEwKOA4Z27H4Gpnya+D/ShZxM60uOmK7pCaDOXaQZc5qDVaRtcZzfrX1uNX1M/cpYs8Jt88Bh8XF8fx48c5deoUI0aMAEzjexRFQavVsm3bNpo0aYJGoyE8PDzdseHh4Xh5eWV4bisrK6ysrHK0fiGEEC9Hl2pg6PITHL1xHwcrLUsH1qecp4NpcdPlXUEXAyUacqz5+1THyKO/1Us5lTJn2SKPyjc9QI6Ojpw9e5bAwMC017vvvkuFChUIDAzEz88PS0tL6tSpw44dO9KOMxqN7Nixg4YN5ZFHIYTIr1INRt5feYq9VyKxsdAQMLAeVYs5mRY3XfYGxIeDZ1UOtxrDkF0jGbJtCIn6RHOXLfIws/YAxcfHExQUlPY+ODiYwMBAXF1dKVGiBGPHjuXOnTssXboUtVpN1apV0x3v4eGBtbV1uu2jR4/G39+funXrUr9+fWbOnElCQgIDBgzItc8lhBAi+xiNCp+sPcPW8+FYatT80q8udUq6Pl7c9MENcPHl8qtT+WDfaFKNqbjbusuj7uKZzBqAjh8/TsuWj2fiHD16NAD+/v4EBAQQGhpKSEhIRoc/Vffu3YmMjGT8+PGEhYVRs2ZNtmzZ8sTAaCGEEHmfoih8+cc51p26g0atYnbv2jQp5wYpiekWN7371nyGHfiMBH0C9bzqMbnJZNSqfHOTQ5hBnpkHKC+ReYCEECJv2HUpggEBx1CpYGb3mrxes5hpcdNVvSDoH7B2Iqb3Gvqe+JbgmGDKOpdlSYclOFrK392FUVa+v/PNIGghhBCFT4sK7nzUtjzuDlam8JO2uOk/oLUhufuvjDw3m+CYYDxtPfm5zc8SfkSmSAASQgiR5xiNCmq1CpVKxcjW5Uwb/7O4Kd2XE+LsybXoazhYOjC3zVy87DJ+4leIf5MAJIQQIk/57VgIfwTe5Zd+dbGz+tfX1J7/S1vclDfmQbk2lAeWdVhGTEoMZV3KmqtkkQ9JABJCCJFnRCem8M2mi8Qlp7L2xG38G5Uy7TgyD3ZPMf3ccSoPyrXG5eExpZ1Lm6NUkc/JEHkhhBB5hrOtJcsG+TG0eWn6NSxp2nhmNWz+xPRzi3H84epBp3WdOBJ6xHyFinxPeoDM4NG9bSGEECYpqUYstaZ/k9f0caamj7Npx78WN8XvXQ6UbczEnSNJVVI5HHpYlrgQL0x6gHLR9ch4us07RKef9pu7FCGEyDNO3LxPy2m7OX0rOv2Om4dgdb+0xU3P1+3Lh3tGk6qk8mrpVxlZa6RZ6hUFgwSgXFTEzopjN+5zMTSWiLhkc5cjhBBmd+5ODP0XH+NOdBLz9l57vONeEKzsDqlJUK4dt1qP472dw0lKTaJB0QZMajRJJjoUL0X+9OQiJ1sLKnqZ5qc4GnzfzNUIIYR5BUXE0W/RUeKSU6lfypXv365p2pEca5roMDkGitfnfucZDNs1kvvJ96noWpEZLWZgobEwa+0i/5MAlMv8fF0BOHJdApAQovAKiUqk94Ij3E9IoXpxJxb2r4uNpcY00eH6oXDvMjh4Q/flLLy8gpuxN/G282ZO6znYW9qbu3xRAMgg6FzWoLQrAQdvcCQ4ytylCCGEWYTFJNN74WHCY3WU97RnyYD6OFg/7NHZ8z+4/DdorKDHcnDw5IPaH6Az6OhVqRfutu7mLV4UGBKAcll93yIAXAmP535CCq52lmauSAghck9UvI7eCw5z634SpYrYsnyQHy6P/h688KdpskNAeXUmeNdGBVhoLPiiwRdmq1kUTHILLJe52llS3tPUfXtUeoGEEIVITJKevguPci0ygaJO1iwf7IeHo7VpZ/gFWP+u6ecG7zFH9YDJRyZjMBrMV7Ao0CQAmYHfw16gwzIOSAhRSCToUhmw+CgXQmNxs7fk18F+FHexNe1MvA+reoI+AXybsbpkdeaensuqy6s4HHrYvIWLAksCkBn4lX44EFqeBBNCFALJegNDlh3nZEg0TjYWLBvkR2n3hwOZDanw+yB4cAOcS7C94UC+OWpa8mJYjWE0LtbYfIWLAk0CkBnUf/gk2KWwWGIS9WauRgghctYve69zICgKO0sNAQPqUamo4+OdOybCtZ1gYcvxV8bz6ZFvUFB4q/xbDKsxzGw1i4JPApAZeDhYU9rdDkWBozekF0gIUbANaV6a12t6s8C/HrVKuDzecWY1HPwJgCuvTOD9wBmkGFNo6dOSz/0+R6WSJYNEzpEAZCaPxgEduS4DoYUQBY+iKCiKAoCVVsMPPWrRsEyRxw3unoI/TUtZ6Jp8wIib64jTx1HTvSbfNfsOrVoeUhY5SwKQmTR4OA7oRMgDM1cihBDZS1EUJv11ga//upgWgtKJj4RVfSA1Gcq1w6rVBD6t9ymVi1RmVutZWGutc79oUeiolKf+6SzcYmNjcXJyIiYmBkdHx+cf8AKiE1O4HBZHDR9nrC00OXINIYQwh1MhD3hjzkEA1r7bkLqlXB/vNOhhyWsQchCKlIN3doC1EwBGxSjre4mXkpXvb+ljNBNnW0v8Shd5fkMhhMhnapVwYdrbNUhMSU0ffgC2fAYhB0m1cmRa9db0S03AG1MAkvAjcpMEICGEENlCbzBioTGFmLfqFH+ywYklcGwBCiq+rvEK6278zZ57Z/izy5+yuKnIdRK3zej2g0Qm/HGO0b8FmrsUIYR4KetO3ua1WQeIjNM9vUHIEdj0EQCza3ViXeRR1Co1Y+qNkfAjzEICkBmpVCqWHLrJH6fvkqBLNXc5QgjxQracC2XMmtNcDI3lt2MhTzaIvQur+4JRz2/lGzEv+gwAn/t9TusSrXO5WiFM5BaYGRVztmFEy7JU9nZEo5b5LoQQ+c+eK5GMXHkKo2K67fVei7LpG+iTYVVviA9ne9HyfKu/A5hmee5WoZsZKhbCRAKQmY1pV8HcJQghxAs5GnyfocuOozcodKpWlP97szrqf/9jTlHgrw/h7klOObrxqa0BxSizPIu8QQKQEEKILDt9K5qBAcdI1htpWcGdGd1rPtmTfWQenF4BKjU+HaZTOmg5Re2KyizPIk+QAGRmiqJwNPg+R4Lv807T0thYypxAQoi87XJYHP6LjxKvS6VBaVd+7lMHS+1/hpRe3wNbx5l+fuUb3Cq9zuIyrdGqtTLLs8gT5E9hHjBqVSBhscnUKelC47Ju5i5HCCEyFHwvgd4LjhCdqKemjzML/Os9OZnrgxuwpj/RKoVjFdvStsF7ANhb2ud+wUJkQJ4CMzOVSoXfw2UxZF0wIURedic6id6/HOZevI6KXg4sGVAfe6v//Ds6JQFW9SYp+QEjfEoxOvkyq6+sMU/BQjyDBKA84NHCqIeDZWV4IUTeFBGXTO9fDnM3JpnSbnYsG+SHk+1/5u9RFNjwHqnh5/i4qDen1ak4WjpSx7OOeYoW4hkkAOUBj3qAAm9Fk6w3mLkaIYR40hfrz3EjKpFizjYsH+yHu4PVk432z0C5sIGv3dzYY6XBSmPFrNazKONcJvcLFuI5zBqA9u7dS+fOnfH29kalUrFhw4Zntt+/fz+NGzemSJEi2NjYULFiRWbMmJGuzcSJE1GpVOleFStWzMFP8fJKu9nhZm9FSqqRwFvR5i5HCCGe8E2XqjQt58aKd/zwdrZ5ssGVbbBjErOcnVjnYItapea7Zt9Ry6NW7hcrRCaYdRB0QkICNWrUYODAgXTt2vW57e3s7BgxYgTVq1fHzs6O/fv3M3ToUOzs7BgyZEhauypVqrB9+/a091pt3h7r/Wgc0KYzoRy5fp8GskiqECIPUBQl7XF1D0drlg3ye3rDe0Hw+2BWOdgx38W0sOmXDb6kVYlWuVWqEFlm1mTQoUMHOnTokOn2tWrVolatx/+aKFWqFOvWrWPfvn3pApBWq8XLyytba81pDXwfBqDgKKCcucsRQhRyulQD7y0/SafqRela+ykLmz6SHAureoIuhvteVYA43qv5Hm+VfyvXahXiReTtrpHnOHXqFAcPHuSbb75Jt/3q1at4e3tjbW1Nw4YNmTJlCiVKlMjwPDqdDp3u8QJ+sbGxOVZzRvwe9vqcDHlASqrxyTk1hBAiF605fpsdlyI4dD2KZuXdcbN/ypgfoxHWDYF7V8CxGO+9vYF68Tep61k39wsWIovy5bds8eLFsbKyom7dugwfPpzBgwen7fPz8yMgIIAtW7bw888/ExwcTNOmTYmLi8vwfFOmTMHJySnt5ePjkxsfI51yHva42lmSrDdy5nZ0rl9fCCH+rVf9Egxu4su8vnWeHn4Adn5NyPV/SNJaQfflYO9BPa96MsuzyBfyZQDat28fx48fZ+7cucycOZOVK1em7evQoQNvv/021atXp127dvz9999ER0ezevXqDM83duxYYmJi0l63bt3KjY+Rjkqlon6ph/MByePwQggzUBQFg1EBQK1W8cWrlWlazv3pjQ/8QOihHxhQ1IN3KtQmukjpXKxUiJeXL2+B+fr6AlCtWjXCw8OZOHEiPXv2fGpbZ2dnypcvT1BQUIbns7Kywsoqg3/h5KIGpV3Zcj6Mw9ejGN6y7PMPEEKIbKIoCv+35TI3oxL4oUetZ9+GP/oLD3ZMZGhRTyK0WhwtbaTXR+Q7L9QDdPv27Qz3HT58+IWLeRFGozHd+J3/io+P59q1axQtWjQXq3oxj8YBnbj5AL3BaOZqhBCFyexdQczdc43N58LYdzUy44anfiVx88cM93Qn2NICLzsvfm7zM05WTrlXrBDZ4IV6gF555RX279+Pq6truu0HDhygU6dOREdHZ+o88fHx6XpmgoODCQwMxNXVlRIlSjB27Fju3LnD0qVLAZg9ezYlSpRIm9dn7969TJs2jffffz/tHGPGjKFz586ULFmSu3fvMmHCBDQaTYY9RHlJBU8HWlZwp1oxJ3SpRiw0+fIOpRAin1m0P5hp264A8HnHSrSu5Pn0hud+J+XPEXzg6cZZayucrZyZ13YeXnb566lbIeAFA1CDBg145ZVX2LVrFw4ODsDjSQ0nTpyY6fMcP36cli1bpr0fPXo0AP7+/gQEBBAaGkpISEjafqPRyNixYwkODkar1VKmTBn+7//+j6FDh6a1uX37Nj179iQqKgp3d3eaNGnC4cOHcXfP4D52HqJWq1g8oL65yxBCFCKrj91i0l8XABjVuhzvNMtgLM/lzRjWDeFzNxcO2dhgo7VhTus5lHaSsT8if1IpiqJk9SCj0chbb73F/fv32bp1KwcPHuS1117jm2++YdSoUTlRZ66KjY3FycmJmJgYHB0dzV2OEELkiI2n7/L+qlMoCgxu4svnnSo9fSzPtV2wohu3VQZ6+ZQkTg2zW8+mkXej3C9aiGfIyvf3CwUggJSUFDp16kRiYiJnzpxhypQpjBgx4oUKzmvMHYCiE1M4duMBrSp6oFHLwEIhRPbbcTGcoctOkGpU6Fm/BJPfqPr08HPzECzvCvpEqNSZG698xbW4G7Qu0Tr3ixbiOXIkAJ05c+aJbXFxcfTs2ZNOnToxbNiwtO3Vq1fPYsl5izkDkNGoUHPSNmKTU/lrZBOqFpOBhUKI7HUw6B79A46Rkmrk9ZreTO9W8+n/2LpzApa8TnRqAs6lW0OPFaC1zP2ChcikrHx/Z3oMUM2aNVGpVPw7Lz16P2/ePObPn5+2bozBICuavyi1WkXdUq7cjEogOlFv7nKEEAXMiZsPGLz0OCmpRtpW9mTa2zWeHn7CzsGyrmyxMDDR24fpzUbQSMKPKEAyHYCCg4Nzsg7xL3P71JGlMIQQ2e783Rj6Lz5KYoqBpuXcmNWr1tOfNo28Asu6cJBkxrp7kKpS2Bt2mEYlWuR6zULklEwHoJIlS+ZkHeJfJPwIIbKb3mBk2PKTxCWnUq+UC/P61sFKq3my4YMbsPR1zulj+MDbi1QVtC/Vno/rfpzrNQuRk17om3bJkiVs2rQp7f0nn3yCs7MzjRo14ubNm9lWXGGnNxhJSpHbiUKIl2ehUfNDj5o0KevGwv71sLV8yr9/Y+7Akte4nhzBMG8vklTQsGhDJjeZjEb9lLAkRD72QgFo8uTJ2NjYAHDo0CFmzZrFd999h5ubGx9++GG2FlhYfbflEtUnbmPVsZDnNxZCiAz8e9xmrRIuLB/sh6O1xZMN4yNg6euExd1mqLc30SqoWqQqM1vOxELzlPZC5HMvFIBu3bpF2bKmtao2bNjAW2+9xZAhQ5gyZQr79u3L1gILKzsrLUl6A0euy8KoQogXExWvo/u8w5y5Hf3shon3YWkXiLrKEg9vwtRQyrEUc9rMwdbCNjdKFSLXvVAAsre3JyoqCoBt27bRtm1bAKytrUlKSsq+6goxP1/TMiNHb9znBadqEkIUclO3Xubojft8tPp02irvT0iOheVvQsR5sPdkdNff6V+lP/PbzsfF2iV3CxYiF73QUhht27Zl8ODB1KpViytXrtCxY0cAzp8/T6lSpbKzvkKrenFnrC3U3E9I4WpEPOU9HcxdkhAin/ni1crEJOkZ067C0x91T0mAFd1IvXsSjY0rqn5/YOFegY/cK+R+sULkshfqAZo9ezYNGzYkMjKS33//nSJFHq5ifuJEvlh0ND+w1KqpXcL0r68j16PMXI0QIr9I1j9+cMLeSsvPfepQxt3+yYb6ZFjVG2PIIcZ5ejGpbmcMbuVzsVIhzOuFl8IoyMy9FMYjP2y/yoztV+hUvSize9U2Wx1CiPwhJlFP74WH6VC1KMNbls24oUEPv/VFubKZ/7m5s8LBBq1Ky/KOy6niViX3ChYim+XITNBnzpyhatWqqNXqpy6L8W/5fSmMvMKvtGkc0JHr99Nm2RZCiKeJSdLTZ+ERzt2JJTQ6mR71fChib/VkQ6MB1g2BK5uZ7+rKCgfTE73fNPlGwo8oVLK0FEZYWBgeHh5PXRbjEVkKI/vU9HHGUqvmXryO6/cSnt6NLYQo9GKS9PRbeISzd2JwtbPk13f8Mgg/RvhzJJxfx2pHJ2Y5mf5O+az+Z3Qq3SmXqxbCvLK0FIa7u3vazxlJSEh4+aoEANYWGmr6OHM0+D5Hrt+XACSEeEJssh7/RUc5fTsGF1sLfh3sR0Wvp3T9Kwps/gQCf2WbnR3fFHEGFIZUH0LvSr1zu2whzC7Tg6BLliyZdgumZMmST7y8vLz4/fffadWqVY4VWxg1ePg4/JFgGQgthEgv7mH4CbwVjbOtBb8ObkClohmEn+0T4Ngv3Fdr+MLLCwWFt8q/xYiaI3K/cCHygCw9BabT6Rg7dix169alUaNGbNiwAYDFixfj6+vLjBkzZCbobNagtOkJu0fjgIQQAiBel4r/oqOcConGycbU81PZO4NBn3u+gwM/AODaaTrTWs6kc+nOfOH3hYwtFIVWluYBGj9+PPPmzaNNmzYcPHiQt99+mwEDBnD48GGmT5/O22+/jUYj68Vkp1olXLDQqAiLTSbkfiIli9iZuyQhhJnF61Lpv+goJ0OicbTW8utgP6p4Oz298cGfYPdk08/tpkCd/jQDmhVvlmv1CpEXZakHaM2aNSxdupS1a9eybds2DAYDqampnD59mh49ekj4yQE2lhpqFHdGq1ZxJTze3OUIIcwsQZfKgMVHOX7zwcPw04CqxTIIPyeXwrYvCNdoGFypPreqdM7dYoXIw7I0D5ClpSXBwcEUK1YMABsbG44ePUq1atVyrEBzyCvzAD0SEpWIm4Pl01dvFkIUGokpqfRffIyjwfdxsNayfJAfNXycn9742k5Y/hYxKoX+ZSoTlBpLHc86LG63WG57iQIrR+YBAjAYDFhaWj4+WKvF3l6eTMppJYrIYoRCCDgQFGUKP1Zalj0r/ERchNX+JGFkuG8lglJj8bDx4Nsm30r4EeKhLAUgRVHo378/Vlam+SWSk5N59913sbNLPy5l3bp12VehEEIIANpW9uS7t6pTzsOemhmFn/gI+LUbel0so0uV57QxHkdLR+a2nUsx+2K5Wq8QeVmWApC/v3+693369MnWYkTGlh66wZrjtxnUxJcuteQvMSEKi6QUA0l6A652pt73bnV9Mm6ckggre2CICWFssZLsVyVjo7VhduvZlHMpl0sVC5E/ZCkALV68OKfqEM9xJzqJs3diOHjtngQgIQqJZL2Bd5YeJzJOx6/v+OH2tNmdHzEaYf1QuHOC+W6ebLVU0Kq1zGwxk5oeNXOtZiHyCxlVm090qVmMykUd8fMtYu5ShBC5JCJWx5XwOOJ1qdy6n/jsALTjK7j4J6gt6NHuJ/ZfWUr/qv1pVKxR7hUsRD4iq8E/RV57CkwIUXhdj4wnKiGFeqVcM250YglsfN/08xvzoUZ3DEYDGrVMTSIKl6x8f2dpHiAhhBA5S5dqIPBWdNr70u72zw4/13bBptH85mDP6tpvQo3uABJ+hHgOCUD5yK37ify8+xpLD90wdylCiBygSzUwbPlJus07xO7LEc8/IOISrPbnLxtLvnFz5esHxzgRfiLnCxWiAJAAlI9cDI3l/7ZcIuDgDXOXIoTIZimpRob/epKdlyJQq8BS85y/nuMjYMXb7Fan8IW7GwC9KvaitkftXKhWiPxPAlA+Ut/XFZUKrkcmEBGXbO5yhBDZJCXVyPAVJ9l+MQIrrZqF/vVoVNYt4wP0SbCyJ0eTw/nI0x2DCjqX7syn9T+ViQ6FyCQJQPmIs60lFTwdADgafN/M1QghsoPeYGTEipP8cyE8Lfw0flb4MRph/bucizzDSE8PUlTQyqcVkxpPQq2Sv9KFyCz5vyWfaVDa9Bj8kesSgITI7/QGIyNXnGLbhXAstWp+6VeXJuWeEX4Adn7Nvct/8q6XB4lqFX5efnzX/Du0apnVRIiskACUz/j5mp4GORIcZeZKhBAvI1lvYNjyE2w5H4alRs38vnVoVt792QedXAb7p1PEYKRfsZZUd6vOD61+wErzjPmBhBBPZdYAtHfvXjp37oy3tzcqlYoNGzY8s/3+/ftp3LgxRYoUwcbGhooVKzJjxown2s2ePZtSpUphbW2Nn58fR48ezaFPkPvqPwxAV8LjuZ+QYuZqhBAvIinFNMPzozE/8/vVoUUFj2cfdH03/PUBAKpmnzCk3SwC2gdgZ2H3zMOEEE9n1gCUkJBAjRo1mD17dqba29nZMWLECPbu3cvFixf54osv+OKLL5g/f35am99++43Ro0czYcIETp48SY0aNWjXrh0REZl4pDQfKGJvRXlPewCOSi+QEPlOXLIe/0VH2Xf1HraWGhYPqPf88BN5mejV/nzt4kBilTeg5TgALDQWuVCxEAVTnpkJWqVSsX79erp06ZKl47p27YqdnR3Lli0DwM/Pj3r16jFr1iwAjEYjPj4+jBw5ks8+++yp59DpdOh0urT3sbGx+Pj45NmZoL/ccI5lh2/Sv1EpJr5WxdzlCCEyKUGXSq9fDnP6dgwOVloCBtajTslnTHIIEB9JwoJWvGOj46y1Fa2Lt2Bm659yp2Ah8plCMxP0qVOnOHjwIM2bNwcgJSWFEydO0KZNm7Q2arWaNm3acOjQoQzPM2XKFJycnNJePj7PWG05D/Ar/WgckAyEFiI/sbXUUKmoI862Fqx4p8Hzw48+Cd2qnoyyTuastRVOlo6MqD0qd4oVooDLlwGoePHiWFlZUbduXYYPH87gwYMBuHfvHgaDAU9Pz3TtPT09CQsLy/B8Y8eOJSYmJu1169atHK3/ZT0aB3QpLJaYRL2ZqxFCZJZKpeLbN6qxcUQTqhV3enZjo5HU9e/ycUowR2yssdVYM7ftPMq6lM2dYoUo4PJlANq3bx/Hjx9n7ty5zJw5k5UrV77U+aysrHB0dEz3yss8HKwp7W6HosDRG9ILJERedut+IhP/PE+qwQiARq3Cx9X2uccZd37N+Ih97LKzxVKlZVabOVR1q5rT5QpRaOTLiSN8fX0BqFatGuHh4UycOJGePXvi5uaGRqMhPDw8Xfvw8HC8vLzMUWqO8fMtwvXIBI5cj6JtZc/nHyCEyHV6g5F+i44SfC8BS62acR0rZe7AU8v54fxCNjo7oUHF9y1nUM+rXs4WK0Qhky97gP7NaDSmDWC2tLSkTp067NixI93+HTt20LBhQ3OVmCM6VvPi3eZl6FCtYAU7IQoSC42azztWoqKXAwMb+2buoOt7YOMoOiYk4qa25pumk2nh0yJH6xSiMDJrD1B8fDxBQUFp74ODgwkMDMTV1ZUSJUowduxY7ty5w9KlSwHT/D4lSpSgYsWKgGkeoWnTpvH++++nnWP06NH4+/tTt25d6tevz8yZM0lISGDAgAG5++FyWNNy7jQt95xJ04QQZmEwKmjUpjW52lT2pEUFd7TPW9wUIPIKrO4LxlQqlOvKX6/9iJ2VQw5XK0ThZNYAdPz4cVq2bJn2fvTo0QD4+/sTEBBAaGgoISEhafuNRiNjx44lODgYrVZLmTJl+L//+z+GDh2a1qZ79+5ERkYyfvx4wsLCqFmzJlu2bHliYLQQQuSEEzcf8NnvZ1jgX5eSRUyTFGYq/CTc4481b+KjJFPbxw+6/IydhXUOVytE4ZVn5gHKS7Iyj4A5xetSOfZwEHTL502kJoTIcQev3WPwkuMkphjoUtObmT1qZe5AfTLblrXlY/UDLFHxW7sllC5aO2eLFaIAysr3d74cBC1Mtp4L46M1p6np4ywBSAgz2305gqHLTqBLNdK0nBuTu1bL3IFGIwd+782n6gcYVSo6+bTB1yuTwUkI8cIkAOVj9X1dKeFqS6WijiiKgkqlMndJQhRKW86FMXLlSfQGhTaVPJjVqzbWFppMHXtq64d8kHSJVLWadm61+bLFNPl/WYhcIAEoH/NxtWXvJy2f31AIkWP+CLzD6NWnMRgVOlUvyszuNbHIzJgf4MKBqQwP3U6yRk0Th9JMab8AjTpzwUkI8XIkAAkhxAv67VgIn607i6LAm7WL891b1dOe/nqem6d/ZcjlAOI0ampbujH9tVWyuKkQuSjfzwMkTI/cXouMN3cZQhQqAQeC+fR3U/jp06AEU7MQfrhzAu+/xlA3OZnqGgdmd/0TG61NzhYshEhHeoDyuYi4ZFpP20NyqoEzE9phYynd50LktDm7g/huy2UA3mnqy7iOlTI/buf+dfi1Gxb6RKba+6F7czH2MtePELlOeoDyOXd7K2ytNOgNCqdCHpi7HCEKNEVRmL7tclr4eb91uSyFn+Cwk/y0pgvGxHvgVR2L7suwt3HJyZKFEBmQAJTPqVQqGpQuAsDhYFkYVYicpEs1si/oHgCftq/I6LblMx1+QqIuMXjLAOZbKyz0KgG914L0/AhhNnILrADw8y3CH4F3OXI9ytylCFGgWVtoCBhQn92XI3i9ZrFMH3c75iaD/upJhMpIWb2BNzsvBQeZnV4Ic5IeoALAr7QrAKduRZOsN5i5GiEKllSDkX8uhKe9d7KxyFL4uRt3h0F/vkUYqfjqU/mlxUxci9XNiVKFEFkgAagAKO1mh5u9FSmpRk7fijZ3OUIUGEajwqhVgbyz9DgL9l3P8vFhCWEM+vMt7hqTKanXs7Del7iVfSUHKhVCZJUEoAJApVKl9QIdkXFAQmQbtVpFpaIOWGhUFHexzdKxeqOeIRu7czs1nuJ6PQsqDcG9Rq8cqlQIkVUSgAqIBr6PApCMAxIiOw1vWZbNo5rSvqpXlo6zuLab4beuUlKvZ1Hx1/FqPDqHKhRCvAgJQLkoIeY2K7eO5Iu/B2T7uf0ePgl24uYDUlKN2X5+IQqL65HxvPfrCRJ0qYCph7WsRxaf1rpzElb70y4hgfUuTSna7n85UKkQ4mVIAMpFKad/5bvQXfwReZyrD65m67nLedjjamdJst7I2TvR2XpuIQqLwFvRvDX3EH+fDeObTRezfPyD5Ad8sPUdwlZ1B30ClG6JxeuzQRY3FSLPkQCUi1xq9qNZYjIAG88vz9Zzq1Qq6pcy3Qb7+2xYtp5biMJg1+UIes4/zP2EFKoXd+KjV8pn6fgYXQxDtg5kR9hhPrZXoXhVhe7LQGuZQxULIV6GBKDc5FiU1+x9Adh0YzMGY/Y+st69vg8Ayw7fJDQmKVvPLURBtvbEbQYvOU6S3kCz8u6sfKcBbvZWmT4+NiWWIdsGcyk6CFeDga+SrVD1/l0mOhQiD5MAlMuaVh+Ak8FAhCGJI6FHsvXcLcq7U9/XlXqlXEhKkfmAhHgeRVGYszuIMWtOYzAqvFGrGAv61cXOKvNzxManxDPsn3e5cP8SLgYDCx4kU7rX7+CQtUHTQojcJQEol1lWfp32SXoANp5dnK3nVqlULOpfj18HN6C0u322nluIgsZoVPhq44W0db2GNi/N92/XwFKb+b8WE/QJDNs+jDP3zuJkMPBLZAzl3l4B7lm7fSaEyH0SgHKbpR2vefoBsCP8KAn6hGw9vX0W/uUqRGGlSzUwctUpAg7eAODLVysztkMl1OqsDVb+39H/ERgZiIPByPywSCq8NhdKNMiBioUQ2U0CkBlUq/0OZVNS8EvWEZMQkSPXiIrXMWnjBc7cjs6R8wuRX8Um6+m/6BibzoRioVHxY89aDGri+0LnGmVRnBrJOuaHRVC5zWSo1DmbqxVC5BTpLjADVammrI7TYhF7G+6cBucX+8v3WaZuvcyqY7e4Eh7H8sF+2X5+IfKjiNhk/Bcf42JoLPZWWub1rUPjsm5ZOoeiKKYV4K9ux23zWJYpBlRNPoT67+RQ1UKInCA9QOagVmNRo7vp59OrcuQSw1uWpXYJZ4Y0K42iKDlyDSHyG71R4UFCCm72Vqwa0iDL4SfFkMLInSP58/hPsLofKAZU1btD6wk5VLEQIqeoFPl2fEJsbCxOTk7ExMTg6OiYMxeJvAKz63HL0orIXquoXapVzlxHCJHOlfA4rLUaShTJ4tpeBj0f7v6QPbf3YGtU+PvWHYqUaga91shcP0LkEVn5/pYeIHNxL8/u4lXoWMyTiYcm5ngvjeRcUVjtuhzBlnOhae/LezpkPfwY9Xy892P23N6DlQI/hkdQxKMKdJOJDoXIryQAmVHdyr2wMhoJTnnA+ajzOXKNpBQDP+24Spc5B0k1yBphonA5cfM+g5cc5/2VgZy7E/NC50g1pvLZ3s/YEbIDy4fhx8/K09TzY51DPcRCiBwnAciM7Gv0pFWSaWmMP88tyZFrpBqNLDoQzOlb0fx+8naOXEOIvKqmjwutK3rwavWiVPDK+qzMqcZUxu0bx7ab27BAxYzwCBop1tBnLTgWzYGKhRC5RQKQOdkV4TWnygBsvrUTvUGf7ZdwsLZgeMuyAMzcfpVkvcwQLQo2g1FB/7C3U6NW8VOvWnzfrQYWmqz/dbf1xlY239iMFhXTwyJopgd6/QbuFbK5aiFEbpMAZGYNag7ELdVAtDGFfbf35Mg1+jQoibeTNaExySw9dCNHriFEXqBLNfD+ylN89vvZtHFvVlqN6bH1F9DRtyMDnKowPTyCFknJ8OYCmehQiAJCApCZaSt0pFNyKgAbzwbkyDWsLTR80NY0Nf/sXdeIScr+niYhzC02WY//oqNsOhvKn6fvcCE09oXOk6hPJCnVtJiw6tQyRgdupmViEnT4TiY6FKIAkQBkblorOhdrDsDx++fRGXQ5cpmutYpR1sOemCQ9v+y9niPXEMJcwmOT6Tb3EIev38feSkvAgPpU8XbK8nliU2J55593+GDXB6Rc+BM2jjLtaPIh+A3J5qqFEOYkASgPqFBnCD+GR7LtdhhWqTnTO6PVqBnzimncwsL9wUTEJefIdYTIbdci4+k65yCXwuJeeIJDgPvJ9xm0dRBnIs9wPuI0t/98FxQj1OwjEx0KUQCZNQDt3buXzp074+3tjUqlYsOGDc9sv27dOtq2bYu7uzuOjo40bNiQrVu3pmszceJEVCpVulfFihVz8FNkg+J1aWntjU1KIlzcmGOXaVfFk5o+ziTpDczaGZRj1xEit2y/EE6XWQe4E52Er5sd699rRNViWe/5iUyMZOCWgVy6fwlXSycWhYZROjkByreHzj/AC44hEkLkXWYNQAkJCdSoUYPZs2dnqv3evXtp27Ytf//9NydOnKBly5Z07tyZU6dOpWtXpUoVQkND01779+/PifKzj0oFNXoCoJxekSNPg5kuo+LT9qYwuOJICCFRiTlyHSFymsGo8P22ywxeepw4XSr1Srmw9t2G+LhmbYJDgLvxd/Hf4s+1mGt4WBchIDyK8vEPwMcP3loMGlkyUYiCyKz/Z3fo0IEOHTpkuv3MmTPTvZ88eTJ//PEHGzdupFatWmnbtVotXl5e2VVm7qjejX+OfM/slKu0PjqVkQ3H5chlGpYpQrPy7uy9Esn0fy4zs0et5x8kRB4SnZjCqFWB7LkSCUD/RqX4vFOlF3rM/WbsTQZvG0xYQhjF7IqyIDyK4tF3wb0i9FwFllkPVEKI/CFfjwEyGo3ExcXh6uqabvvVq1fx9vamdOnS9O7dm5CQkGeeR6fTERsbm+6V61xKkupekWuWFvx1fSNGJedmbf6knWks0B+n73Lhrhk+qxAv6PzdGDrP2s+eK5FYW6iZ2b0mE1+r8kLhByBeH098SjylHEqy5EEKxSODwLE49FkHtq7PP4EQIt/K1wFo2rRpxMfH061bt7Rtfn5+BAQEsGXLFn7++WeCg4Np2rQpcXFxGZ5nypQpODk5pb18fHxyo/wntKzWD3ujkbup8ZwIO5Fj16lazIlXqxdFUSDgYHCOXUeI7KQ3GBmy9AS37idRwtWWdcMa06VWsZc6Z5UiVZjfeg6LEy3wvHMKbFyg7zpwernzCiHyvjyzGrxKpWL9+vV06dIlU+1XrFjBO++8wx9//EGbNm0ybBcdHU3JkiWZPn06gwYNemobnU6HTvf48fPY2Fh8fHxydjX4p0mOZcLC2qyzt+EN72ZMapu5sVEvIvheAnsuR9DTrwRWWk2OXUeI7HToWhQL9wfz/ds1cLK1eKFzBEYEolapqe5eHRQFNgyD0ytBawP+f4JP/WyuWgiRW7KyGny+HN23atUqBg8ezJo1a54ZfgCcnZ0pX748QUEZP/VkZWWFlZVVdpeZddaOdHarxbrkS2wLPcC41GSstdY5cilfNzt83Xxz5NxCZJeI2GSCIuJp9PCx9oZlitCwTJEXPt+xsGMM3zEcrVrLsg7LKHNsiSn8qDTQbYmEHyEKkXx3C2zlypUMGDCAlStX0qlTp+e2j4+P59q1axQtmj8WLqxd6x2K6VNJUAzsuvFPrlwz1WDk9gN5IkzkLTfuJdDpp/28s/Q4QREZ38LOrH239zFs+zCSUpOoWqQqRc/9CQd+MO18fRaUb/fS1xBC5B9mDUDx8fEEBgYSGBgIQHBwMIGBgWmDlseOHUu/fv3S2q9YsYJ+/frx/fff4+fnR1hYGGFhYcTExKS1GTNmDHv27OHGjRscPHiQN954A41GQ8+ePXP1s70odZlWvJpi+jmnVoj/twt3Y3llxl4GLzmO0Zgn7oYKAUBxFxvKuNtR3MUWrfrl/qrafnM77+96H51BRwufFvzk0QLb7RNNO9t8BTV7vXzBQoh8xay3wI4fP07Lli3T3o8ePRoAf39/AgICCA0NTfcE1/z580lNTWX48OEMHz48bfuj9gC3b9+mZ8+eREVF4e7uTpMmTTh8+DDu7u6586FelkZL55KvcOvGRl63SM3xy3k7W3MvXodWoyY4KoEy7vY5fk0hMpKUYkCjVmGpVaPVqJnTuw5WWjV2Vi/+V9XGaxv58sCXGBQD7Uu1Z3LRNlisehh4GgyHxqOyqXohRH6SZwZB5yVZGUSVI8LOwdzGoLGEjy7n+OO4x27cp1JRR+xf4ktGiJcVEpXI0OUn8PN1ZeJrVbLlnHtv72XEjhEoKLxe5nW+Kvk6mqWvgz4Bqr0Nb8yHl+xdEkLkHVn5/pb/8/Mir6rgWQ0MKXB+fY5frl4pVwk/wqx2XY7g1Z/2cTE0lr/O3CUqPnsWBa7vVZ/6XvXpUaEHkyr0RbOyuyn8lGkFr8+R8CNEISb/9+dVNXpw1cKC6ed+4eqDq7lySUVR2HIujARdzt96EwLAaFT4ccdVBgYcIzY5lVolnNk4sglF7F/uqcxHHdvWWmtmt5nNuEoDUC9/CxKjwLs2dFsGWsvs+AhCiHxKAlBeVe1tfnZxYrE6gT9yYTA0wPurAnl3+QkW7ZfJEUXOi0nSM2TZcab/cwVFgd5+JVg1pAFFnWxe+JyKovDDyR+YcXJGWgiySklC9etbEBMCrmWg9xqwkrFuQhR2EoDyKgdPOjuUA2DTja2kGnO+V6ZtZU8A5u+9zv2ElBy/nii8LofF8fqs/Wy/GIGlVs13b1Xn2zeqvdSknEbFyP8d+z8WnF3A4nOLOR15GvRJsLInRJwHey/oux7s3LLxkwgh8isJQHlYk+oDcDEYuGdM5vCdQzl+vVerFaVyUUfidKn8vDvjiSOFeBkbT9+ly+wD3IhKpJizDb+/24hudV9u+RmD0cCkQ5P49eKvAHzh9wU13arB74Mh5CBYOUKf38GlZHZ8BCFEASABKA+zqPwaHZJMPT9/nluc49dTq1V80t60UOqSQze5G52U49cUhYfeYOTrvy4wcuUpkvQGmpR1Y+PIJlQr7vRy5zXqGbd/HL9f/R21Ss03jb+he4VusGk0XPoLNFbQc6Xp4QIhhHhIAlBeZmHDa14NAdgZcYL4lPgcv2Tz8u74+bqSkmrkh+25M/haFHyRcTr6LDjCwofjy4a1KMOSgfVxtXu5gchJqUmM3jWav4P/RqvS8l2z73i97OuwewqcCACVGt5cAKWaZMOnEEIUJBKA8rjKtd/BN0WPDiP/XN+U49dTqVR80r4iAGtO3CIoIudDlyj4YpL0nL8bi52lhrl9avNp+4po1KqXPu+piFPsub0HS7UlM1vOpF2pdnD0F9jzf6YGnb6Hyq+99HWEEAWPBKA8TlWyEa8ZLHA2GEi6eyJXrlmnpAttK3tiVOD7bZdz5Zqi4EnWG9J+Luthz6xetfhjRGPaV82+dfkaeTfiiwZfsKDdApr7NDfNm/X3x6adLcZB3YHZdi0hRMEiASivU6vpVfYtdobcoVdo7j2e/nG7CqhVsPlcGKdvRefadUXBsPNSOC2n7WbPlci0bS0qeFDWw+Glz30m8gyh8aFp77tV6EYtj1pwfQ+sGwIoUHcQNP/kpa8lhCi4JADlA7a1+mABELQD4sJz5ZrlPR3oWrs4AP+35RKyYorIin1X7xEak8z8vdey9bw7Q3YyaOsghm0fRozu8SLIhJ6GVb1Ns6dXeg06TgXVy99iE0IUXBKA8oMiZaB4fYyKgcBjs3ItjHzQphyWGjUHr0WxP+herlxT5E+KohD/rxnEP25XgdFty7OgX71su8aqS6v4cPeHJBuSKeZQDAu1hWnH/WBY/hakxEGpptD1F1C/+HxCQojCQQJQPmGs3p03i3nR99YGztw7kyvXLO5iS58GpnlTvttyGaNReoHEk8Jjkxmy7AQDFh9N+zNia6nl/dblsLF8+SBiVIzMODGDb498i1Ex8ma5N/mh5Q/YWthC5GUI6AQJEab183r8ChbWL31NIUTBJwEon1BX7UrFh4NKN54NyLXrDm9ZBicbC6oXd0KXasy164q8T1EUVh+7RZvpe/jnQjinQqI5eyfm+QdmQYohhbH7xrLo3CIARtYayYSGE9CqtXD7OCxqB7F3wK089FkL1i83p5AQovCQJcDzC1tXOjtX5S/9NTbf3sMnhhQsNTm/mGMReysOfNZKVosX6dy6n8i49WfZd9V0a7R6cSe+e6s6Fb0cs/U6U49NTZvjZ2KjiaY5fsA0Hu63vqaV3YvVgV5rwK5Itl5bCFGwSQ9QPuJXazAeqanEKnr2huzKtetK+BGPGI0KAQeCaTdzL/uu3sNKq2Zsh4qsG9Yo28MPwDvV36Gsc1lmt5n9OPyc+x1WdDeFn9Itod+fEn6EEFkmASgf0ZR7hU46022oP88F5Pr1L4bGMvq3QO7F63L92sL8rkXG023eISZuvEBiioH6pVzZPKopQ5uXQavJvr9KopOj0372sPVgbee1NPJuZNpw9BdYOwiMeqjSFXqtlpXdhRAvRAJQfqK1pHPxVgDsu3+eB8kPcu3SBqPC0GUnWHfqDt/8dSHXrivML9VgZM7uIDr8sI/jNx9gZ6nh69ersGpIA0q7Z2/4OBx6mI7rOvL39b/TtmnUGlAU2DUF/h4DKFBvsGmJC23O3wYWQhRMEoDymXJ1BlNJl0IqCjuv5fzSGI9o1CoWD6hHs/LujO9cJdeuK8zrwt1Yusw5wHdbLpOSaqRZeXe2ftiMvg1Loc6GpSz+beO1jQz7Zxhx+jj+uPbH4+kejAZT8NnzP9P7FmOh4zR51F0I8VJkcEd+412b0QY7LO/eppYudx9LL+Nuz9KB9dNt23/1Ho3LFkElk84VSCuO3uTcnVicbCz48tXKvFm7WLb/XiuKwoKzC/jx1I8AdCjVgW+afGO6TmoKrB8K59cBKtMEh/XfydbrCyEKJwlA+Y1KRYOqvWHHJDjzG9Tua7ZSVh4NYey6s3StVYzJXathbSH/Ii8IDEYlbaHST9tXxKiYJsX0cMj++XVSjalMPjKZNVfWADCg6gA+qP0BapUadPHwWx+4vgvUFtB1HlR9M9trEEIUTnILLD+q1g1QwY19KA9umq0Mo2L6olx36g49fzlMRFyy2WoRLy8xJZWv/7rAgIBjabefHKwtmPxGtRwLPx/s+oA1V9agQsU4v3GMrjPaFH4SomDpa6bwY2EHvX6T8COEyFYSgPIjZx8e+DbimyIudNvSF6NingkKe/uVZOnA+jjZWHAqJJousw5w/m72ToQnck9ErI7lh2+y90okh6/fz/HradVayjqXxUpjxYyWM+hZsadpR/QtWNwe7pwAG1fw3whlW+d4PUKIwkWlyCqXT4iNjcXJyYmYmBgcHbN/bpPsoDu5lJaB/0ecRs3CVxZQv6if2WoJvpfAoCXHuB6ZgI2Fhhnda9C+alGz1SMyR1EUToY8oHYJl7RxPb8dC8HD0ZqWFTxypQajYuRm7E18nXxNGyIvw7I3TLM7OxaDvuvBvUKu1CKEyP+y8v0tPUD5lFWVN2iXZJqP589cXBrjaXzd7Fj/XmOalnMjSW/g3eUnmbXzqqwgn0cZjQqbz4by2qwDvPnzoXS9Pd3rlcjR8BMYEcionaPQGUx/dtUq9ePwc/sELGr/eGmLQdsk/AghcowEoPzKyoHX3OsC8E/YIRL1iWYtx8nGgsX969G/USkApm27wqhVgSQ/XL9MmF9KqpHVx2/RZsYehv16krN3YrCx0HAjKiFXrr8jZAeDtw1m562dzD8zP/3OoB2wpDMk3TctbTFwKzgVz5W6hBCFkzwFlo/VrP0OxXcP57YF7LyxjVfLdTFrPVqNmomvVaG8pwPj/zjHn6fvcjMqgfn96uLpKCt0m0tSioFVx0L4Ze917saYBqo7Wmvp36gU/Rv74mqX85MJrry0kilHpqCg0Lx4cwZVHfR457nfYd1Q0+zOZVpBt2Uyu7MQIsfJGKCnyA9jgAAwGvh5bjXm2Glo5FiOeW+sM3dFaQ5eu8d7v54kOlGPl6M1v/SrS7XislJ3bopJ1LP00A0WH7zB/YQUANwdrBjcxJdefiVwsLbI8Rr0Bj3fHfuOVZdXAfB2+bcZ5zfOtJo7mJa2+PtjQDEtbfHGPJndWQjxwrLy/S09QPmZWsOrvh2YE7GNw7FXCU8Ix9PO09xVAdCojBsb3mvM4KXHuRYZT2hMkgSgXBIRl8zC/cH8ejiEeF0qAD6uNrzbvAxv1i6ea/M1hSeEM3rPaM5EngFgVO1RDKo6yDTgWlFg9/8ez+5c7x3o8B2o5a68ECJ3SADK53xqD6L92vUUMyhokmMgjwQggFJudqx7rxH7rtzjlSpe5i6nUHiQkELz73aT9HDsVUUvB4a1KEOnakWzdcHSzEhVUrkZexMHSwf+1/R/NCvezLTDaITNn8CxX0zvW4yF5p+CzCYuhMhFEoDyO8/KTNUWh8jTcG0PFClv7orScbS2oFP1x4/E336QyJzd1/iyU2VsLGXm6OwQFpOMl5NpjJWLnSWtKnoQFpvMey3K0Kqih9mWKSlmX4wfWv6Ah40HPo4+po2ytIUQIo+Q/uaCoMbDCeROrzRvHc9hNCq89+tJVhwJ4cs/zpm7nHwvJdXI4CXHaPx/O7n5rye5pr1dg7XvNqR1Jc9cDT8J+gQ+2v0Ru2/tTttWx7PO4/Cji4cV3UzhR20Bby2S8COEMBuzBqC9e/fSuXNnvL29UalUbNiw4Znt161bR9u2bXF3d8fR0ZGGDRuydevWJ9rNnj2bUqVKYW1tjZ+fH0ePHs2hT5BHVH2LVJWGvffPszFw/vPbm4lareLzjpWo4OnA6LZ5q6cqP7LUqtEbFIyKwoGgqLTtNpaaXO/1uR59nZ6berLt5jYmHJzw5LQM/13aovdqqNo1V2sUQoh/M2sASkhIoEaNGsyePTtT7ffu3Uvbtm35+++/OXHiBC1btqRz586cOnUqrc1vv/3G6NGjmTBhAidPnqRGjRq0a9eOiIiInPoY5mfvzsEyDRju5cG0s/PRG/XmrihDfqWLsHlUU7ydbdK2BUXEmbGi/MFgVPj7bChd5xwgLObxmmtfdKrEjtHN6eVXwmy1bb2xlZ6behIcE4yHrQc/tvoRWwvbxw2iQ55c2qJMK7PVK4QQkIceg1epVKxfv54uXbpk6bgqVarQvXt3xo8fD4Cfnx/16tVj1qxZABiNRnx8fBg5ciSfffZZps6Zbx6D/xf92bW0OTae+xoNM5tPp3WptuYuKVO2nAtl2K8nGdGyLB+2KY9aLQNhH4lN1nPg6j12X45k95UIwmNNsycPbVaasR0rmbk602KmM0/MZMmFJQDU86rH1GZTKWJT5HGj67thzQDTBIeOxR8ubSG9f0KInFFoHoM3Go3ExcXh6uoKQEpKCidOnGDs2LFpbdRqNW3atOHQoUMZnken06HT6dLex8bG5lzROcSi4qu8vmccix00fH1wAtU8auBhmzvrOb2MC6FxKAr8tDOIoIh4vu9WA1vLfP3H8oUpisKlsDhT4LkcwYmbD0g1Pv73yb8nLzQ3nUHHu/+8y/Hw4wAMqDKA92u//3h+H0WBQ7Pgn/GgGMG7FnT/FZyKmbFqIYR4LF9/00ybNo34+Hi6desGwL179zAYDHh6pn8U3NPTk0uXLmV4nilTpvDVV1/laK05zsKaYSU7sD9sC1eJY8zu0SxsvxgLdc5PdvcyRrctj4+LDZ+vP8fmc2EcDb5P03JuNC3nTtNybngU8Bmk45L1HAiKYvflCHZfjiQsNjnd/tJudjSv4E7LCh7U93XNtTl8nsdKY0UZ5zJciLrA142/5pVSrzzemZIAf440zfAMULM3dJoOFgX791IIkb/k2wC0YsUKvvrqK/744w88PF6up2Ps2LGMHj067X1sbCw+Pj4vW2Kus2n5BTPmbaKHq5FTkaeZfnw6n9b/1NxlPdfbdX3wdbPjvV9PEhGnY0PgXTYE3gWggqcDTcu50aScG36+RfL9o/OKoqAopN3qm73rGnP3XEvbb22hpmHpIrSo4EGLCu6ULGJnrlKfoCgKyYZkbLSm8Vuf1PuEvpX7UtKx5ONG94Phtz4Qfg7UWmj/P6g3WOb4EULkOfkyAK1atYrBgwezZs0a2rRpk7bdzc0NjUZDeHh4uvbh4eF4eWU8EZ+VlRVWVlY5Vm+usXenZMcZfLvxHUZ5urPq0kp6VOyR/gsqj6pbypX9n7biZMgD9l2NZN/Ve5y9E8Pl8Dguh8exYH8wlho19XxdWNS/Hlba/BeEpv9zhbXHb/HNG1VpVdHUS9mygjtbz4fRvLw7LSq406B0kTzTy/NvSalJTDo0icjESOa2nYtWrcVSY5n+z1bQDlg7EJKjwc4Dui2Bko3MVrMQQjxLvgtAK1euZODAgaxatYpOnTql22dpaUmdOnXYsWNH2mBqo9HIjh07GDFihBmqNYNKnWl18XU+vLmJGpZulLR2M3dFmWapVdOgdBEalC7Cx+1MsxofuHaPfVfuse9qJHdjkomKT0kXfubsDsLDwZq2lT1xsskbt/sURSEoIp49VyLp36hU2gzMUfE67sYks+dyZFoAqu/ryq4xLcxY7fOFxIbw4e4PufLgChqVhsCIQOp61X3cQFHgwEzYMck03qdYXei+DBy9zVazEEI8j1kDUHx8PEFBQWnvg4ODCQwMxNXVlRIlSjB27Fju3LnD0qVLAdNtL39/f3744Qf8/PwICwsDwMbGBicn0zpTo0ePxt/fn7p161K/fn1mzpxJQkICAwYMyP0PaC4d/o+BP++DyGD4ZwJ0mmbuil6Ii50lr1b35tXq3iiKwrXIBB4kpqTtT9Yb+GH7VXSpRraPbpYWgMJjk3G0tsiV22W6VAMxSXpiEvXciEpMG8tzJzoJgJo+ztQtZRqk36dBSdpU9qRh6cdPSZlrlubM2n1rN+P2jSNOH4ertSvTmk9LH3508fDHe3DhD9P72v2g4zTQFoAeVSFEgWbWAHT8+HFatmyZ9v7ROBx/f38CAgIIDQ0lJCQkbf/8+fNJTU1l+PDhDB8+PG37o/YA3bt3JzIykvHjxxMWFkbNmjXZsmXLEwOjCzQbZ3h9Fix7A479wrUStdmgC2V0ndF5/gs3IyqVirIe9um26fRG3mlamouhsZRxf7zv678usO18OHVLuaQNpq5c1DHDR+wVRcGogObh/rCYZE7cfICdlYYWFR6PLxu58hQRscmmwJOkJzpRn7bm1n896s369y93paKOVCqaP6ZVMBgNzDk9h/lnTBNr1nCvwffNv0+/2G7UNVjVGyIvmmZ27jgV6haif2gIIfK1PDMPUF6SH+cBeqpNHxF/fCHtShQnVq1iTN0x+FfxN3dVOe7Vn/Zx7k76qQyK2FnSoEwRLDVqohNTiH7YaxP9MMz81LMWHauZ1izbdCaU4StOUq+UC2vefTyGpf6324mI0/FfKhU42VjgZm9FozJFaFHBnYal3fL1gO2vD33N6iurAehVsRdj6o7BQvOvW4xXtsG6wZAcA/ZepltePvXNVK0QQpgUmnmAxHO0nYT9tZ2MvB/Bt26uzDgxg8pFKlPPq565K8tRG0c04fq9BPZfNY0dOnQtiqiEFDadCc3wmOjEx7NneztbU9/Xlcr/6a354tXKqABnWwucbCxwtrHEydYCByttgZvAsUfFHmwP2c6YumPoXKbz4x1GI+z/HnZ+Cyjg4wfdloJDxg8ZCCFEXiQ9QE9RYHqAAEKOoCxuzzg3F/6yt6OIdRFWd16dLyZJzC4pqUYCb0Vz/OZ9LNRqnGwscLK1wNnGAmdbS5xtLXCxtcRSW7jXBr4ec53STqXT3ifqE9MvaaGLg/XvwqW/TO/rDjI95q61zOVKhRDi6bLy/S0B6CkKVAAC+Gc8SQd/pHfxYlzVqqnlUYuF7Rbm+UkSRe5I1Cfy3bHv+CPoDxa1X0Qtj1pPNrp31TTe595l0FhCp+9NA56FECIPycr3d+H+J29h0fJzbNwrMTM0DAfUnIo4xfTj081dlcgDDt09RNc/u/L71d8xKAYuRF14stGlv+GXVqbw4+ANAzZL+BFC5HsSgAoDrRW8MZcSRvj24SSRl+5fIsWQ8pwDRUEVmxLLhIMTGPLPEO7E36GoXVHmtZ1H70q9HzcyGmH3/2BVT9DFQolGMHQPFK+b8YmFECKfkEHQhUXRGtD8M1ru+oafoxJo8NpXaDUydqMw2nt7L18d/IqIpAgAelbsyajao7Cz+NeyG8kxsG4oXNlsel9/KLT7FjRy21QIUTBID1Bh0uRDKFaHJrFRaP8aZZrBF0g1ppq5MJGbIhIjiEiKoJRjKZa0X8I4v3Hpw0/EJdMtryubQWMFXX6Gjt9J+BFCFCjSA1SYaLTQZS7MawrXdqI/Np+pShT3ku7xffPv8+0kieLZFEXhXtI93G3dAXiz3JsoKHQu3Rlr7X9WaL+40fSkV0o8OBY3ze9TrLYZqhZCiJwlPUCFjXt5aDMRgGu7v2bN5TX8c/MflpxfYt66RI4ISwhjxM4R9NzUk/iUeMA0q/bb5d9OH36MBtjxtWkl95R4KNXUNN5Hwo8QooCSAFQY1R8KpZpSMTGOT1NNtz5mnpzJsbBjZi5MZBejYmT15dV0+aMLe2/v5X7yfU5FnHp64/gIWNEd9j1cM67BcOi7Aezyz0K6QgiRVRKACiO1Gl6fDZYOdA85y6sOZTEoBj7e8zERiRHmrk68pJDYEAZvG8zXh78mQZ9AdffqrOm8hqbFmz7Z+Px6mO0HQf+A1ga6/gLtJ5tulwohRAEmAaiwcikJ7SejAsZf2E95hxJEJUcxZs8Y9Eb9cw8XeY+iKCw5v4Q3/3yTY2HHsNHa8Em9T1jafillnMukb5wQBWsGwJr+kHQfPKvB4O1QvZtZahdCiNwmAagwq9UXyrXDJjWFGZEPcLCwl0kS8zGVSsWZyDMkG5Lx8/Lj99d+p2/lvmjU/1mU9dImmNMAzq8DlQaafQLv7ASvquYpXAghzED6uQszlQpe+xHmNKBE6Hm+Ld6bz2MCC/xiqQWJ3qAnyZCEo6VpyvexfmNpUqwJXcp2efKpvqQHsPkzOLPK9N69oukRdxnoLIQohGQtsKcocGuBPc+5dbB2AKg0xPhvwKlUM3NXJDLh/L3zfHnwS3wdffm+xffPbnz1H/hzJMSFgkoNjUZCi3FgYf3s44QQIh/Jyve39AAJqNrVtML3ud9x+usjGLoXLGwISwjD3sIee0t7c1co/iU5NZk5gXNYcmEJRsXIvcR7RCRG4GHr8ZTGsbB1HJxaZnrvWgbemAs+9XO3aCGEyGNkDJAw6TgN7L3g3hXY8TXHwo7RbWM3vjzwJdJJmHccDzvOWxvfYvH5xRgVIx19O7Khy4anh5/ru+HnRg/DjwoavAfv7pfwI4QQSA+QeMTWFV77CVa8DYfnYOVdiTh9HNtDtrPk/BL6V+1v7goLtUR9ItNPTOe3y78B4GHrwZcNvqSFT4snG+viYfsEOLbA9N65pGmsT6nGuVewEELkcdIDJB4r/wrU9gcUqu/8jk9rjQJkksS8QEFh7+29gGkpiw2vb3h6+Ll5EOY2fhx+6g6CYQcl/AghxH/IIOinKHSDoP9NF2e6bRIdglKrL5+7OrLx+kaKWBdhdefVT7/VIrJdiiGFTdc38WrpV7F4uAjpsbBjGBUjfkX9njxAn2RayuLwHEAxreP1+iwo0zJ3CxdCCDPKyve39ACJ9KwcTLdLUKE6tYwv3RpS3qU8UclRfLT7I/QGmSQxJ+mNetZeWcur619l/MHx/HHtj7R99bzqPT383DoGc5vA4dmAArX6wHsHJfwIIcQzSAASTyrVxDRgFrDZ9BEzGkzAwcKBwMhAll5YaubiCqZUYyrrr66n8/rOfHXoK0ITQvGw8cBGa/OMg3TwzwRY9ApEBZkGsfdabVrmxNop94oXQoh8SAZBi6dr/SUEbYd7lymxZwbfNvmW7SHb6VWpl7krK1AUReGv638x9/RcQuJCAChiXYTB1QbzdoW3sdJYPf3Au6dg/TCIvGh6X707tP+faTC7EEKI55IeIPF0Fjam+WJUGji/jpax9/m2ybdpPRIxuhg+2fMJ16KvmbnQ/E2lUvHntT8JiQvBxcqFMXXHsPnNzfSp3Ofp4Sc1BXZNhl9am8KPnTt0Xw5d50v4EUKILJBB0E9RqAdB/9euybDn/8DGBd47DA5eAHx//HsCzgegVqnpUrYL79V4D087TzMXm/cZFSM7QnZQ26M2RWyKAHA28ixHwo7Qq2IvbC1sMz447BxseBfCzpreV+4Cnb4HO7ecL1wIIfKBrHx/SwB6CglA/2LQw4LWEHoayrWDXr+BSkVwTDA/nvyR7SHbAbDWWNOnch8GVh2Ig6WDmYvOexRFYdetXcwJnMPlB5fxr+zPmHpjMndwXDgcnQcHfgSjHmxcTcGnatecLVoIIfIZCUAvSQLQf0RchHnNwJBimiyxdr+0XYERgUw/MZ1TEacAcLZyZliNYTJW6CFFUdh3Zx+zA2dzIeoCAPYW9gyqNojB1QY/++DbJ+DIXDi/3hR8ACp0gldngIP0tgkhxH/JWmAie3lUglZfwD/jYfOn8OAm+A0Few9qetRkSfsl7Lq1i5knZxIcE8yVB1fMXXGecOjuIWadmsWZe2cAsNHa0KdSH/yr+ONklcFTWqkpcOEPU/C5c/zxdh8/aDgcKr0G/13lXQghRJZJD9BTSA/QUxgN8OtbcG2n6b3GCmr1hoYjoEgZwPQo94agDTQr3ixtwsTgmGDCEsJo6N3QXJWbzeQjk1l5aSXWGmt6VuxJ/6r9cbXOYKByfAQcXwzHF0J8uGmbxhKqvgn1h0Cx2rlXuBBC5FNyC+wlSQDKgNEAl/+G/TP/1TuhgsqvQeNRUKzOE4eM3DGS3bd308i7ER/W+ZCKrhVzteTcdCL8BE6WTpR1KQtAZGIkAecDGFB1AG42GQxUvnMCjsyHc78/vs1l7wX1BkGd/mAvM28LIURmSQB6SRKAnkNRTGtOHfgBrm59vL1UU2j8AZRtDSoVBqOBqcen8tvl30g1pqJCRafSnRhZayTe9t5mKz+7JOoTORZ2jEOhhzh49yDBMcE0K96M2a1nP/vA1BS4+KfpNtftf62xVrwe+L1rus2ltczZ4oUQogCSAPSSJABlQfgFOPgTnF0NxlTTNs+qph6hKm+AxoJbcbf46eRPbL6xGQALtQU9K/ZkSPUhGY+FycMCzgWw5/YeAiMDSX30mQGtSssb5d5gnN84tOqnDK+Lj4ATAXBsIcSHmbapLUxPc9UfCsWf7EETQgiReRKAXpIEoBcQcxsO/2z6gk+JN21z8jEN3K3VF6zsOX/vPDNOzOBI2BEAPq77Mf2q9Mv4nHlAaHwo56LO0bZk27Rtg7YO4mjYUQCK2RejsXdjGno3pH7R+jhaPuXPy91TcGSe6TaXIcW0zd7TtFJ7nf7yRJcQQmSTfBOA9u7dy9SpUzlx4gShoaGsX7+eLl26ZNg+NDSUjz76iOPHjxMUFMT777/PzJkz07UJCAhgwIAB6bZZWVmRnJyc6bokAL2EpAemHo4jcyEh0rTN2tk0kLf+EBQ7Nw7cPcDKSyuZ3mJ62mzHYQlhuNu4o1FrzFc7j29rHbx7kEOhhwiOCQZgT/c9aQOYd9zcwb2kezTyboSPo8/TT2TQP7zNNQ9uHXm8vVhd022uyq/LbS4hhMhm+eYx+ISEBGrUqMHAgQPp2vX5k7rpdDrc3d354osvmDFjRobtHB0duXz5ctp7lTw2nHtsXKDZGNPTYadXmm6P3b8Ge7+Dgz+iqtmbJo1G0ORf42RSjam8+8+7qFQq3ij7BkVsiuBi7YKLlQsu1i64WrtiqcnZsLArZBdLLyx94raWWqWmmls1opKi0gJQ65KtMz5Rwj04sdgUAuNCH57EwnQ70G8oFK+bkx9DCCFEJpk1AHXo0IEOHTpkun2pUqX44YcfAFi0aFGG7VQqFV5eXpk+r06nQ6fTpb2PjY3N9LEiAxbWUHeAadLES5vgwEzTE0/HF5oCQuXXTeOEvGsRHBNMRFIEcSlxTD0+9YlTdSvfjS8bfglAXEocn+79NC0YOVs5p/3XxdoFb3vvtEfwMxIaH8qh0EPU86qHj4OpByc2JZbj4aYn24rbF6eRdyMaeTeiXtF6T7+t9W+KAreOwsklcHYtGB7+WbLzgLoDTb8ODpn/8yiEECLnFciJEOPj4ylZsiRGo5HatWszefJkqlSpkmH7KVOm8NVXX+VihYWIWmN6TL5SZ7h54OGTY9tMsxufXw++zSjXeBSb39jMissrCHoQRLQumvvJ94nWRROdHI2LtUva6e4l3WPfnX0ZXq57he580eALwLRga/8tprl3XKxdsNHaEBgRyI3YGwCMqTsG/yr+ADQu1pgvG3xJw6INM76t9V/hF+DsGlPoiQl5vN27FvgNgypdQJvBau5CCCHMqsAFoAoVKrBo0SKqV69OTEwM06ZNo1GjRpw/f57ixYs/9ZixY8cyevTotPexsbH4+GTyS1BkjkoFpZqYXuHnTetanVsLwXsheC9OntUY1ngUNB0MGou0wxRFSXdLytXala8afcWD5Aeml+5Bup+97B73tNxPvk9QdNATpTy6rfXvSQndbNzoVqHb8z/Hg5umwcxn10LE+cfbLe1NIa/uINNtLrntKoQQeVqeeQpMpVI9dxD0v7Vo0YKaNWs+MQj6v/R6PZUqVaJnz558/fXXmTq3DILOJdG3Hj85pk8wbbNygpKNHoclr2qmXqQXkKhPJDAykAfJD4jWRROri6WcS7mMn9bKSMI9U2/V2bVw6/Dj7RpLKPcKVHsLyrcHC5sXqlMIIUT2yDeDoHODhYUFtWrVIijoyZ4AYWbOPtB+MjT/OP2TY1c2m14A1k5QsvHjQORZNdOByNbClkbejV6sNl0cXPrbdIvr2k5QDA93qMC3KVR729TjY+PyzNMIIYTImwp8ADIYDJw9e5aOHTuauxSRkUdPjjX+AMLOwI39ptfNg5AcY1p+4/LfprYvEYieKzUFgrabQs/lzZCa9Hifdy1T6KnSFRyLZs/1hBBCmI1ZA1B8fHy6npng4GACAwNxdXWlRIkSjB07ljt37rB06dK0NoGBgWnHRkZGEhgYiKWlJZUrVwZg0qRJNGjQgLJlyxIdHc3UqVO5efMmgwcPztXPJl6ARmta9LNYbWj8PhhSHwaifQ8D0aGnBCLnpwQideavaTSaBmefXWNahT05+vE+1zJQvRtUfQvcymbnJxVCCGFmZh0DtHv3blq2bPnEdn9/fwICAujfvz83btxg9+7dafueNqdPyZIluXHjBgAffvgh69atIywsDBcXF+rUqcM333xDrVq1Ml2XjAHKowypEHb6Xz1EhyAlLn2bzAQiRYHQ06bQc24dxN19vM/eyzSmp9pbULSmDGYWQoh8JN/MBJ1XSQDKJx4FouCHPUQhhx4vw/GItfPjMFS0hqnt2TUQdfVxGysn06P61d42tTPzbNRCCCFejASglyQBKJ8ypJp6dm48IxA9orU2PblV7W0o11bm6xFCiAJAngIThZNGa1pRvXgdaPLBk4Eo9LTpsfpqb0PFTmAt4VYIIQorCUCi4PpvIBJCCCEeysLjMkIIIYQQBYMEICGEEEIUOhKAhBBCCFHoSAASQgghRKEjAUgIIYQQhY4EICGEEEIUOhKAhBBCCFHoSAASQgghRKEjAUgIIYQQhY4EICGEEEIUOhKAhBBCCFHoSAASQgghRKEjAUgIIYQQhY4EICGEEEIUOlpzF5AXKYoCQGxsrJkrEUIIIURmPfrefvQ9/iwSgJ4iLi4OgP9v796DoqrfP4C/F5ZdEFwuiwKr3BQNVESl4aKM0EDijIVRoxMVojEq3i9FaGkmf3xRNJVhmhzJSicnTdOcyawQ0zEui9xCBRlkuGSCqASKgFz2+f3Rb8+wssISIB32ec3sCOd8Pp/9vH3Y8fFwFpydnYd5J4wxxhjrr0ePHsHa2rrXMRIypE0yMhqNBnfu3MHo0aMhkUiey3M+fPgQzs7O+PPPP6FQKJ7Lcz5PnE/cOJ+4cT5x43yGIyI8evQIKpUKJia93+XDV4D0MDExwfjx44fluRUKxYj8AtfifOLG+cSN84kb5zNMX1d+tPgmaMYYY4wZHW6AGGOMMWZ0uAH6j5DL5dixYwfkcvlwb2VIcD5x43zixvnEjfMNDb4JmjHGGGNGh68AMcYYY8zocAPEGGOMMaPDDRBjjDHGjA43QIwxxhgzOtwADZLPPvsMbm5uMDc3h7+/P3Jzc3sdf/LkSXh6esLc3Bze3t746aefdM4TET7++GM4OTnBwsICYWFhKC8v1xnj5uYGiUSi89i1a9egZwMGP9/p06cxb948KJVKSCQSFBUV9Vijra0Na9asgVKphJWVFd544w3cvXt3MGMJhiNfSEhIj/rFxcUNZizBYObr6OhAQkICvL29YWlpCZVKhSVLluDOnTs6azQ0NODtt9+GQqGAjY0NYmNj0dzcPGLyifn198knn8DT0xOWlpawtbVFWFgY1Gq1zhix1g8wLJ+Y69ddXFwcJBIJDhw4oHNczPXr7ln5BqV+xAbs+PHjJJPJ6Msvv6QbN27Q8uXLycbGhu7evat3fGZmJpmamlJycjKVlJTQtm3byMzMjK5duyaM2bVrF1lbW9MPP/xAf/zxB0VERJC7uzu1trYKY1xdXSkxMZFqa2uFR3NzsyjyHT16lHbu3ElpaWkEgAoLC3usExcXR87OzpSRkUF5eXkUEBBAs2fPHjH5goODafny5Tr1a2pq+s/na2xspLCwMDpx4gTdvHmTsrOzyc/Pj3x9fXXWmT9/Pvn4+FBOTg5duXKFPDw8KCoqasTkE/Pr79ixY5Senk4VFRV0/fp1io2NJYVCQfX19cIYsdbP0Hxirp/W6dOnycfHh1QqFe3fv1/nnJjrZ0i+wagfN0CDwM/Pj9asWSN83tXVRSqVipKSkvSOX7x4MS1YsEDnmL+/P61cuZKIiDQaDTk6OtKePXuE842NjSSXy+nbb78Vjrm6uvb4ohgKg52vu8rKSr0NQmNjI5mZmdHJkyeFY6WlpQSAsrOzB5Cmp+HIR/RPA7Rhw4YB7d0QQ5lPKzc3lwBQdXU1ERGVlJQQALp69aow5vz58ySRSOivv/4aSJwehiMf0ch4/Wk1NTURALpw4QIRjbz6PZ2PSPz1u337No0bN46uX7/eI8tIqF9v+YgGp378LbABam9vR35+PsLCwoRjJiYmCAsLQ3Z2tt452dnZOuMBIDw8XBhfWVmJuro6nTHW1tbw9/fvseauXbugVCoxc+ZM7NmzB52dnYMVDcDQ5DNEfn4+Ojo6dNbx9PSEi4tLv9bpy3Dl0zp27Bjs7e0xbdo0bN26FS0tLf1eozfPK19TUxMkEglsbGyENWxsbPDiiy8KY8LCwmBiYtLjWxEDMVz5tEbC66+9vR2HDh2CtbU1fHx8hDVGSv305dMSa/00Gg2io6MRHx+PqVOn6l1DzPXrK5/WQOvHvwx1gO7fv4+uri44ODjoHHdwcMDNmzf1zqmrq9M7vq6uTjivPfasMQCwfv16zJo1C3Z2dsjKysLWrVtRW1uLffv2DTiX1lDkM0RdXR1kMlmPf3D6u05fhisfALz11ltwdXWFSqVCcXExEhISUFZWhtOnT/cvRC+eR762tjYkJCQgKipK+EWGdXV1GDt2rM44qVQKOzs70dVPXz5A/K+/H3/8EW+++SZaWlrg5OSE9PR02NvbC2uIvX695QPEXb/du3dDKpVi/fr1z1xDzPXrKx8wOPXjBkjENm/eLHw8ffp0yGQyrFy5EklJSSP2R6aPJCtWrBA+9vb2hpOTE0JDQ1FRUYGJEycO484M19HRgcWLF4OI8Pnnnw/3dgZdb/nE/vp76aWXUFRUhPv37yMtLQ2LFy+GWq3u8Q+nWPWVT6z1y8/PR0pKCgoKCiCRSIZ7O4PO0HyDUT/+FtgA2dvbw9TUtMe7k+7evQtHR0e9cxwdHXsdr/2zP2sCgL+/Pzo7O1FVVdXfGM80FPkM4ejoiPb2djQ2Ng5onb4MVz59/P39AQC3bt0a0DrdDWU+bXNQXV2N9PR0nasjjo6OqK+v1xnf2dmJhoYG0dSvt3z6iO31Z2lpCQ8PDwQEBODw4cOQSqU4fPiwsIbY69dbPn3EUr8rV66gvr4eLi4ukEqlkEqlqK6uxnvvvQc3NzdhDbHWz5B8+vyb+nEDNEAymQy+vr7IyMgQjmk0GmRkZCAwMFDvnMDAQJ3xAJCeni6Md3d3h6Ojo86Yhw8fQq1WP3NNACgqKoKJicmg/g9uKPIZwtfXF2ZmZjrrlJWVoaampl/r9GW48umjfau8k5PTgNbpbqjyaZuD8vJyXLhwAUqlsscajY2NyM/PF45dvHgRGo1GaPQGw3Dl00fsrz+NRoMnT54Ia4i5fvp0z6ePWOoXHR2N4uJiFBUVCQ+VSoX4+Hj88ssvwhpirZ8h+fT5V/Ub0C3UjIj+eRugXC6nr7/+mkpKSmjFihVkY2NDdXV1REQUHR1NW7ZsEcZnZmaSVCqlvXv3UmlpKe3YsUPv2+BtbGzo7NmzVFxcTAsXLtR5G3xWVhbt37+fioqKqKKigr755hsaM2YMLVmyRBT5Hjx4QIWFhXTu3DkCQMePH6fCwkKqra0VxsTFxZGLiwtdvHiR8vLyKDAwkAIDA0dEvlu3blFiYiLl5eVRZWUlnT17liZMmEBz5879z+drb2+niIgIGj9+PBUVFem8DfXJkyfCOvPnz6eZM2eSWq2m33//nSZNmjRkb8N93vnE/Pprbm6mrVu3UnZ2NlVVVVFeXh4tW7aM5HI5Xb9+XVhHrPUzJJ+Y66ePvndEibV+huQbrPpxAzRIUlNTycXFhWQyGfn5+VFOTo5wLjg4mGJiYnTGf/fddzR58mSSyWQ0depUOnfunM55jUZD27dvJwcHB5LL5RQaGkplZWXC+fz8fPL39ydra2syNzcnLy8v+t///kdtbW2iyPfVV18RgB6PHTt2CGNaW1tp9erVZGtrS6NGjaLIyEidBknM+Wpqamju3LlkZ2dHcrmcPDw8KD4+fkh+DtBg59O+tV/f47fffhPGPXjwgKKiosjKyooUCgUtW7aMHj16NCLyifn119raSpGRkaRSqUgmk5GTkxNFRERQbm6uzhpirZ8h+cRcP330NUBirZ8+T+cbrPpJiIgMv17EGGOMMSZ+fA8QY4wxxowON0CMMcYYMzrcADHGGGPM6HADxBhjjDGjww0QY4wxxowON0CMMcYYMzrcADHGGGPM6HADxBhjjDGjww0QY4wxxowON0CMMcYYMzrcADFmhJYuXYrXXnttuLdh1EJCQrBx48bh3gZjRosbIMaY0Wlvbx/uLQyakZSFseeJGyDGmI6ff/4ZQUFBsLGxgVKpxCuvvIKKigqdMadOnYK3tzcsLCygVCoRFhaGx48f93nuyZMnWL9+PcaOHQtzc3MEBQXh6tWrve4nJCQEa9euxdq1a2FtbQ17e3ts374d3X+Pc1971q6xceNG2NvbIzw83OCsISEhWLduHTZu3AhbW1s4ODggLS0Njx8/xrJlyzB69Gh4eHjg/PnzwhyNRoOkpCS4u7vDwsICPj4+OHXqlHB+6dKluHz5MlJSUiCRSCCRSFBVVdXnvN6y6JObm4uQkBBYWFjA09MTeXl5OHToECIiInr9O2fMGHADxBjT8fjxY2zevBl5eXnIyMiAiYkJIiMjodFoAAC1tbWIiorCu+++i9LSUly6dAmvv/46iKjXcwDwwQcf4Pvvv8eRI0dQUFAADw8PhIeHo6Ghodc9HTlyBFKpFLm5uUhJScG+ffvwxRdfGLxn7RoymQyZmZk4ePCgwfO0c+3t7ZGbm4t169Zh1apVWLRoEWbPno2CggLMmzcP0dHRaGlpAQAkJSXh6NGjOHjwIG7cuIFNmzbhnXfeweXLlwEAKSkpCAwMxPLly1FbW4va2lo4Ozv3Oa+3LE/LyclBcHAwFixYgOLiYnh5eSExMRG7d+/Gzp07+/w6YGzEI8aY0YmJiaGFCxcaNPbevXsEgK5du0ZERPn5+QSAqqqqeozt7VxzczOZmZnRsWPHhGPt7e2kUqkoOTn5mc8fHBxMXl5epNFohGMJCQnk5eVl8J6Dg4Np5syZ/c6qnRsUFCR83tnZSZaWlhQdHS0cq62tJQCUnZ1NbW1tNGrUKMrKytJZOzY2lqKionTW3bBhg/B5f+YZkiUwMFBnjydOnCATExOKjIzscy5jxoCvADHGdJSXlyMqKgoTJkyAQqGAm5sbAKCmpgYA4OPjg9DQUHh7e2PRokVIS0vD33//3ee5iooKdHR0YM6cOcJzmZmZwc/PD6Wlpb3uKSAgABKJRPg8MDAQ5eXl6OrqMmjPAODr69vvrFrTp08XPjY1NYVSqYS3t7dwzMHBAQBQX1+PW7duoaWlBS+//DKsrKyEx9GjR3t8e627/szTl6W727dvIzs7G3FxccIxqVQKIuKrP4z9P+lwb4Ax9t/y6quvwtXVFWlpaVCpVNBoNJg2bZpws62pqSnS09ORlZWFX3/9Fampqfjoo4+gVqvh7u7+zHPDuWcAsLS0/FfzgH8ate4kEonOMW1zptFo0NzcDAA4d+4cxo0bpzNPLpc/M0N/5unL0p22oZw1a5ZwrKysDH5+fjqNG2PGjK8AMcYEDx48QFlZGbZt24bQ0FB4eXkJV3C6k0gkmDNnDnbu3InCwkLIZDKcOXOm13MTJ04U7lvR6ujowNWrVzFlypRe9/V0A5WTk4NJkybB1NTU4D3/26z9NWXKFMjlctTU1MDDw0Pn4ezsLIyTyWTCFaz+zDNEU1MTTE1NhcasoaEBe/fuxahRowacj7GRgq8AMWakmpqaUFRUpHPM1tYWSqUShw4dgpOTE2pqarBlyxadMWq1GhkZGZg3bx7Gjh0LtVqNe/fuwcvLq9dzlpaWWLVqFeLj42FnZwcXFxckJyejpaUFsbGxve61pqYGmzdvxsqVK1FQUIDU1FR8+umnBu9Zn387ry+jR4/G+++/j02bNkGj0SAoKAhNTU3IzMyEQqFATEwMAMDNzQ1qtRpVVVWwsrKCnZ2dQfMMMWPGDHR1dSE5ORmLFi3Chg0b4ObmhpKSElRXV8PV1XXAORkTveG+CYkx9vzFxMQQgB6P2NhYSk9PJy8vL5LL5TR9+nS6dOkSAaAzZ84QEVFJSQmFh4fTmDFjSC6X0+TJkyk1NbXPc0REra2ttG7dOrK3tye5XE5z5syh3NzcXvcaHBxMq1evpri4OFIoFGRra0sffvihzk3Rfe356RuODZ33rLmurq60f/9+nWPd52k0Gjpw4AC98MILZGZmRmPGjKHw8HC6fPmyML6srIwCAgLIwsKCAFBlZaVB856V5WmJiYmkVCrJ3Nycli5dSvfv36dZs2aRp6dnn3MZMwYSom4/TIMxxv5jQkJCMGPGDBw4cGC4t8IYG0H4HiDGGGOMGR1ugBhjjDFmdPhbYIwxxhgzOnwFiDHGGGNGhxsgxhhjjBkdboAYY4wxZnS4AWKMMcaY0eEGiDHGGGNGhxsgxhhjjBkdboAYY4wxZnS4AWKMMcaY0eEGiDHGGGNGhxsgxhhjjBmd/wPlLii6pkEgWQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(alphas, cv_risks, \"-.\", label=\"5-fold CV\")\n", + "plt.plot(alphas, alo_risks, \"-\", label=\"RandALO\")\n", + "plt.plot(alphas, true_risks, \"--\", label=\"True risk\")\n", + "\n", + "plt.xlabel(r\"Lasso parameter $\\alpha$\")\n", + "plt.ylabel(\"Risk\")\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example 2: Logistic regression\n", + "\n", + "RandALO can also be applied to classification problems for logistic regression models. Let's repeat the experiment above but now for binary classification. We'll generate binary labels using the same data as before and this time generate test data to evaluate risk estimation quality since we don't have a closed-form risk expression." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# y = (rng.uniform(0, 1, (n,)) < scipy.special.expit(5 * X @ beta)).astype(float) * 2 - 1\n", + "n = 8000\n", + "p = 2000\n", + "s = 1000\n", + "X = rng.integers(0, 2, (n, p)) * 2 - 1\n", + "beta = np.zeros(p)\n", + "beta[:s] = rng.normal(0, 1 / np.sqrt(s), (s,))\n", + "y = np.sign(X @ beta)\n", + "\n", + "n_test = 20000\n", + "X_test = rng.integers(0, 2, (n_test, p)) * 2 - 1\n", + "y_test = np.sign(X_test @ beta)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that we have to use $\\pm 1$ binary labels with RandALO.\n", + "\n", + "We'll use the zero–one loss as our risk funciton, which we'll have to define.\n", + "For classification, the value of `y_hat` passed to the risk function is not the output labels, but instead the real-valued output from the `decision_function` method from scikit-learn. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Performing 5-fold CV with scikit-learn\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 10/10 [01:13<00:00, 7.40s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5-fold CV computed in 73.970 seconds\n", + "Applying RandALO\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 10/10 [00:30<00:00, 3.07s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RandALO computed in 30.726 seconds\n" + ] + } + ], + "source": [ + "def zero_one_risk_fun(y, z):\n", + " return torch.mean(torch.lt(y * z, 0).float()).item()\n", + "\n", + "# select model and a meaningful range of hyperparameters\n", + "kwargs = {\n", + " \"penalty\": \"l1\",\n", + " \"solver\": \"liblinear\",\n", + "}\n", + "Cs = np.logspace(-2, 1, 10)\n", + "\n", + "# first compute cross-validation\n", + "# ==============================\n", + "print(\"Performing 5-fold CV with scikit-learn\")\n", + "cv_risks = []\n", + "tic = time.monotonic()\n", + "\n", + "for C in tqdm(Cs):\n", + " lr = LogisticRegression(C=C, **kwargs)\n", + " cv_results = cross_validate(lr, X, y, cv=5, scoring=\"accuracy\")\n", + " cv_risks.append(1 - np.mean(cv_results[\"test_score\"]))\n", + "\n", + "toc = time.monotonic()\n", + "cv_time = toc - tic\n", + "print(f\"5-fold CV computed in {cv_time:01.3f} seconds\")\n", + "\n", + "# next, use RandALO, and hold onto the models to evaluate test error\n", + "# ==================================================================\n", + "print(\"Applying RandALO\")\n", + "alo_risks = []\n", + "models = []\n", + "tic = time.monotonic()\n", + "\n", + "for C in tqdm(Cs):\n", + " lr = LogisticRegression(C=C, **kwargs)\n", + " lr.fit(X, y)\n", + " alo = RandALO.from_sklearn(lr, X, y)\n", + " alo_risks.append(alo.evaluate(zero_one_risk_fun))\n", + " models.append(lr)\n", + "\n", + "toc = time.monotonic()\n", + "alo_time = toc - tic\n", + "print(f\"RandALO computed in {alo_time:01.3f} seconds\")\n", + "\n", + "# evaluate test error\n", + "test_risks = []\n", + "for model in models:\n", + " test_risks.append(1 - model.score(X_test, y_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This time we can compute the RandALO risk estimates in less than half the time of 5-fold CV. And once again, our risk estimates are just as good if not better." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAG5CAYAAAB1OMuOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACGS0lEQVR4nO3dd3yT1f7A8U9Gk3TvQUuh7D0LVJYgliGIojhAEURcXOQKdSDeK6h4BRGRe5XrRMSfesE9QFmVJdsisvcoowNaukfa5Pn9ERpIB01L0rTl+/b1vJo8z3nO802IzbfnnOcclaIoCkIIIYQQwkrt6gCEEEIIIWobSZCEEEIIIUqRBEkIIYQQohRJkIQQQgghSpEESQghhBCiFEmQhBBCCCFKkQRJCCGEEKIUSZCEEEIIIUrRujqAuspsNnP+/Hm8vb1RqVSuDkcIIYQQdlAUhezsbMLDw1GrK24nkgSpms6fP09kZKSrwxBCCCFENZw5c4aGDRtWeFwSpGry9vYGLG+wj4+Pi6MRQgghhD2ysrKIjIy0fo9XRBKkairpVvPx8ZEESQghhKhjKhseI4O0hRBCCCFKkQRJCCGEEKIU6WITQgghymEymSgqKnJ1GKKK3Nzc0Gg0112PJEhCCCHEVRRFITk5mYyMDFeHIqrJz8+PsLCw65qGRxIkIYQQ4iolyVFISAgeHh4y110doigKeXl5pKamAtCgQYNq1yUJkhBCCHGZyWSyJkeBgYGuDkdUg7u7OwCpqamEhIRUu7tNBmkLIYQQl5WMOfLw8HBxJOJ6lPz7Xc8YMkmQhBBCiFKkW61uc8S/nyRIQgghhBClSIIkhBBC3MAOHTrETTfdhMFgoHPnznad8/DDDzNixIhrlunfvz9Tpky57vhcRRIkIYQQoh54+eWXUalUNlvr1q0rPW/mzJl4enpy+PBh4uPjayDSK4xGI3PnzqVTp054eHgQFBRE7969Wbx4MUVFRQwfPpwhQ4aUe+6mTZtQqVTs2bPHKbHJXWxCCCFEPdGuXTvWrl1rfa7VVv41f/z4cYYNG0bjxo2dGVoZRqORwYMH89dffzFr1ix69+6Nj48P27ZtY968eXTp0oUJEyYwcuRIzp49S8OGDW3OX7x4Md26daNjx45OiU9akIQQQoh6QqvVEhYWZt2CgoKuWV6lUpGQkMCrr76KSqXi5ZdfBmDv3r0MGDAAd3d3AgMDefzxx8nJyamwntzcXMaOHYuXlxcNGjTgrbfeqjTWBQsWsHHjRuLj45k0aRKdO3emadOmPPDAA2zfvp0WLVpw++23ExwczKeffmpzbk5ODl9//TUTJkyo9DrVJQlSLbNyXzJjPt7O4s0nXR2KEEKIq+QZi6u8FZvM1vOLTWbyjMUUFJnsqrc6jh49Snh4OE2bNuXBBx8kMTHxmuWTkpJo164dzzzzDElJSTz77LPk5uYyePBg/P392blzJ19//TVr167lqaeeqrCe5557jg0bNvDjjz+yevVq1q9fz65du6557S+++ILY2Fi6dOlS5pibmxuenp5otVrGjh3Lp59+iqIo1uNff/01JpOJ0aNHV/KOVJ90sdUySZn5/H7sIgoK43s3cXU4QgghLms7Y1WVz1n4QFeGdbTM5rxqfwqTvtxFTJMAlj3R01qmzxvrSM81ljn31JxhVbpWTEwMn376Ka1atSIpKYlXXnmFvn37sm/fPry9vcs9JywsDK1Wi5eXF2FhYQB89NFHFBQU8Nlnn+Hp6QnAu+++y/Dhw3njjTcIDQ21qSMnJ4dFixbx+eefc+uttwKwZMmSMl1ipR09epT+/ftX+roeeeQR3nzzTTZs2GAtv3jxYkaOHImvr2+l51eXtCDVMv1aBgOw8+Qlcgur9xeEEEKIG89tt93GvffeS8eOHRk8eDC//PILGRkZfPXVVwA8+eSTeHl5WbeKHDx4kE6dOlmTI4DevXtjNps5fPhwmfLHjx/HaDQSExNj3RcQEECrVq2uGe/VLULX0rp1a3r16sUnn3wCwLFjx9i0aZNTu9dAWpBqnSZBnjQK8CAxPY+tx9OIbRta+UlCCCGc7sCrg6t8jk5zpR1icLtQDrw6GHWpSQx/n3bLdcdWHj8/P1q2bMmxY8cAePXVV3n22Wedcq3qaNmyJYcOHbKr7IQJE5g8eTILFy5k8eLFNGvWjH79+jk1PmlBqmVUKpW1FWnDkQsujkYIIUQJD522ypv2qgRJq1HjodNicNPYVe/1ysnJ4fjx49YFW0NCQmjevLl1q0ibNm3466+/yM3Nte7bvHkzarW63FahZs2a4ebmxvbt2637Ll26xJEjR64Z3wMPPMDatWv5888/yxwrKiqyuf59992HWq3myy+/5LPPPuORRx5x+mznkiDVQiUJ0vojqXY3QQohhLixPfvss2zYsIFTp06xZcsW7rrrLjQaTZUHMj/44IMYDAbGjRvHvn37WLduHZMnT+ahhx4qM/4IwMvLiwkTJvDcc8/x22+/sW/fPh5++GHU6munGFOmTKF3797ceuutLFy4kL/++osTJ07w1VdfcdNNN3H06FGba9x///1Mnz6dpKQkHn744Sq9puqQLrZaqGezQHQaNWfS8zl5MZemwRX3FQshhBAAZ8+eZfTo0aSlpREcHEyfPn3Ytm0bwcHBVarHw8ODVatW8fTTT9O9e3c8PDwYOXIk8+fPr/CcN998k5ycHIYPH463tzfPPPMMmZmZ17yOXq9nzZo1vP3223zwwQc8++yzeHh40KZNG/7+97/Tvn17m/ITJkxg0aJFDB06lPDw8Cq9pupQKdJEUS1ZWVn4+vqSmZmJj4+Pw+t/8ONtbD6WxszhbeVuNiGEqCEFBQWcPHmSJk2aYDAYXB2OqKZr/Tva+/1dK7rYFi5cSFRUFAaDgZiYGHbs2FFh2Y8++oi+ffvi7++Pv78/sbGxZco//PDDZaZbLz1VeXp6Og8++CA+Pj74+fkxYcKEa06CVdOs3WyHZRySEEIIUdNcniAtW7aMuLg4Zs6cya5du+jUqRODBw8mNTW13PLr169n9OjRrFu3jq1btxIZGcmgQYM4d+6cTbkhQ4aQlJRk3f73v//ZHH/wwQfZv38/a9asYfny5WzcuJHHH3/caa+zqvq3CgFg24m0MpOKCSGEEMK5XJ4gzZ8/n8cee4zx48fTtm1b3n//fTw8PKzzHZT2xRdf8Le//Y3OnTvTunVrPv74Y8xmc5kF9vR6vc106/7+/tZjBw8eZOXKlXz88cfExMTQp08f3nnnHZYuXcr58+ed+nrt1SLEiwa+BgqLzWw/me7qcIQQQogbiksTJKPRSEJCArGxsdZ9arWa2NhYtm7dalcdeXl5FBUVERAQYLN//fr1hISE0KpVKyZOnEhaWpr12NatW/Hz86Nbt27WfbGxsajVapvbFK9WWFhIVlaWzeZMV9/uv/5w+a1pQgghhHAOlyZIFy9exGQylbltMDQ0lOTkZLvqmDZtGuHh4TZJ1pAhQ/jss8+Ij4/njTfeYMOGDdx2222YTJauquTkZEJCQmzq0Wq1BAQEVHjd2bNn4+vra90iIyOr8lKrpX8rS4K09XhaJSWFEEII4Uh1+jb/OXPmsHTpUtavX28zSn3UqFHWxx06dKBjx440a9aM9evXW9eJqarp06cTFxdnfZ6VleX0JKl38yA+Hd+dm5oGOvU6QgghhLDl0hakoKAgNBoNKSkpNvtTUlKsi+ZVZN68ecyZM4fVq1fTsWPHa5Zt2rQpQUFB1unWw8LCygwCLy4uJj09vcLr6vV6fHx8bDZn8za40b9VSJlZV4UQQgjhXC5NkHQ6HdHR0TYDrEsGXPfs2bPC8+bOncusWbNYuXKlzTiiipw9e5a0tDTrdOs9e/YkIyODhIQEa5nffvsNs9lss9ieEEIIIW5MLr+LLS4ujo8++oglS5Zw8OBBJk6cSG5uLuPHjwdg7NixTJ8+3Vr+jTfe4KWXXuKTTz4hKiqK5ORkkpOTrXMY5eTk8Nxzz7Ft2zZOnTpFfHw8d955J82bN2fwYMtCg23atGHIkCE89thj7Nixg82bN/PUU08xatSoGpmdsyryjSZe/+Ugdy7cjLHY7OpwhBBCiBuCyxOk+++/n3nz5jFjxgw6d+7M7t27WblypXXgdmJiIklJSdby7733HkajkXvuuYcGDRpYt3nz5gGg0WjYs2cPd9xxBy1btmTChAlER0ezadMm9Hq9tZ4vvviC1q1bc+uttzJ06FD69OnDhx9+WLMv3g56rZrvdp3jrzMZ/HFKbvcXQghR+6hUKn744QdXh+FYiqiWzMxMBVAyMzOdfq3/bT+t/LT7nJKZb3T6tYQQ4kaWn5+vHDhwQMnPz3d1KFU2btw4BVAARavVKlFRUcpzzz1XI68FUL7//vsy+x9//HFFrVYrX331VZljM2fOVDp16lRhncXFxcr8+fOV9u3bK3q9XvHz81OGDBmi/P7775XGc61/R3u/v13egiQqN6pHI4Z3CsfH4ObqUIQQQtRiJatInDhxwroI7MyZM10SS15eHkuXLuX555+vcPLniiiKwqhRo3j11Vd5+umnOXjwIOvXrycyMpL+/fvXSGuVJEhCCCFEPVGyikRkZCQjRowgNjaWNWvWAJCWlsbo0aOJiIjAw8ODDh06lFmGq3///vz973/n+eefJyAggLCwMF5++WWbMkePHuXmm2/GYDDQtm1ba/2lff3117Rt25YXXniBjRs3cubMGbtfx1dffcU333zDZ599xqOPPkqTJk3o1KkTH374IXfccQePPvooubm5VXtzqkgSpDriWGo2C9cdY90hmVVbCCFqlKKAMdc1m6JUO+x9+/axZcsWdDodYFnhPjo6mhUrVrBv3z4ef/xxHnrooTILvi9ZsgRPT0+2b9/O3LlzefXVV61JkNls5u6770an07F9+3bef/99pk2bVu71Fy1axJgxY/D19eW2227j008/tTv2L7/8kpYtWzJ8+PAyx5555hnS0tIqTMwcpU5PFHkjWb4niQVrjzK0Qxi3tA6p/AQhhBCOUZQHr7voDucXz4PO0+7iy5cvx8vLi+LiYgoLC1Gr1bz77rsARERE8Oyzz1rLTp48mVWrVvHVV1/Ro0cP6/6OHTtau+VatGjBu+++S3x8PAMHDmTt2rUcOnSIVatWWe/6fv3117ntttts4jh69Cjbtm3ju+++A2DMmDHExcXxz3/+E5VKVenrOHLkCG3atCn3WMn+I0eO2Pu2VIu0INURJeuybTp6kWKT3O4vhBCirFtuuYXdu3ezfft2xo0bx/jx4xk5ciQAJpOJWbNm0aFDBwICAvDy8mLVqlUkJiba1FF68uUGDRpYJ1c+ePAgkZGRNlPilDdv4SeffMLgwYMJCgoCYOjQoWRmZvLbb7/Z/VqU62g9cwRpQaojOjb0w9/DjUt5Rfx5JoPuUQGVnySEEOL6uXlYWnJcde0q8PT0pHnz5oAlSenUqROLFi1iwoQJvPnmm/z73/9mwYIFdOjQAU9PT6ZMmYLRaLS9pJvtDUEqlQqz2f4/zE0mE0uWLCE5ORmtVmuz/5NPPrFrya+WLVty8ODBco+V7G/ZsqXdMVWHJEh1hEatom+LYH766zwbDl+QBEkIIWqKSlWlbq7aQq1W8+KLLxIXF8cDDzzA5s2bufPOOxkzZgxgGU905MgR2rZta3edbdq04cyZMyQlJVlXp9i2bZtNmV9++YXs7Gz+/PNPNJorS2Xt27eP8ePHk5GRgZ+f3zWvM2rUKB544AF+/vnnMuOQ3nrrLQIDAxk4cKDdcVeHdLHVISXdbBuOXHBxJEIIIeqCe++9F41Gw8KFC2nRogVr1qxhy5YtHDx4kCeeeKLMWqiViY2NpWXLlowbN46//vqLTZs28Y9//MOmzKJFixg2bBidOnWiffv21u2+++7Dz8+PL774wlo2Pz+f3bt322zHjx9n1KhR3HXXXYwbN45FixZx6tQp9uzZwxNPPMFPP/3Exx9/jKenc5NWSZDqkL4tLX25e89lciG70MXRCCGEqO20Wi1PPfUUc+fO5ZlnnqFr164MHjyY/v37ExYWxogRI6pUn1qt5vvvvyc/P58ePXrw6KOP8q9//ct6PCUlhRUrVljHPZU+96677mLRokXWfUeOHKFLly422xNPPIFKpeKrr77ixRdf5O2336ZVq1b07duX06dPs379+irHXR0qxdWjoOqorKwsfH19yczMxMfHp8auO+w/m9h/Pov593Xi7q4Na+y6QghxIygoKODkyZM0adIEg8Hg6nBENV3r39He729pQapj+reSbjYhhBDC2SRBqmP6tbTMgbTxyAVMZmn8E0IIIZxBEqQ6pksjP7z1Wi7lFbH3XKarwxFCCCHqJUmQ6hg3jZrezS2DtTcclm42IYQQwhkkQaqDSsYhrT8i67IJIYQQziATRdZB/VuF8GS/ZtxyOVESQgghhGNJglQHhfkaeOG21q4OQwghhKi3pItNCCGEEKIUSZDqqCKTmXWHU5nz6yHMcru/EEII4VCSINVRZkXhb5/v4v0NxzmYnOXqcIQQQogKnTp1CpVKxe7dux1a1pkkQaqj9FoNd3WN4IGYRhjcNJWfIIQQot5SqVTX3F5++WWXxhcZGUlSUhLt27d3aRxVIYO067DX7+rg6hCEEELUAklJSdbHy5YtY8aMGRw+fNi6z8vLy/pYURRMJhNabc2kAEajEZ1OR1hYWI1cz1GkBUkIIYSo48LCwqybr68vKpXK+vzQoUN4e3vz66+/Eh0djV6v5/fff+fhhx9mxIgRNvVMmTKF/v37W5+bzWZmz55NkyZNcHd3p1OnTnzzzTfXjCUqKopZs2YxduxYfHx8ePzxx8t0m126dIkHH3yQ4OBg3N3dadGiBYsXLy63PpPJxCOPPELr1q1JTEy8nrepSqQFqY4rNpn562wG3gY3WoZ6uzocIYSot/KK8io8plFr0Gv0dpVVq9QYtIZKy3q4eVQjyoq98MILzJs3j6ZNm+Lv72/XObNnz+bzzz/n/fffp0WLFmzcuJExY8YQHBxMv379Kjxv3rx5zJgxg5kzZ5Z7/KWXXuLAgQP8+uuvBAUFcezYMfLz88uUKywsZPTo0Zw6dYpNmzYRHFxz8/9JglTHvbHyEB9tOsnoHpHMvrujq8MRQoh6K+bLmAqP9Y3oy39j/2t93v+r/uQXl/3CB+gW2o3FQ660lgz5dgiXCi+VKbd33N7riLasV199lYEDB9pdvrCwkNdff521a9fSs2dPAJo2bcrvv//OBx98cM0EacCAATzzzDPW56dOnbI5npiYSJcuXejWrRtgaXUqLScnh2HDhlFYWMi6devw9fW1O3ZHkASpjuvVPIiPNp1kw+ELKIqCSqVydUhCCCFqoZJkxF7Hjh0jLy+vTFJlNBrp0qXLdV1r4sSJjBw5kl27djFo0CBGjBhBr169bMqMHj2ahg0b8ttvv+Hu7l6l2B1BEqQ67qYmgei0as5nFnAsNYcW0s0mhBBOsf2B7RUe06ht7yZef9/6CsuqVbbDf1eOXHldcdnL09PTNg61GkWxnUevqKjI+jgnJweAFStWEBERYVNOr9dzLaWvVdptt93G6dOn+eWXX1izZg233norkyZNYt68edYyQ4cO5fPPP2fr1q0MGDDgmvU5gyRIdZy7TsNNTQPZeOQCG45ckARJCCGcpCpjgpxV1pGCg4PZt2+fzb7du3fj5uYGQNu2bdHr9SQmJl6zO+16rj9u3DjGjRtH3759ee6552wSpIkTJ9K+fXvuuOMOVqxY4ZQYrkUSpHqgX8tgNh65wPrDF3i0b1NXhyOEEKIOGDBgAG+++SafffYZPXv25PPPP2ffvn3W7jNvb2+effZZpk6ditlspk+fPmRmZrJ582Z8fHwYN25cta89Y8YMoqOjadeuHYWFhSxfvpw2bdqUKTd58mRMJhO33347v/76K3369Kn2NatKEqR6oF/LYGYBO06mk2csxkMn/6xCCCGubfDgwbz00ks8//zzFBQU8MgjjzB27Fj27r0yOHzWrFkEBwcze/ZsTpw4gZ+fH127duXFF1+8rmvrdDqmT5/OqVOncHd3p2/fvixdurTcslOmTMFsNjN06FBWrlxZZqySs6iU0h2Qwi5ZWVn4+vqSmZmJj4+PS2NRFIW+c9dx9lI+nzzcjQGtQ10ajxBC1FUFBQWcPHmSJk2aYDAYKj9B1ErX+ne09/tbJoqsB1QqFf1aWuaG2HD4goujEUIIIeo+SZDqiZIEaf0RSZCEEEKI6yUJUj3Rq3kQbhoVp9PyOHUx19XhCCGEEHWaJEj1hJdeS7fGAQBskFYkIYQQ4rpIglSP9Gt1eRySJEhCCCHEdakVCdLChQuJiorCYDAQExPDjh07Kiz70Ucf0bdvX/z9/fH39yc2NtamfFFREdOmTaNDhw54enoSHh7O2LFjOX/+vE09UVFRqFQqm23OnDlOe401oX+rYDpF+tE9KsDVoQghRJ1mNptdHYK4Do7493P5hDnLli0jLi6O999/n5iYGBYsWMDgwYM5fPgwISEhZcqvX7+e0aNH06tXLwwGA2+88QaDBg1i//79REREkJeXx65du3jppZfo1KkTly5d4umnn+aOO+7gjz/+sKnr1Vdf5bHHHrM+9/au27NQtw7z4cdJvV0dhhBC1Fk6nQ61Ws358+cJDg5Gp9PJGpd1iKIoGI1GLly4gFqtRqfTVbsul8+DFBMTQ/fu3Xn33XcBS9YXGRnJ5MmTeeGFFyo932Qy4e/vz7vvvsvYsWPLLbNz50569OjB6dOnadSoEWBpQZoyZQpTpkypVty1aR4kIYQQjmM0GklKSiIvL8/VoYhq8vDwoEGDBuUmSPZ+f7u0BcloNJKQkMD06dOt+9RqNbGxsWzdutWuOvLy8igqKiIgoOJupczMTFQqFX5+fjb758yZw6xZs2jUqBEPPPAAU6dORast/y0pLCyksLDQ+jwrK8uu+Fwhu6CIhNOX6N+qbAucEEKIa9PpdDRq1Iji4mJMJpOrwxFVpNFo0Gq1193y59IE6eLFi5hMJkJDbWd+Dg0N5dChQ3bVMW3aNMLDw4mNjS33eEFBAdOmTWP06NE2meLf//53unbtSkBAAFu2bGH69OkkJSUxf/78cuuZPXs2r7zyip2vzHXyjMV0e20thcVmfp92Cw39XbMIohBC1GUqlQo3Nzfrwq3ixuPyMUjXY86cOSxdupT169eXOyV8UVER9913H4qi8N5779kci4uLsz7u2LEjOp2OJ554gtmzZ6PX68vUNX36dJtzsrKyiIyMdOCrcQwPnZYOEb6k5xlJySqQBEkIIYSoBpcmSEFBQWg0GlJSUmz2p6SkEBYWds1z582bx5w5c1i7di0dO3Ysc7wkOTp9+jS//fZbpeOEYmJiKC4u5tSpU7Rq1arMcb1eX27iVBsteaQHnvo6nfsKIYQQLuXS2/x1Oh3R0dHEx8db95nNZuLj4+nZs2eF582dO5dZs2axcuVKunXrVuZ4SXJ09OhR1q5dS2BgYKWx7N69G7VaXe6dc3WNJEdCCCHE9XH5N2lcXBzjxo2jW7du9OjRgwULFpCbm8v48eMBGDt2LBEREcyePRuAN954gxkzZvDll18SFRVFcnIyAF5eXnh5eVFUVMQ999zDrl27WL58OSaTyVomICAAnU7H1q1b2b59O7fccgve3t5s3bqVqVOnMmbMGPz9/V3zRlzlTNYZACJ9rq8Lz1hspqDYhI9B+tCFEEKIKlFqgXfeeUdp1KiRotPplB49eijbtm2zHuvXr58ybtw46/PGjRsrQJlt5syZiqIoysmTJ8s9Dijr1q1TFEVREhISlJiYGMXX11cxGAxKmzZtlNdff10pKCiwO+bMzEwFUDIzMx3xFlh98NcHSvtP2yszNs+4rno+3nRCafvSr8rsXw46KDIhhBCi7rP3+9vl8yDVVc6aB2l70nYeXf0ovnpf1t23Djd19Vp/ftx9jqeX7qZNAx9+fbqvw+ITQggh6jJ7v79rxVIj4oro0GgCDAFkFmbyR/IflZ9QgT7Ng1Cp4GBSFilZBQ6MUAghhKj/JEGqZbRqLQMaDQBg9enV1a4n0EtPxwhfQBavFUIIIapKEqRaaFDjQQD8lvgbxebiatfT7/JM2pIgCSGEEFUjCVIt1D2sO356P9IL0klISah2Pf1aBgPw+9GLFJtkZWohhBDCXpIg1UJatZZbG90KwNrTa6tdT+dIP3zd3cjML+KvsxkOik4IIYSo/1w+D5Io3/2t7qd3RG96h/eudh0atYq+LYJYvieJDYcvEN244gV9hRBCCHGFtCDVUm0C2zCw8UA83K5vLbWSbjYZhySEEELYTxKkeq4kQdpzLpO0nEIXRyOEEELUDZIg1WK5Rbm8t/s9Hl31KCazqVp1hPgYaNvAB0WBTUcvOjhCIYQQon6SBKkW06l1/N+B/2N78nZ2X9hd7Xr6tZJuNiGEEKIqJEGqxdw0btzS6BYAVp+q/qSR/VoGo1WrKCiqXiuUEEIIcaORBKmWG9h4IGC53d+sVG8uo26N/dk1YyDvjYl2ZGhCCCFEvSUJUi3XK7wXnm6epOansufCnmrVodWo8TFUb9FbIYQQ4kYkCVItp9Po6B/ZH4BVp1Zdd33ZBUXXXYcQQghR30mCVAeUrM225vSaanez5RtNjFi4ma6z1pCZJ0mSEEIIcS2SINUBvcJ7EeoRSo+wHuQV5VWrDnedhpzCYopMCrsSLzk4QiGEEKJ+USmKorg6iLooKysLX19fMjMz8fHxcfr1zIoZter68tk/Ey/RwNedMF+Dg6ISQggh6hZ7v79lLbY64nqTI4AujfwdEIkQQghR/0kXW21TlA/7f4DissuCKIrCofRDnM85X/NxCSGEEDcQSZBqE0WBD/rB1+Pg2Noyh2fvmM29P9/L0sNLq32JNQdSGPvJDj75/eT1RCqEEELUa5Ig1SYqFbSwTAzJ3q/LHO4W2g2ANafWUN2hY+cu5bHxyAVWH0iudphCCCFEfScJUm3T4R7Lz8O/QkGWzaE+EX0waAyczTnLwfSD1aq+f6sQAP44dYmcwuLrClUIIYSoryRBqm0adIbAFlBcAIeW2xzycPOgb8O+gGVOpOqICvKkcaAHxWaFLccuXm+0QgghRL0kCVJto1JBx/ssj/d8VeZwyaSRq0+trnY3W7+WwQBsOHKhejEKIYQQ9ZwkSLVRSTfbyQ2QnWJz6OaGN6PX6EnMTuTIpSPVqr5/K0uCtP7whWonWUIIIUR9JglSbRTQFBp2B8UM+7+zOeTh5kGfiD4A/Jb4W7Wqv6lpIDqNmnMZ+Ry/kHvd4QohhBD1jSRItVWHey0/y+lme6T9I3ww8AMe7fhotar20Gnp0SQAkG42IYQQojySINVW7e4GlQbO74K04zaHOgZ3pFd4L9zUbtWuvqSbTRIkIYQQoixJkGorr2BodovlcTmtSNerZKD2thNp5BtNDq9fCCGEqMskQarNOly+m23vV5ZZtq+Slp/Gmzvf5Mk1T1ar6uYhXoT7GjAWm9l2Mu16IxVCCCHqFUmQarPWw0DrDuknLF1tV9FpdHx56Es2n9/M8YzjFVRQMZVKRb/Lk0ZuOCzdbEIIIcTVJEGqzfRe0Hqo5fEe26VHvHXe9ArvBVjmRKqOkm62rcelBUkIIYS4miRItV1JN9u+b8FkuzSIddLI09VLkPq0CGLxw935flKv6wpRCCGEqG8kQartmt8K7gGQm2qZOPIq/SP7o1VrOZZxjBOZJ6pctZdeyy2tQ/DQaR0VrRBCCFEvSIJU22ncoN1dlsd7bbvZfPW+3NTgJgDWnKre2mxCCCGEKEsSpLqgZG22gz9DUb7NoevtZisoMjHn10Pc9d/NFBbL7f5CCCEESIJUNzTsAb6NwJgDh3+1OXRL5C2EeYbRNaQrReaiKlet16r5dtdZ/kzM4I9TlxwVsRBCCFGn1YoEaeHChURFRWEwGIiJiWHHjh0Vlv3oo4/o27cv/v7++Pv7ExsbW6a8oijMmDGDBg0a4O7uTmxsLEePHrUpk56ezoMPPoiPjw9+fn5MmDCBnJwcp7y+66ZWX1nAtlQ3m5/Bj9UjV/OPm/5RrZm1VSoVcQNb8u9RnWkf7uuIaIUQQog6z+UJ0rJly4iLi2PmzJns2rWLTp06MXjwYFJTU8stv379ekaPHs26devYunUrkZGRDBo0iHPnzlnLzJ07l//85z+8//77bN++HU9PTwYPHkxBQYG1zIMPPsj+/ftZs2YNy5cvZ+PGjTz++ONOf73VVtLNdnQN5KXbHFKpVNdV9egejbizcwS+HtVfukQIIYSoVxQX69GjhzJp0iTrc5PJpISHhyuzZ8+26/zi4mLF29tbWbJkiaIoimI2m5WwsDDlzTfftJbJyMhQ9Hq98r///U9RFEU5cOCAAig7d+60lvn1118VlUqlnDt3zq7rZmZmKoCSmZlpV3mH+G9vRZnpoyg7F5U5VGwqVnYk7VBSclNqLh4hhBCijrH3+9ulLUhGo5GEhARiY2Ot+9RqNbGxsWzdutWuOvLy8igqKiIgwLI6/cmTJ0lOTrap09fXl5iYGGudW7duxc/Pj27dulnLxMbGolar2b59e7nXKSwsJCsry2arcR3vtfwsNWkkwHMbn+ORVY/w8/Gfq1X18Qs5vLf+OOsOld9yJ4QQQtxIXJogXbx4EZPJRGhoqM3+0NBQkpOT7apj2rRphIeHWxOikvOuVWdycjIhISE2x7VaLQEBARVed/bs2fj6+lq3yMhIu+JzqPYjARUkboGMMzaHrLf7n67e7f7L/0rijZWH+OqPM5UXFkIIIeo5l49Buh5z5sxh6dKlfP/99xgMBqdea/r06WRmZlq3M2dckEj4NoTGvS2P931jc+jWRreiVqnZn7afcznnyjn52vq1siw78vvRixSZzNcdqhBCCFGXuTRBCgoKQqPRkJKSYrM/JSWFsLCwa547b9485syZw+rVq+nYsaN1f8l516ozLCyszCDw4uJi0tPTK7yuXq/Hx8fHZnOJCrrZAt0D6RZq6TKszqSRHSN8CfDUkV1YzJ+JGdcbpRBCCFGnuTRB0ul0REdHEx8fb91nNpuJj4+nZ8+eFZ43d+5cZs2axcqVK23GEQE0adKEsLAwmzqzsrLYvn27tc6ePXuSkZFBQkKCtcxvv/2G2WwmJibGUS/POdreCRodpO6HlP02h0omjaxON5taraJviyAANhyRcUhCCCFubC7vYouLi+Ojjz5iyZIlHDx4kIkTJ5Kbm8v48eMBGDt2LNOnT7eWf+ONN3jppZf45JNPiIqKIjk5meTkZOscRiqViilTpvDaa6/x008/sXfvXsaOHUt4eDgjRowAoE2bNgwZMoTHHnuMHTt2sHnzZp566ilGjRpFeHh4jb8HVeLuDy0siRB7vrI5dGvjW1GhYs/FPSTlJFW56n4tLd1s6w9fuO4whRBCiLrM5QnS/fffz7x585gxYwadO3dm9+7drFy50jrIOjExkaSkK1/27733HkajkXvuuYcGDRpYt3nz5lnLPP/880yePJnHH3+c7t27k5OTw8qVK23GKX3xxRe0bt2aW2+9laFDh9KnTx8+/PDDmnvh16Nk0sh934L5ynihIPcgokOjAdhwdkN5Z17TzZcTpP3ns0jNLqiktBBCCFF/qRRFUVwdRF2UlZWFr68vmZmZNT8eqSgf3mwBxmwY/ys07mU99Gfqn2hVWtoHta/WBJLD3/mdvecyeeveToyMbujIqIUQQgiXs/f72+UtSKIa3Nyh7R2Wx6W62bqEdKFDcIdqz65t7WY7It1sQgghblySINVVHS7fzXbgByg2Oqza/pdv99909AImszQuCiGEuDFJglRXNbkZvEIh/xIcW2tz6FzOOV7e8jJP//Z0lavtHOmHj0FLRl4Re85mOChYIYQQom6RBKmuUmsuz6wN7LWdE0mj0vDt0W9Zd2YdqXlVu2Vfq1HTt4XczSaEEOLGJglSXVbSzXb4VyjMtu4O8wyjU3AnFBTWnl5bwckVKxmHtEHGIQkhhLhBSYJUl4V3gcDmUJwPB5fbHBrYeCAAq0+vrnK1/VoF88TNTXl+SCuHhCmEEELUNZIg1WUqFXS4z/J4r+3dbCUJ0q6UXVzMv1ilakN9DEwf2oZezYIcEqYQQghR10iCVNeVTBp5Yj1kX1l/LtwrnA5BHVBQiD8dX/65QgghhCiXJEh1XWAziIgGxQz7v7c5dD3dbMUmMxuOXOCNlYcwy+3+QgghbjCSINUH1+hma+zTmM4hnanqhOlmBf72eQLvrT/OgaQsR0UqhBBC1AlaVwcgHKD93bDqRTiXAGnHLa1KQEPvhvw84udqzaqt06oZ0SUCk1nB4CZ5tBBCiBuLJEj1gVcINO0Px+MtcyL1f8F6qLpLjgD8664ODghOCCGEqHukaaC+6Hi5m23PV1CqO81oMrL+zHoyCzNrPi4hhBCiDpIEqb5oPQy07pB+HM7/aXPo8TWPM/m3ydWaNNJkVvgz8RJHUrIrLyyEEELUE5Ig1Rd6b2h1m+VxqaVH+kT0AWDN6TVVrvbNVYe5679b+OT3k9cdohBCCFFXSIJUn5R0s+37Fswm6+6S2/23J22vcjdbz2aBgGVdtqreCSeEEELUVZIg1SfNbgV3f8hJgZMbrLsb+zSmlX8ripVifkv8rUpVxjQJwOCmJjmrgCMpOY6OWAghhKiVJEGqT7Q6aHeX5fEe22626k4aaXDTcFNTSyvShiOp1x+jEEIIUQdIglTfdLjX8vPgz1CUb909MMqSIG1L2lblbrZ+LYMBSzebEEIIcSOQBKm+ibwJfCPBmA1HVlp3N/VtSnO/5hSbi9mRvKNKVZYkSDtPpZNbWOzQcIUQQojaSBKk+katvrKAbalutn/E/IMf7/zR2t1mryZBnkQGuFNkUth6PM1RkQohhBC1liRI9VHJ2mxHV0NeunV3t7BuNPVrWuXqVCoV/VuGALBexiEJIYS4AUiCVB+FtoXQ9mAuggM/OqTKq8chye3+Qggh6jtJkOqrkm62vd/Y7D5y6QjPrH+G5zc+X6XqejYLxE2j4uylfE5ezHVUlEIIIUStJAlSfdX+coJ0+nfIPGtzaPXp1cSfjifHaP+8Rp56Ld2jAgDYcETuZhNCCFG/SYJUX/lFQuPelsdXtSK18GtBlE8URrORDWc3VHBy+fq3ktv9hRBC3BgkQarPSuZEumptNpVKZb2Lraprs/VrGULHhr50j/J3WIhCCCFEbSQJUn3W9k5Qu0HKPkg5YN09KGoQAL+f+528ojy7q2sV5s1PT/XhqQEtHB6qEEIIUZtIglSfeQRAi8tzHl3VitTKvxWNvBtRaCpk49mNLgpOCCGEqL0kQarvrN1s34DZDNh2s1V1bTaAnMJiGagthBCiXpMEqb5rdRvovCEzEc5st+4eGDWQdoHtiA6NrlJ1+UYT3V5bw7hPdnAm3f7uOSGEEKIukQSpvnNzhzbDLY/3fmXd3S6wHUtvX8qDbR6sUnXuOg0dI/yICvQgOavAkZEKIYQQtYYkSDeCjpe72fZ/D8XG667uk/HdWf/cLdZ5kYQQQoj6RhKkG0GTfuAZAvmX4PhvNodyjDksP7Gc/OJ8u6vz0msdHaEQQghRq0iCdCNQa6D9SMvjq7rZAEavGM30TdPZcm5LlastMpnJKihyRIRCCCFErSIJ0o2ipJvt0C9QmG3d3a9hPwBWnV5VpeoWbz5Jl1fXsHDdMYeFKIQQQtQWLk+QFi5cSFRUFAaDgZiYGHbs2FFh2f379zNy5EiioqJQqVQsWLCgTJmSY6W3SZMmWcv079+/zPEnn3zSGS+v9gjvCgHNoDgfDq2w7h4YZbndf8OZDRSaCu2uLsBTZ7ndX5YdEUIIUQ+5NEFatmwZcXFxzJw5k127dtGpUycGDx5MampqueXz8vJo2rQpc+bMISwsrNwyO3fuJCkpybqtWWNZTuPee++1KffYY4/ZlJs7d65jX1xto1JBx/ssj/dc6WbrGNSRMM8w8orzqtTN1rdFMCoVHErOJjlT7mYTQghRv7g0QZo/fz6PPfYY48ePp23btrz//vt4eHjwySeflFu+e/fuvPnmm4waNQq9Xl9umeDgYMLCwqzb8uXLadasGf369bMp5+HhYVPOx8fH4a+v1imZNPLEesixJKEqlYrYRrFA1SaNDPDU0amhHwAbZdJIIYQQ9YzLEiSj0UhCQgKxsbFXglGriY2NZevWrQ67xueff84jjzyCSqWyOfbFF18QFBRE+/btmT59Onl51570sLCwkKysLJutzglsZulqU0yWW/4vGxw1GID1Z9ZjNNk/DUC/lsEAMqu2EEKIesdlCdLFixcxmUyEhoba7A8NDSU5Odkh1/jhhx/IyMjg4Ycfttn/wAMP8Pnnn7Nu3TqmT5/O//3f/zFmzJhr1jV79mx8fX2tW2RkpENirHHldbMFdyTEI4Scohz2XNhjd1X9WlkSpE1HL1BsMjs0TCGEEMKV6vWENosWLeK2224jPDzcZv/jjz9ufdyhQwcaNGjArbfeyvHjx2nWrFm5dU2fPp24uDjr86ysrLqZJLW7G1a9COf+gLTjENgMtUrNnL5ziPSOJMyz/LFd5enU0A8/Dzcy8orYfSaDbjJxpBBCiHrCZS1IQUFBaDQaUlJSbPanpKRUOAC7Kk6fPs3atWt59NFHKy0bExMDwLFjFd+yrtfr8fHxsdnqJO9QaNrf8njvN9bd3cO6Vyk5AtCoVfRtId1sQggh6p9qJUhnz56t8Ni2bdvsqkOn0xEdHU18fLx1n9lsJj4+np49e1YnLBuLFy8mJCSEYcOGVVp29+7dADRo0OC6r1sndLjczbb3a1CUMoeVcvZVpGQc0nq53V8IIUQ9Uq0EadCgQaSnp5fZv3nzZoYMGWJ3PXFxcXz00UcsWbKEgwcPMnHiRHJzcxk/fjwAY8eOZfr06dbyRqOR3bt3s3v3boxGI+fOnWP37t1lWn7MZjOLFy9m3LhxaLW2vYjHjx9n1qxZJCQkcOrUKX766SfGjh3LzTffTMeOHavyNtRdrYeB1gBpRyFpt3X37tTdPLnmSV7d9qrdVd3cMgiAvecyuZhj/zxKQgghRG1WrQTppptuYtCgQWRnX5mReePGjQwdOpSZM2faXc/999/PvHnzmDFjBp07d2b37t2sXLnSOnA7MTGRpKQka/nz58/TpUsXunTpQlJSEvPmzaNLly5lutHWrl1LYmIijzzySJlr6nQ61q5dy6BBg2jdujXPPPMMI0eO5Oeff67q21B3GXyg1W2Wx3u+tu4uMhex+fxmVp9aTZHZviVEQrwNtAu3dDduOiqtSEIIIeoHlVKV/pTLzGYz99xzD+np6axatYotW7Zwxx138Nprr/H00087I85aJysrC19fXzIzM+vmeKRDv8DS0eAVBnEHQK3BZDYx4OsBpBek837s+/SO6G1XVXNXHuK/649zZ+dw/j2qi5MDF0IIIarP3u/varUgqdVqli5dipubGwMGDOCOO+5g9uzZN0xyVC80jwV3f8hJhpMbAdCoNdZJI9ecXmN3Vf1bhaBRq8gzmpwSqhBCCFHT7G5B2rOn7Pw42dnZjB49mmHDhjFx4kTr/hthLE+db0EC+HkKJCyGzmNgxEIAtidt59HVj+Kn9+O3+37DTe1WaTXFJjO5RhO+7pWXFUIIIVzJ3u9vuxMktVqNSqWyucPp6uclj1UqFSZT/W9JqBcJ0qnN8OlQ0PvAs0fBzUCxuZgBXw3gUuElPhz4IT3Dr/+OQiGEEKK2sPf72+6JIk+ePOmQwEQt0qgn+DSErLNwZCW0G4FWreXWxrfyzZFvWH16dZUTpNzCYjz19Xr+USGEEDcAu7/JGjdu7Mw4hCuo1dDhHti8wDInUrsRAAyJGkJiViJdQ7raXVVBkYkHPtrG3nOZ7PxHLH4eOufELIQQQtSAag3SXrJkCStWrLA+f/755/Hz86NXr16cPn3aYcGJGlCyNtvR1ZB/CYCYBjEsGryI4c2G212NwU1DTmExRSaFhNOXnBGpEEIIUWOqlSC9/vrruLu7A7B161beffdd5s6dS1BQEFOnTnVogMLJQttBSDswGeHAj9dV1RsjO7L5hQHc2ia08sJCCCFELVatBOnMmTM0b94cgB9++IF77rmHxx9/nNmzZ7Np0yaHBihqQId7LD+vWpsN4GL+Rb498i0ms32D7rs08ifCz93R0QkhhBA1rloJkpeXF2lpaQCsXr2agQMHAmAwGMjPz3dcdKJmlCRIp36HzHMAmMwm7v7xbl7e+jK7Une5MDghhBCi5lUrQRo4cCCPPvoojz76KEeOHGHo0KEA7N+/n6ioKEfGJ2qCXyNo1AtQYJ+lFUmj1nBzw5uBqk0a+duhFB5evINFv8tdj0IIIequaiVICxcupGfPnly4cIFvv/2WwMBAABISEhg9erRDAxQ1pOO9lp9Xrc02KGoQAGtPr8WsmO2q5tylfNYfvsDq/ckOD1EIIYSoKdVai03Uk4kir5aXDvNagrkI/rYNQtpQZCqi37J+ZBdls2TIErqGVn7bf2JaHje/uQ6tWsWfMwbibZDZtYUQQtQeDl+Lbc+ePZjNZuvja22iDvIIsKzPBpY5kQA3jRu3NLoFgNWnV9tVTaNAD5oEeVJsVth8LM0poQohhBDOZneC1LlzZy5evGh93KVLFzp37lxm69JFVnOvs0q62fZ+DZcbFgc2tgzAX3N6jd3dbP1aBgOw4cgFx8cohBBC1IAqLTUSHBxsfVyR3Nzc649KuEbL20DnBRmJcGY7NLqJnuE98XTzJD0/nRMZJ2ju37zSavq1CubTLafYeOSCdX0+IYQQoi6p1lIj5S07UlhYyMKFC5k7dy7JyTJAt07SeUCb4fDX/2DPV9DoJvQaPe8MeIeW/i3x1fvaVc1NTQLRadWcy8jn+IUcmod4OzlwIYQQwrGqdBdbYWEh06dPp1u3bvTq1YsffvgBgMWLF9OkSRPefvttmUm7rutwuZtt//dgKgKge1h3u5MjAHedhpuaWu5s/OqPsw4PUQghhHC2KiVIM2bM4L333iMqKopTp05x77338vjjj/P2228zf/58Tp06xbRp05wVq6gJTfqBZzDkp8Px38octncc0tibLK2Mi34/yb5zmQ4NUQghhHC2KiVIX3/9NZ999hnffPMNq1evxmQyUVxczF9//cWoUaPQaDTOilPUFI0W2o+0PN7zlXX3xrMbGfPLGBYkLLCrmti2oQzr2ACTWeG5b/ZQZLIvsRJCCCFqgyolSGfPniU6OhqA9u3bo9frmTp1qgzCrW863Gf5efgXKMwBwGgy8teFv1h1ahX2Tp31yh3t8Pdw42BSFu+vP+6saIUQQgiHq1KCZDKZ0Ol01udarRYvLy+HByVcLKIrBDSFojw4tAKA3hG9cde6cz73PAfSDthVTZCXnpnD2wHwzm/HOJaa47SQhRBCCEey+y42AEVRePjhh9Hr9QAUFBTw5JNP4unpaVPuu+++c1yEouapVJZWpA1zYO9X0Ol+3LXu3NzwZladWsWq06toF9TOrqru7BzOqv3JtI/wpXGgh5MDF0IIIRyjSi1I48aNIyQkBF9fX3x9fRkzZgzh4eHW5yWbqAdK7mY7vg5yLBM+WieNPLXG7m42lUrFfx/syqRbmuOmqdbSf0IIIUSNq1IL0uLFi50Vh6htgppDeBc4/6fllv+Yx+kb0ReDxsDZnLMcTD9I28C2dlV19Rg1Y7GZrIIigrz0zopcCCGEuG7yJ72oWMlg7b2Wu9k83Dzo27AvYFl6pKoOJWdx+zubmPzln3a3QAkhhBCuIAmSqFj7kaBSw9mdkH4CgGFNhnFL5C10Du5c5erc3TQkpudxNDWbM+n5Dg5WCCGEcByVIn/KV0tWVha+vr5kZmbi4+Pj6nCc57MRcGId3PIP6Pf8dVe3/nAqnRr64e+pq7ywEEII4WD2fn9LC5K4tpLB2nu+Agfk0v1bhUhyJIQQotaTBElcW5vhoDVA2lFI+su6OzErkaWHllZ7LJGiKPz013lW7ktyVKRCCCGEw1TpLjZxAzL4QMshcOAH2Ps1hHcmryiPu368C6PZSHRoNC38W1S52p/3JPH3//2Jv4cb3aIC5K42IYQQtYq0IInKdSy5m+0bMJvwcPOgV0QvAFafXl2tKm9rH0brMG8u5RXx8k/7HRWpEEII4RCSIInKNR8IBj/ISYZTmwAY1HgQYJk0sjrcNGrevKcTGrWK5XuSWL0/2VHRCiGEENdNEiRROa0O2t5pebz3awD6R/ZHq9ZyPPM4xzOqtxBth4a+PNa3KQD//GEfmflFDglXCCGEuF6SIAn7lHSzHfgJigrw1nnTK/xyN9up6nWzAUyJbUHTIE9Sswv51wr7FsEVQgghnE0SJGGfRr3ApyEUZsHRVcCVbrbqjkMCMLhpeOOejqhU8NUfZ9l09IJDwhVCCCGuhyRIwj5qNXQYaXm8x7L0SEk327mcc1zIq35i0z0qgLE3NQbghW/3kltYfN3hCiGEENdDEiRhv5K12Y6uhvxL+Op9WTRoERvu30CwR/B1Vf38kNZE+LlzLiOfN1cddkCwQgghRPW5PEFauHAhUVFRGAwGYmJi2LFjR4Vl9+/fz8iRI4mKikKlUrFgwYIyZV5++WVUKpXN1rp1a5syBQUFTJo0icDAQLy8vBg5ciQpKSmOfmn1T2g7CG4DJiMc/BmArqFdcde6X3fVnnots+/uAMCSraf441T6ddcphBBCVJdLE6Rly5YRFxfHzJkz2bVrF506dWLw4MGkpqaWWz4vL4+mTZsyZ84cwsLCKqy3Xbt2JCUlWbfff//d5vjUqVP5+eef+frrr9mwYQPnz5/n7rvvduhrq5dUKuh41dIjpZjMpuuq/uaWwdwb3RBFgee/3UNB0fXVJ4QQQlSXSxOk+fPn89hjjzF+/Hjatm3L+++/j4eHB5988km55bt3786bb77JqFGj0OsrnnlZq9USFhZm3YKCgqzHMjMzWbRoEfPnz2fAgAFER0ezePFitmzZwrZt2xz+GuudkrXZTv0OWecBWHFiBXf9eBeL9y++7ur/Oawtwd560nONHEvNue76hBBCiOpwWYJkNBpJSEggNjb2SjBqNbGxsWzduvW66j569Cjh4eE0bdqUBx98kMTEROuxhIQEioqKbK7bunVrGjVqdM3rFhYWkpWVZbPdkPwaQaOegGKZWRvIL87nWMax67rdv4SvhxsfPBTNmqn9aB/he931CSGEENXhsgTp4sWLmEwmQkNDbfaHhoaSnFz9WZVjYmL49NNPWblyJe+99x4nT56kb9++ZGdnA5CcnIxOp8PPz69K1509eza+vr7WLTIystox1nklrUh7Ld1sAxoNQKPScDD9IGeyz1x39V0b+RPsLWuzCSGEcB2XD9J2tNtuu417772Xjh07MnjwYH755RcyMjL46quyY2aqYvr06WRmZlq3M2euPxGos9qOALUWkvdC6iECDAF0C+sGwJrT1Vt6pCKr9iezZMsph9YphBBCVMZlCVJQUBAajabM3WMpKSnXHIBdVX5+frRs2ZJjx44BEBYWhtFoJCMjo0rX1ev1+Pj42Gw3LM9AaH65i/Ly0iPWSSMd0M1WYvuJNJ74vwT+teIgx1KzHVavEEIIURmXJUg6nY7o6Gji4+Ot+8xmM/Hx8fTs2dNh18nJyeH48eM0aNAAgOjoaNzc3Gyue/jwYRITEx163XrP2s32NSgKtza6FbVKzf60/ZzLOeeQS/RoEsDAtqE82rcJDf09HFKnEEIIYQ+tKy8eFxfHuHHj6NatGz169GDBggXk5uYyfvx4AMaOHUtERASzZ88GLAO7Dxw4YH187tw5du/ejZeXF82bNwfg2WefZfjw4TRu3Jjz588zc+ZMNBoNo0ePBsDX15cJEyYQFxdHQEAAPj4+TJ48mZ49e3LTTTe54F2oo1oNBZ0XZJyGMzsIbBRDt9Bu7EjewdrTaxnXbtx1X0KlUvHBmGjUapUDAhZCCCHs59IE6f777+fChQvMmDGD5ORkOnfuzMqVK60DtxMTE1GrrzRynT9/ni5dulifz5s3j3nz5tGvXz/Wr18PwNmzZxk9ejRpaWkEBwfTp08ftm3bRnDwlZme3377bdRqNSNHjqSwsJDBgwfz3//+t2ZedH2h84DWt8OepZbB2o1iuKPZHYR6hNI+qL3DLnN1clRsMpNdUIy/p85h9QshhBDlUSmKorg6iLooKysLX19fMjMzb9zxSMfWwucjwSMQnjkMGjenXer4hRymLtuNl17LF4/GoFJJq5IQQoiqs/f7u97dxSZqUJP+4BkMeWlwfJ1TL6VVqziSks2W42ks3XkD30EohBCiRkiCJKpPo4V2l5douTwnkqIoHEw7yNJDSx16qcaBnjw7qBUAr684SFJmvkPrF0IIIa4mCZK4Ph3vs/w8tAIKc7iYf5H7lt/Hv7b/i+Tc6k/4WZ7xvZvQOdKP7MJi/vn9PqR3WAghhLNIgiSuT0Q0+DeBojw4/AvBHsF0CbEMpF97eq1DL6VRq5h7T0fcNCriD6Xy01/nHVq/EEIIUUISJHF9VKorrUilJo109KzaAC1DvZk8oAUAr/x8gLScQodfQwghhJAESVy/kkkjj8VD7kViG1tm2f4z9U9S81Idfrkn+zWjdZg36blGXv75gMPrF0IIISRBEtcvqAU06AyKCfZ/T5hnGB2DO6KgsOzwModfTqdVM/eejqhV8PNf51lzIKXyk4QQQogqkARJOEZJN9sey91sD7V5CIBFexexO3W34y/X0I/Hbm4KwD++30tmfpHDryGEEOLGJQmScIz2I0GlhrM7IP0kQ5oMYVjTYQR7BFd+bjVNjW1JkyBPUrMLeX3FQaddRwghxI1HEiThGN5h0ORmy+N93wDwz5h/8s3wb+gc0tkplzS4aXhjZEcAlv1xhi3HLzrlOkIIIW48kiAJxykZrL3na1AUvHRe+Op9rYcLigscfskeTQJ4uFcUj/ZpQpdIf4fXL4QQ4sYkCZJwnDbDQaOHi4cheY91t6IofHvkW4Z8O4QzWY5fJmTm8Lb88/a2uOs0Dq9bCCHEjUkSJOE4Bl9oNcTy+PJgbQCzYubnEz+TVpDGtE3TKDI7dkD11QvXms0KyZmOb6kSQghxY5EESThWh8t3s+37FswmADRqDbP7zMZb583ei3t5b/d7Trn02Ut53PvBVh78eBsFRSanXEMIIcSNQRIk4VgtBlpakrKT4PRm6+4GXg14uefLAHy892N2Ju90+KW99W4kpueRnFnAgaQsh9cvhBDixiEJknAsrR7a3ml5fFU3G8CgqEHc3eJuFBRe2PQCmYWZDr20r4cb/32wK6vj+tG1kQzYFkIIUX2SIAnHK+lmO/ATFNmOB5rWfRpRPlGk5qXy8paXURTFoZfuHhVAhJ+7Q+sUQghx45EESThe497gEwGFmXBouc0hDzcP5tw8B71GTzO/ZpgVs9PC2HT0Al9sP+20+oUQQtRfWlcHIOohtRo6Pwgb58Lql6B5LLj7WQ+3C2zHypErCXIPcloICacv8dCiHeg0anpEBdAi1Ntp1xJCCFH/SAuScI4+UyGgGWSfh9X/KHP46uTIaDJSZHLsrf9dG/nRv1UwRpOZ57/dg8ns2K48IYQQ9ZskSMI5dB5w50JABX9+DkfXlFvsZOZJxvwyhn/v+rdDL69SqXj9rg546bX8mZjBki2nHFq/EEKI+k0SJOE8jXvCTRMtj3/6O+RnlClyOus0B9MPsuTAErac2+LQy4f7ufPCba0BeHPVYRLT8hxavxBCiPpLEiThXANeumZXW//I/oxqNQqAF39/kbT8NIde/oEejYhpEkB+kYnp3+9x+F1zQggh6idJkIRz2dHV9ky3Z2ju15y0gjRmbJnh0CRGrVbxxsiOGNzUbD6WxrKdjl8LTgghRP0jCZJwvtJdbQW2E0QatAbeuPkNdGodG89u5H+H/ufQy0cFefLMwFYA/GvFQVmrTQghRKUkQRI1Y8BLENDU0tW26sUyh1v6t+SZbs8A8NYfb3Hk0hGHXv6RPk3oFOlHdmEx//xhr3S1CSGEuCZJkETN0HnAnf/lWl1to1uP5uaGN9M6oDXuWsfOhq1Rq3jzno64aVSsPZjKz3uSHFq/EEKI+kUSJFFzKulqU6lUzOk7h09v+5RI70iHX75lqDdP3dICgJd/2k9aTqHDryGEEKJ+kARJ1KxKutq8dd64qd2sz7OMWQ69/MT+zWgd5k2esZg95xy7WK4QQoj6QxIkUbPKdLWtLbdYkamI+X/MZ/j3w0nNS3Xc5bVq/j2qC6un9OOWViEOq1cIIUT9IgmSqHlXd7X9XLarDUBBYWvSVtIL0vnH7/9w6KK2rcK8aRTo4bD6hBBC1D+SIAnXKOlqyzpXblebTqPjjZvfwKAxsC1pG5/t/8wpYWw/kcaCtY69Y04IIUTdJwmScA07utqa+jZlWo9pAPz7z3+zP22/Q0M4nZbL6I+2sWDtUbYcu+jQuoUQQtRtkiAJ17Gjq21ki5HENoql2FzMtI3TyCty3HpqjQM9eeimxozu0Yj2DX0dVq8QQoi6TxIk4Vo2XW1l12pTqVS83OtlQjxCOJ11mjd2vuHQy88c3o7Zd3fAx+BWeWEhhBA3DEmQhGvZdLX9X7ldbb56X+b0nYO3mzfdQrs59PJqtcr6WFEULsrcSEIIIagFCdLChQuJiorCYDAQExPDjh07Kiy7f/9+Ro4cSVRUFCqVigULFpQpM3v2bLp37463tzchISGMGDGCw4cP25Tp378/KpXKZnvyyScd/dKEvezoause1p1V96xieLPhTgkhJauAsZ/sYNSH2ygsNjnlGkIIIeoOlyZIy5YtIy4ujpkzZ7Jr1y46derE4MGDSU0tf96bvLw8mjZtypw5cwgLCyu3zIYNG5g0aRLbtm1jzZo1FBUVMWjQIHJzc23KPfbYYyQlJVm3uXPnOvz1iSqopKsNLJNIlsgoyMBkdlwiY9BqOJSczbHUHN6JP+aweoUQQtRNLk2Q5s+fz2OPPcb48eNp27Yt77//Ph4eHnzyySfllu/evTtvvvkmo0aNQq/Xl1tm5cqVPPzww7Rr145OnTrx6aefkpiYSEJCgk05Dw8PwsLCrJuPj4/DX5+oAp0H3LmQa3W1ldietJ27f7qbj/d+7LDL+3q4MevOdgC8t+E4+8/LLNtCCHEjc1mCZDQaSUhIIDY29kowajWxsbFs3brVYdfJzLR80QUEBNjs/+KLLwgKCqJ9+/ZMnz6dvLxr3x1VWFhIVlaWzSYcrHEviLnc1VlBVxtAal4qF/Iv8N5f7/HXhb8cdvkh7RswtEMYJrPC89/socjkuMkphRBC1C0uS5AuXryIyWQiNDTUZn9oaCjJyckOuYbZbGbKlCn07t2b9u3bW/c/8MADfP7556xbt47p06fzf//3f4wZM+aadc2ePRtfX1/rFhnp+MVUBXDrjEq72m5vejtDmwzFpJiYtnEaOcYch13+lTva4+fhxv7zWXy48YTD6hVCCFG3uHyQtjNNmjSJffv2sXTpUpv9jz/+OIMHD6ZDhw48+OCDfPbZZ3z//fccP368wrqmT59OZmamdTtz5oyzw78x2dHVplKp+OdN/yTCK4JzOef41/Z/Oezywd56ZtzeFoB/xx/lWKrjki8hhBB1h8sSpKCgIDQaDSkpKTb7U1JSKhyAXRVPPfUUy5cvZ926dTRs2PCaZWNiYgA4dqziwbl6vR4fHx+bTTiJHV1t3jpv5vSdg0alYfmJ5fx8/GeHXf6uLhH0bxWMsdjMtG/3YDIrDqtbCCFE3eCyBEmn0xEdHU18fLx1n9lsJj4+np49e1a7XkVReOqpp/j+++/57bffaNKkSaXn7N69G4AGDRpU+7rCwW6dAf5NrtnV1jmkM092siRS/9r+L85kO6ZVT6VS8fpdHfDSa0k4fYnPtp5ySL1CCCHqDpd2scXFxfHRRx+xZMkSDh48yMSJE8nNzWX8+PEAjB07lunTp1vLG41Gdu/eze7duzEajZw7d47du3fbtPxMmjSJzz//nC+//BJvb2+Sk5NJTk4mPz8fgOPHjzNr1iwSEhI4deoUP/30E2PHjuXmm2+mY8eONfsGiIrpPGDEVRNIHiv/rrbHOjxG15Cu3BxxM356P4ddPtzPnRduaw3A3JWHOZ2WW8kZQggh6hOVoigu7T949913efPNN0lOTqZz58785z//sXZ59e/fn6ioKD799FMATp06VW6LUL9+/Vi/fj1g+eu/PIsXL+bhhx/mzJkzjBkzhn379pGbm0tkZCR33XUX//znP6vUbZaVlYWvry+ZmZnS3eZMv74A298Dnwj421YwlF0zLa8oD3ete4X/9tVlNiuM+mgbO06mE+Sl4z+ju9CrWZBDryGEEKJm2fv97fIEqa6SBKmGGPPgvV5w6SR0eQjufPeaxRVF4UL+BUI8Qhxy+fMZ+UxY8gcHk7IY3imcd0Z3cUi9QgghXMPe7+96fRebqAfs7GoDyCzMZMq6KTyw4gEyCx0z0WO4nzvfTezFpFuaMfvuDg6pUwghRO0nCZKo/a6+q+2niieQdFO7cTzzOCl5Kbyy9RUc1TjqrtPw3ODWeOm1gKWVatbyAzLbthBC1GOSIIm64eq72lb/s9wiHm4evHHzG2jVWtacXsP3x753Sihf7khk0e8nGfXBNjLzipxyDSGEEK4lCZKoG67uatv1WYVdbe0C2/H3Ln8HYM6OOZzMPOnwUG7vEM6A1iE8N6QVvh5uDq9fCCGE60mCJOoOO7vaxrUbR0yDGPKL85m2cRpGk9GhYfh6uPHx2G48dFNj674jKdkyFYAQQtQjkiCJuuXWlyrtalOr1Pyr97/w0/txMP0g7/z5jsPDUKtV1mkFMvOLeOyzP7j9nd9ZcyClkjOFEELUBZIgibpF52lXV1uoZyiv9HqFCK8Ibm10q1NDKiw2EeipI7ugmMc++4M3Vh6i2GR26jWFEEI4l8yDVE0yD5KL2TGBJEChqRC9Ru/0cIzFZmb/epDFm08B0LNpIP8Z3YVgb+dfWwghhP1kHiRRv9nR1QbYJEfncs457Nb/0nRaNTOHt+PdB7rgodOw9UQaw/6ziT9OpTvlekIIIZxLEiRRN9nZ1Vbiu6PfMeKHESw9vNSpYd3eMZyfnupN8xAvUrMLGfXhNj7edMJpiZkQQgjnkARJ1F2Ne0HME5bH17irDSC/OJ8CUwHzds7j6KWjTg2reYg3P07qzR2dwik2K7y24iCTvtxFdoHMmSSEEHWFJEiibrNjAkmAB1o/QJ+IPhjNRp7f+DwFxQVODctTr+Xfozrzyh3tcNOo+GVvMne+u5nDydlOva4QQgjHkARJ1G12drWpVCpe6/0agYZAjmUcY37CfKeHplKpGNcrimVP9KSBr4ETF3MZsXAz+87JEiVCCFHbSYIk6j47u9oC3QN5rc9rAPzv0P/YcGZDjYTXtZE/yyf3oW+LIDpH+tE6zLtGriuEEKL6JEES9YOdXW19IvrwUNuHAHhp80tczL9YI+EFeun5dHwPPhgbjVZj+d+usNjE+Yz8Grm+EEKIqpEESdQPZbra4issOqXrFNoGtmVU61H46f1qLESNWoWP4crabf9acZCh/9nExiMXaiwGIYQQ9pEESdQfdna16TQ6Ph/6OX/r/De0am0NBnhFvtHE7jMZZOQVYTLLFABCCFHbSIIk6hdrV9vZa3a1uamvtOQYTUbO5Zyrieis3HUavnqiJx+P7cYtrUOs+2W+JCGEqB0kQRL1i84T7lxoeVxJVxvAmewzPPjLgzy55knyivJqIMArDG4aYtuGXoklPY9h//md3WcyajQOIYQQZUmCJOqfqN4Q86TlcSUTSHq7eZOen86prFPM3Tm3hgIs35yVhziQlMW972/h/7aektYkIYRwIUmQRP1k09X2UoXF/Ax+vN73dVSo+Pbot6w5vaYGg7Q1++4ODG4XSpFJ4aUf9zN12W7yjMUui0cIIW5kkiCJ+smmq23JNbvaYhrE8Ej7RwB4ecvLJOcm10SEZfgY3Hh/TDQvDm2NRq3ih93nGbFwM8cv5LgkHiGEuJFJgiTqryp0tU3qMon2ge3JMmYxfdN0TGZTDQVpS6VS8fjNzfjy0RiCvfUcScnhznc38+veJJfEI4QQNcVYbLZ5Xljsmt/DJSRBEvWbnV1tbmo33rj5Ddy17vyR8gef7v+05mIsR0zTQFZM7kOPJgHkFBYz8YtdvLb8AEUmc+UnCyFELVBkMpOSVcD+85lsPHKBH/48x8ebTvDGykMknL5kLff70Yu0eWklI9/bYnN+XqFrEyTXTAIjRE0p6Wr7dKilq63tndD81nKLNvJpxD9i/sE3R75hSJMhNRxoWSE+Br58NIY3Vx3mg40n+Pj3k/x1NoN3H+hKqI/B1eEJIW4wZrPCpTwjBcVmIvzcrfvfW3+cs5fySMsxkpZbSFqukbQcI5n5RRXWFeipI7qxPwBeBi35RSbSc402ZbQalXNeiJ1UitwqUy1ZWVn4+vqSmZmJj4+Pq8MRlfl1Gmx/H3wawt+2gqH8fzNFUTArZjRqTQ0HeG0r9yXz3Nd/kV1YTJCXjndGd6Vns0BXhyWEqMMURSG7sJi0HCPpuYVczLEkNmk5l5OcXCP3dWtI3xbBAKw7lMr4T3fSPsKH5ZP7Wuvp/+Y6TqWVP02KRq0iwFNHoKeOQC8dgZ56Ar10DGwbSq9mQQAUFJlIySog0EuPl9757Tb2fn9LC5K4Mdw6A46sgksnLRNI3vGfcoupVCo0qivJ0eH0w7QKaFVTUVZoSPswWoV5M/HzBA4lZ/PDn+ckQRJClCunsJjjqTkoQOdIP+v+l37Yx+n0PNJyCkm/3MpjrKTbvkOEjzVBCvTSAVBUbNuuMrpHI/KMJoK8dAR66Qnw1Fkee+rxdXdDrb52S5DBTUPjQM+qv1AnkxakapIWpDro1GZLVxvAmO8q7GorMT9hPov3LWZ239nc3vT2GgiwcvlGE+9tOM7f+jfD4Fa7WrmEEM5TUGQiNauQlOwCUrMKSc0uIOXyz9SsQsb3juLWNpaJZ9cfTuXhxTtp28CHX56+0tIzYN56TlzMLVO3p05DoJf+SgtPSWuPl56YJgG0j/AFwGRWMCsKbpq6PXxZWpCEKK3krrbt71vuartGVxuAu8bSx/7attdo5tuMNoFtairSCrnrNMQNbGl9bjYrvPTjPh6IaUS7cF8XRiaEqI48YzE5BcWEXDWu8N9rj3IqLZeUrAJSswtJzSogq+Dac6L1aRFkTZCCvfU08DUQ7K23KfP3W1tQZDITdDkZsnR96XHX2ffHlkatQoNrxwXVJGlBqiZpQaqjjLnwXm9LV1vXcRV2tQEUm4t5ZNUj/Jn6JwC3RN7C4x0fp31Q+5qKtlKLfj/JrOUH8HV3Y/MLA2qk/14IUbncwmJSswttkpzU7EJi24TSo0kAABuPXGDsJztoHebNyik3W8+Nnb+BY6ll5z8zuKkJ8TYQ6qMnxNtAyOWfoT56OkX60SzYq8ZeX10mLUhClKcKd7Vp1Vrm95/P69tfZ+3ptaw7s451Z9bRK7wXr/R6hTDPsBoOvqyRXSP4/egFhnUMl+RIiBq2cl8yZ9LzynR3pWYXklNYfouPn4ebNUEqaeHJLtU6NK5nY3KNJmsiFOqjJ9jbgI9Bi0p147TguJq0IFWTtCDVcb88Dzs+qPSuthInMk6waN8iVpxYga/el5UjV+Kudb/mOTVFURSbX5r7zmXiY3CjUaCHC6MSou4oMpnJKSgmLddoTXLaNPChVZg3ALvPZDB12W78Pdz47m+9recNnL+Bo+W09JTw0GkI9bF0dYX6GAjx1jOgdQi9m1vu3io2mckvMuGll8SnJkkLkhDXEjsTjq6u9K62Ek39mvKvPv9iYqeJnM46bU2OzIqZFza9wKDGgxjQaABqVc0PXrz6F2taTiGPffYHuYXFvH1/Z+uYBCHqI/PlQcPay4OGM/KM7D6TgUatst55BfCf+KOcTssjp7CInMJisgss436yCorJKSyioKjsnVzPDW5lTZAMbmpOXswl01NnU6Z/q2DahvsQcjkBCva+0uIT4mOotFVXq1HjXccHPNdn0oJUTdKCVA9U8a628sSfjmfK+ikANPNtxoQOE7ityW1o1a752yMlq4AnP0/gz8QMACbd0oy4ga3QVHKbrRA1raDIRGGxGV93N+u+3w6lkJZjvJLEFBaTXVBEdkHx5c02wckxFvPy8HaM6xUFwPYTadz/4TaaBnny27P9rfUOWbCRQ8nZlcbkbdBak527uzbknuiG1lj/OpNBiI+BJkG173Z0UTX2fn9LglRNkiDVE1XsaistvSCdzw98zv8O/Y+cIktTe0OvhkzoMIE7mt2BTqOrpAbHMxabef2Xg3y65RQAvZoF8p/RXQjy0l/7RCEqYDYrNnPZnLqYS/7lBKegyESesfiqJMbSKlO6leaJm5txS+sQANYeSOHRz/6gU6QfP0660mXVe85vnMvIr1Jszw1uxaRbmgNwLDWbp5fuJtLfg/cfiraW+XJ7Ipn5RXgZtPgYtHjptXgb3C7/1OJjcMNTr7G2RIn6TRIkJ5MEqZ4w5sJ7veDSKYh+GIb/u1rVZBuzWXpoKf934P+4VGhZYyjEI4Qvhn7hssHcP/11nhe+3UPe5cGe/32wK9GNA1wSi7h+iqJQZFIoLDZRUGTGx12LXmu5PTslq4ATF3Lx83CjTQPL7yOzWeHz7actLTVFZgqKbX+WJDelf/5jWBtr99TyPeeZsnQ3PZoE8OVjN1lj6fTK6msuI1GeWXe246GeUQBsO5HGqA+30SzYk/hn+lvLxC3bTVquEW+D9vJ2JYkpSWpKjpU8v/p9EMIeMgZJCHtY72obBgmfWu5qazagytV467x5rONjPNjmQb458g1L9i8hyD2IUI8rY4BMZlONLmFyR6dw2jbw5on/S+D4hVzu/2AbzYK90KhVaDUq1CoVDXwNvDfmyl/ar/y8n8S0PJ6ObUHHhn4A7DiZzrKdZ9CoQaNWo1WrLPOhqFVo1SrUl39q1Co0KhUajQq9VsOEPk2s9a4/nEpqViE9mgQQdbmLIjW7gD8TM8qtoyQ+rVptvVbJ1jjAw9qakZJVQHZBEYGeevwvjw/JLSzmWGoOJkVBURRMZjArlrEq5suPS46ZzVgf928VYp18c8/ZDI6m5NAqzNs6SV5mfhE/7T53ebK8K3Va67+8/8p1Lc/H946yrp238cgF1h5MoWsjf0Z0iQAgu6CIqcv+otCauFgSoPJ+mq/6c/bLx2KsSzWs3p/MSz/uZ0i7MGvLiUoFM3/aT1X/BL56PSytWk2xWaGw1CrrgV463DRqDG5q9Fo1nnrtVYmM2+VWGS1eVyU5HRtemacrurE/e18ehKfO9ito/v2dqxasEE7k8gRp4cKFvPnmmyQnJ9OpUyfeeecdevToUW7Z/fv3M2PGDBISEjh9+jRvv/02U6ZMqXKdBQUFPPPMMyxdupTCwkIGDx7Mf//7X0JDZUDrDSmqD/R4wtLV9uPkanW1lfBw82Bsu7GMaj2K1LxU6wDqHGMO9/x8D8OaDmNMmzH4G/wd+Qoq1DzEmx+f6sML3+5h+Z4kDqfYjsNIy7W9E2/HyXT2n8/ioZ6NrftOXszh211nq3RdT51tgvTJ5lNsPHKBt+7tZE2Q9pzJ5In/S6jqS+LQrCEYLieas385yA+7z/PPYW14tG9Ty/HkLEa+t7XK9W55YQDhlxfg/OHP83yy+SQT+zezJkjpuUZe+nF/leu9vWMDa4K091wmn209TWGR2ZogAaw9mFLleo1XJS2BXnqah3gR5ntlskGVSsWdncJRq1To3dTotRrrT8M1fpYMTAbo1zKYbdNvxUNvm9j/dlWrT3W4adR1fjZmUf+5NEFatmwZcXFxvP/++8TExLBgwQIGDx7M4cOHCQkJKVM+Ly+Ppk2bcu+99zJ16tRq1zl16lRWrFjB119/ja+vL0899RR33303mzdvdurrFbVY7Ew4usrS1bbmpWp3tZXQaXQ09G5ofb78xHLO5Zzjwz0f8n8H/o/7Wt7HuHbjCPYIvkYtjuGl1/LO6C5M7N+MjLwiis2W1o5is4Jea/sl9fStLUjPNdp8SXZs6McLt7XGZLa0ilx9vslsxmTG8vNyq0mxSUFXqt4ukX64qVU08LvyBe5t0NK1kR8mBZt6rq7/ynUsrT4mk4L6qrv2PPVa/DzcbK6n12qI8HNHpbLM/KtWqVCrQK2ytECpVCo0ai7vtxwraVUr0TzEi34tg2ly1fpQXnotQzuEXV6v70qdavWVOlQl9alKrmNZqLNEt8b+/H1Ac9pFXGlN8dBpef2uDui1agxumis/3dQYykli9Jdbba6+e3FohwYM7dCgzL/9glFdrv3hqIS7TmP3LMtC1DcuHYMUExND9+7deffddwEwm81ERkYyefJkXnjhhWueGxUVxZQpU8q0IFVWZ2ZmJsHBwXz55Zfcc889ABw6dIg2bdqwdetWbrrpptKXKpeMQaqHTv1u6WoDeOj7anW1VcSsmIlPjOejPR9xMP0gADq1jrta3MX49uOJ8IqopAYhhBCOYO/3t8vaOI1GIwkJCcTGxl4JRq0mNjaWrVur3jxub50JCQkUFRXZlGndujWNGjW65nULCwvJysqy2UQ9U9LVBpa12goc92+sVqkZ2Hggy25fxsJbF9I5uDNGs5Flh5dxx/d3cKngksOuJYQQ4vq5LEG6ePEiJpOpzLif0NBQkpOTnVZncnIyOp0OPz+/Kl139uzZ+Pr6WrfIyMhqxShqudiZ4B8FmWcsXW0OplKpuLnhzXx222d8MvgTbmpwE7GNY23GJCXlJDn8ukIIIapGRsnZafr06WRmZlq3M2fOuDok4Qwld7WB5a6247855TIqlYruYd35aNBHvNb7Nev+k5knGfLdEJ6Kf4q/LvzllGsLIYSonMsSpKCgIDQaDSkptndvpKSkEBZWvXlj7KkzLCwMo9FIRkZGla6r1+vx8fGx2UQ95cSutvK4aa7MJLwzeSeKorDh7AbG/DKGR1c9yvak7ch0ZUKIG0KxEU5uhDUz4f2+kHPBZaG4LEHS6XRER0cTHx9v3Wc2m4mPj6dnz55OqzM6Oho3NzebMocPHyYxMbHa1xX1kJO72ipyX6v7+GnET4xoPgKtSsv25O08uvpRxvw6ho1nN0qiJISof9KOw46P4MtR8EYULBkOmxdA8h44sc5lYbn0Nv+4uDjGjRtHt27d6NGjBwsWLCA3N5fx48cDMHbsWCIiIpg9ezZgGYR94MAB6+Nz586xe/duvLy8aN68uV11+vr6MmHCBOLi4ggICMDHx4fJkyfTs2dPu+9gEzeA0hNI6rygzXBo2B2cPNljlG8Us3rPYmKniSzet5jvjn7Hngt7ePH3F1k9cjUebh5Ovb4QQjhVYTac3ATH4+HYWsv0KlfzDIZmt1rWx2weW24VNcHlS428++671kkdO3fuzH/+8x9iYmIA6N+/P1FRUXz66acAnDp1iiZNmpSpo1+/fqxfv96uOuHKRJH/+9//bCaKrErXntzmf4P49QXY/t6V5+4B0GIQtBpi+R+4mhNKVsXF/It8tv8zAt0DGdduHGBZdmJt4lr6R/bHTe1WSQ1CCOFCZjOk7LUkQ8d+gzPbwXzVUjVqLTTqaZlapfmtENoB1M7r4JK12JxMEqQbhNkMB36Aw7/A0TVQkHHlmNoNGveCVrdByyEQUDZ5d5aNZzcyKX4S4Z7hPNL+EUa0GIFeI4vRCiFqiZwLlptcjsdbfuaWGkvk38SSDDW7FZr0Bb13+fU4gSRITiYJ0g3IVAxntsGRlXB4JaQdtT0e1MrSstTyNktXnMZ5Pdi/nvyVOTvmkF6QDkCwezDj2o3j3pb3ShecEKLmmYrgzA5LK9HxeEgqdReumyc0uflyUjQAApu5Jk4kQXI6SZAEaccvJ0u/QuJWMBdfOebub+mKaznY0odu8K24nmoqKC7g26Pf8un+T0nOtczh5af3Y0ybMTzS/hGbu+OEEMLh0k9eHkf0m+XOM6PtWo+Edbg8ligWImNAqyu/nhomCZKTSYIkbORnWH5RHF4JR1eX6orTWrriWg6xbA7+y6nIVMTPJ35m0d5FJGYn0tK/Jd8M/8ZmrS4hhLhuhTmWJZmOx8OxeEg/bnvcI/DyOKJYaHoLeNfOBeAlQXIySZBEhUzFcHaHpWXpyEq4eMT2eGCLK11xkTEO64orNhez+tRqfPW+9I7oDUBeUR4f7PmA0a1HE+ZZvfnFhBA3KEWBlH2WZOh4PCRuA5PxynG1Fhr2gOaXk6KwTqBWU2QuIrMwE6PJSIGpgMLiQgpNhdbHUb5RNPZpDMCFvAv8dPwn63GjyUhBcQGFpkL+1ulvRPo4ftUKSZCcTBIkYbe043BklSVZOr3ZtivO4ActBlpalprfaumac6Al+5cw7495aNVa7mx2J73CexHhFUEDrwb46/2llUkILHeFFpuLKbp8Z5VBa0CtstxFVWwuxqSYrGVVqGx+atQaa1mzYrbOVVby/5a1fA3/v6YoCoWmwitbsSUBCfEIwVdv6fJPyU3hz9Q/bROT/EsUpB3GmH6C2JSTdMqwdN8f0LnxH38/jG7uFBp8KNB5UKhxo1AporC4kKe6PMV9re4D4I/kPxi/anyFsT3d9Wke7fCopd60A9y//P5yyy0ZsoSuoV0d+bYA9n9/u3QeJCFuCIHNoOffLFtBpuWOjpKuuPx02Pu1ZVNpbLvigppf96XbBrYlOjSahJQEvj36Ld8e/dZ6zF3rzk8jfrK2LCWkJJCal0q4VzjhnuEEugdaf/EL4SqKomA0W76884vzyS/Op6C4gAJTAe2D2lunuUhISeBQ+qErxy+XKTnnpZtesq55+PHej/nmyDfWMgXFBTZJ0Hd3fEcL/xYAfLTnI/77138rjO/LoV/SIbgDAJ/t/4y3Et6qsOzHgz4mpoFlypllh5bxr+3/AspJplDx7wH/5uaGNwPw8/GfeXnLy9Z6Spd/ve/rDGw8EID4xHimbZxGoamw3Bhe7/M6w5sNByzJyXMbn6sw3ghTDp3cPCCqL9kNWrH5zA+WA6YsyLddYSCnKMf62KA1AKDX6NFr9Bg0BnQaHQatAb1Gj7/+yh+CgYZA7mx2Jwbt5TIag/W8cK/wCmOrCZIgCVGTDL7Q7i7LZjbB2Z1XuuIuHIJTmyzb6n9AYPMryVKjm6Aag667h3Xn0yGfsitlF98e/ZbTWac5n3OeC/kXKDQVEmgItJb99si3/HziZ+tznVpHA68GhHuGE+4VzjPdnsFbZ7kVN68oD71Gj8bJk2aKuiHbmE2OMYd801XJSXGB9fnQJkOtX+q/nvyVvRf32iQxecV51ueLhyzGXesOwCtbX+HbI9+iUH5Hx7r71hHkHgTAqlOr+N+h/1UY49Ndn7YmSLlFuZzLOefIt8AuJQkNgHL5P6DcGfKv3mdWzBjNxjJlSlyd3JnMpjLJkVqltiYqKpUKMhLhWDxBR36ie2EROlMxBkVBrygYFAWdwR+DX2NadRwOHR4ErZ5m+ReZ1Sj6SgKjvZL86DV6gj2CrddrF9iOPWP32NVqFuoZymt9Xqu0nCtIF1s1SRebcLj0k5e74n6FU5ttJ1Iz+Fr6+FveBi1ir7srrtBUSGpeKpHeV/r3P9zzIZvPbeZ87nlS81IxK2brMRUqEsYkWO+Me3HTi/x66lfCPMKsXXYliVS4VzhdQrqgVcvfX3VZXlEeqXmpXMi/QFp+GhfyL3Ah/wIX8y6SXZTNOwPesZZ9+ren+e1MxQs7/zHmD+s8XS9uetEmES9tw/0bCDAEAPDattdYdniZ9Zib2g2D1oC7xh2D1sCiwYusLaA/HvuR38/9jkFrwKAx4K61lDFoLY+HNhlq7Vo6l3OOi/kXbcq5a92tn1m9Rm9tPTWajBSZi6wJizWpufzTQ+thPa9k7ExFZb3dvK3/D+UX55NblGs5Xqo8gK/e1/qe5RXlkWXMqrBef72/dXqPvKI8LhVeupK8aPVoi4yoErdYxhIdW1t2ihL3AGh2i+WOs2YDwKdBhf8+9YGMQXIySZCEUxVkWbrijqyCo6sgL+3KMZXG0qLUcohlksrA5uDg8Q1F5iJSclM4n3Oe87nnySjI4OH2D1uPP7LqEXYm7yz33NLJ1Md7P+ZExgkaeDWwJFOelp9hnmHoNLXjtt8bhVkxk16Qbk14LuZf5GL+RS7kXSDLmMXsvrOtZf+29m9sOrepwrp2PLjD2tIzbeM01p5ea01IPLQe1kTFoDWw4JYFeLp5ArDy5EoOph+0SXSuTlJiGsRYE4PMwkyKzEXWL3qZNd5OigKpB6/MSXR6K1zdqqTSWOZqK5moMbyz05dQqk0kQXIySZBEjTGb4Owflm64Iysh9YDt8YCmlpalloMtY5hqYP4jk9nEhfwLnMs5Z0mics6TlJvEuZxzGE1Glty2xFr24ZUPk5CSUKYOFSpCPEJYOXKl9S/wHUk7MJqNhHuF08CzgfULWFyb0WS0JDolSU/eRS4WXCSjIIN/3PQPa7mn4p9iw9kNFdaz/YHt1paImVtmsvLkSoI9gglyDyLY3fIzyD2IYI9gBjUeZB1roiiKDPivSWYz5F2E7CTITrn8M9nyMycFzu+G7PO25/hGXlnKo0k/cPdzReS1giRITiYJknCZS6ev6or73fa2W72v5RdgyyGWu+M8AlwX52UbzmzgWMYxa2tUSTKVX5xPiEcI8ffGW8uWTqYCDAHWrrsI7wimdp1q/SI+eukoRpMRrVqLm9oNrVprfeymdsPP4FfTL9WhFEUhpyjH2q1VkgBdKrjElOgp1nLPb3ieX0/9WmE9pZOe749+j7/B35r0BLoHEuweTLBHMHc1v8tatthcLN2kNc1stty4cXXCU14ClJNiezdsebQGiOpzZaLGoBYOb2muqyRBcjJJkEStUJgNx9ddbl1aZfmrsoRKbZlnqaQrLqhlrfkFqSgKlwovkVGQQVO/ptb9L295mb8u/EVSbpJ1fEaJ0snUqOWj2J+2v9z6/fR+bBp1pXtowqoJ/Jn6p00SVfLT082Tb++4cnff/D/msz9tv7WsVqXFTeNmfTyr9yxrkrbixAqOZxzHTWNJyqxlVZZz72h+h7Vb6FD6IS7kXbCJIduYzcX8i6QVpDGh/QRrvS9veZkVJ1ZQYCoo9/VdnfS8vOVlvj36LW5qN5uWnmAPS/Iztu1Ya/dWXlGeNVZRgxQF8i4nPjnJVyU/ybZbTnLliY+VyrLqvXfYVVsD8Aq13DkbeRO4GZz6suoquc1fiBuB3hva3mHZzGY4l2BpWTqyyjLBW+JWy7Z2pmVxyFZDofVQyy9PJ64VVxmVSkWAIcA6GLfEy71eBiwJVJYxy9p9dz73fJk6gtyDCPMMs85fU2wupshURJG5qEzLR8lA26KrB75fVpI8lDiUfogdyTvKjVutUtvccbPm9BriE+PLLQtwe7PbrY8/2//ZNQcnj2o1Ci+dl/V5SXLk5eZlTXiCDEEEeQRRrFz5Ev17178zNXoqPjqfSru5ZJ0+B1MUyL9km/CUSYBSLPtMFd+FVoZHkCXZsUl+wq7s8woDr5Aa6U6/kUkLUjVJC5Ko9TISr0xQeXKj7S9o9wBLy1LroZZxCTrPiuupg0xmk80UBBkFGRSYCq4kUpd/FpuLMStmOod0tpZNSEngQt4Fm7IljxVFsRms/t3R7zicftimvqvrX3DLAmsc7/z5DpvObqJYsSRyxeZiPN08CfKwtPo8E/2MtVswOTeZInMRQe5BMg7LFRTFslzQtbq5SvZXMN9QuTwCbZOc0omPdxh4htSaNcvqK+liczJJkESdUpgDJ9bBoV8sLUz5l64c0xos6ya1HmZJmryCK65HiPqgMBsyzkDmGcsfEplnLM+zzl9pASouv3uzXO4Bl5Oc0KuSncvdXdaEKFQSn1pCEiQnkwRJ1FmmYjizDQ6tsGwZp686qLJMIdBqqCVhcvDCukI4XUm3V8bpq5KgUsnQ1X8gXIvBzzbhKS8B8gqVsT51jCRITiYJkqgXFAVS9sPhX+DQckj6y/Z4cGtLotRqGIR3AbUsPSJczGy2dHGVJDxXtwCV/Cw1wL9cBj/wiwTfRpd/RoJvhG0XmCQ+9ZIkSE4mCZKolzLOWJY+ObzCMoXA1XfUeDew3A3Xahg06QtaveviFPWXqQiyzlk+izbJT+LlbrBz9g149gq1JD0lyY9fI9vnBvm9faOSBMnJJEES9V5+BhxdY2lZOrYWjFcWo0TnbVnypPXtljlWbuBJ50QVGfMg8+yVhKd0C1B2Ely1zE25VBrwCS+b9JS0CPk2lNYfUSFJkJxMEiRxQykuhJObLMnS4V8tA1lLqLUQ1fdyV9xtli8ncePKzyg77ufqJOjquboqotFbPkd+jcp2g/lFgne4S6epEHWbJEhOJgmSuGGZzXB+15VB3hcP2x5v0NmSLLUeBiFta83klMKBFMXSCnTuD8syOGnHriREhVmVn6/ztu36Kt0N5hks492E00iC5GSSIAlx2cVjljFLh36BM9vhqhXJ8Wts6YarBZNTiutgzIXzf8LZnZaE6Owftq2IpXkEXmnt8WtcthvM4CeJs3AZSZCcTBIkIcqRk2qZmPLQCssSKFdPomednHLY5ckpZVbnWslshrSjlxOhywlR6v6y44JUGghrb1kVPqStJRHyi7R0jdWziUdF/SIJkpNJgiREJYy5cPw3S7J0ZGXZySmbDbDMt9TqNvAMcl2cN7q89CvJ0Lk/4GwCFGaWLecTAQ27QUQ3S1LUoJMkuaJOkgTJySRBEqIKTMWWNeFK5lvKSLxyrGRRXZmc0vlMRZY1+q5uHUo/Xrac1t0y71XDbleSIt+Imo9XCCeQBMnJJEESoppKJqc8tMIydkkmp3QORbHMGXT1uKGk3eUvoRHY4koyVNJlJguhinpKEiQnkwRJCAcpmZzy0HI4vVkmp6wuYy6c3305IdoJ5xIscwqVZvC7kghFdIOIruARUNPRCuEykiA5mSRIQjhB/qXLk1OuqHhyylbDILQtuPtbNrcbcLV7s9lya/25P64kRCkHQDHZlisZSF0ybqhhd0sXptxBJm5gkiA5mSRIQjhZcSGc3HjV5JQp5ZfTGiyJksHvStLk7m+Z3dvmealN7113EoW8dEuLUEl32bk/oKCcgdTe4Vdahxp2s8xJJQOphbAhCZKTSYIkRA0ymy0JwuEVcHStpeso/1LZFpOqUGkqT6JKbwY/MPg6dz4nU5FljJZ17NDOawyk7mzbXSYDqYWolCRITiYJkhAupihQmA0FGZZkqcKtnOPlDVSuCr2vnclVqTLljaHKPHfVLfZ/WCZkLHcgdfPLiVC05WdoOxlILUQ12Pv9LdPaCiHqJpXKsiK7wceyREVVFOWXnzhVlmQZsy3nF2ZatozTVbuum8dVrVG+kH4Sss+XLWfwtR03JAOphahxkiAJIW48bu6WzadB1c4zFVnG/tidWF1OrgoyLDNRF+VZtqxzV+pUaSytQSXjhhp2h4BmMr2BEC4mCZIQQthL42aZ9buqM3+bzZZFXEsnT94NLOOIZGkOIWodSZCEEMLZ1OrL45H8gCYuDkYIYQ9pwxVCCCGEKKVWJEgLFy4kKioKg8FATEwMO3bsuGb5r7/+mtatW2MwGOjQoQO//PKLzXGVSlXu9uabb1rLREVFlTk+Z84cp7w+IYQQQtQtLk+Qli1bRlxcHDNnzmTXrl106tSJwYMHk5qaWm75LVu2MHr0aCZMmMCff/7JiBEjGDFiBPv27bOWSUpKstk++eQTVCoVI0eOtKnr1VdftSk3efJkp75WIYQQQtQNLp8HKSYmhu7du/Puu+8CYDabiYyMZPLkybzwwgtlyt9///3k5uayfPly676bbrqJzp078/7775d7jREjRpCdnU18fLx1X1RUFFOmTGHKlCnVilvmQRJCCCHqHnu/v13agmQ0GklISCA2Nta6T61WExsby9atW8s9Z+vWrTblAQYPHlxh+ZSUFFasWMGECRPKHJszZw6BgYF06dKFN998k+Li4nJqsCgsLCQrK8tmE0IIIUT95NK72C5evIjJZCI0NNRmf2hoKIcOHSr3nOTk5HLLJycnl1t+yZIleHt7c/fdd9vs//vf/07Xrl0JCAhgy5YtTJ8+naSkJObPn19uPbNnz+aVV16x96UJIYQQog6r97f5f/LJJzz44IMYDAab/XFxcdbHHTt2RKfT8cQTTzB79mz0+rLLAUyfPt3mnKysLCIjI50XuBBCCCFcxqUJUlBQEBqNhpQU21W6U1JSCAsLK/ecsLAwu8tv2rSJw4cPs2zZskpjiYmJobi4mFOnTtGqVasyx/V6fbmJkxBCCCHqH5eOQdLpdERHR9sMnjabzcTHx9OzZ89yz+nZs6dNeYA1a9aUW37RokVER0fTqVOnSmPZvXs3arWakJCQKr4KIYQQQtQ3Lu9ii4uLY9y4cXTr1o0ePXqwYMECcnNzGT9+PABjx44lIiKC2bNnA/D000/Tr18/3nrrLYYNG8bSpUv5448/+PDDD23qzcrK4uuvv+att94qc82tW7eyfft2brnlFry9vdm6dStTp05lzJgx+Pv7O/9FCyGEEKJWc3mCdP/993PhwgVmzJhBcnIynTt3ZuXKldaB2ImJiaivWrSxV69efPnll/zzn//kxRdfpEWLFvzwww+0b9/ept6lS5eiKAqjR48uc029Xs/SpUt5+eWXKSwspEmTJkydOtVmjJEQQgghblwunweprpJ5kIQQQoi6p07MgySEEEIIURu5vIutrippeJMJI4UQQoi6o+R7u7IONEmQqik7OxtA5kISQggh6qDs7Gx8fX0rPC5jkKrJbDZz/vx5vL29UalU5Zbp3r07O3furNKxkgkoz5w5U+fGNl3r9dbW61S3rqqeZ295e8pVVqa+fbZupM9Vdc511GdLPle1/1p18XNVWRlXfK4URSE7O5vw8HCbm8BKkxakalKr1TRs2PCaZTQaTYX/sNc6BuDj41OnftlA5a+pNl6nunVV9Tx7y9tTrrIy9e2zdSN9rqpzrqM+W/K5qv3Xqoufq8rKuOpzda2WoxIySNuJJk2aVK1jdVVNvSZHXqe6dVX1PHvL21OusjL17bN1I32uqnOuoz5b8rmq/deqi5+rysrU5s+VdLHVMjJ9gHAW+WwJZ5DPlXCG2vC5khakWkav1zNz5kxZ9004nHy2hDPI50o4Q234XEkLkhBCCCFEKdKCJIQQQghRiiRIQgghhBClSIIkhBBCCFGKJEhCCCGEEKVIgiSEEEIIUYokSHXYmTNn6N+/P23btqVjx458/fXXrg5J1BN33XUX/v7+3HPPPa4ORdRhy5cvp1WrVrRo0YKPP/7Y1eGIeqQmfkfJbf51WFJSEikpKXTu3Jnk5GSio6M5cuQInp6erg5N1HHr168nOzubJUuW8M0337g6HFEHFRcX07ZtW9atW4evry/R0dFs2bKFwMBAV4cm6oGa+B0lLUh1WIMGDejcuTMAYWFhBAUFkZ6e7tqgRL3Qv39/vL29XR2GqMN27NhBu3btiIiIwMvLi9tuu43Vq1e7OixRT9TE7yhJkJxo48aNDB8+nPDwcFQqFT/88EOZMgsXLiQqKgqDwUBMTAw7duyo1rUSEhIwmUxERkZeZ9SitqvJz5W4cV3v5+z8+fNERERYn0dERHDu3LmaCF3UcnXld5gkSE6Um5tLp06dWLhwYbnHly1bRlxcHDNnzmTXrl106tSJwYMHk5qaai3TuXNn2rdvX2Y7f/68tUx6ejpjx47lww8/dPprEq5XU58rcWNzxOdMiPLUmc+WImoEoHz//fc2+3r06KFMmjTJ+txkMinh4eHK7Nmz7a63oKBA6du3r/LZZ585KlRRhzjrc6UoirJu3Tpl5MiRjghT1HHV+Zxt3rxZGTFihPX4008/rXzxxRc1Eq+oO67nd5izf0dJC5KLGI1GEhISiI2Nte5Tq9XExsaydetWu+pQFIWHH36YAQMG8NBDDzkrVFGHOOJzJURl7Pmc9ejRg3379nHu3DlycnL49ddfGTx4sKtCFnVEbfodJgmSi1y8eBGTyURoaKjN/tDQUJKTk+2qY/PmzSxbtowffviBzp0707lzZ/bu3euMcEUd4YjPFUBsbCz33nsvv/zyCw0bNpTkStiw53Om1Wp56623uOWWW+jcuTPPPPOM3MEmKmXv77Ca+B2ldXiNosb06dMHs9ns6jBEPbR27VpXhyDqgTvuuIM77rjD1WGIeqgmfkdJC5KLBAUFodFoSElJsdmfkpJCWFiYi6ISdZ18rkRNkM+ZcJba9NmSBMlFdDod0dHRxMfHW/eZzWbi4+Pp2bOnCyMTdZl8rkRNkM+ZcJba9NmSLjYnysnJ4dixY9bnJ0+eZPfu3QQEBNCoUSPi4uIYN24c3bp1o0ePHixYsIDc3FzGjx/vwqhFbSefK1ET5HMmnKXOfLacdn+cUNatW6cAZbZx48ZZy7zzzjtKo0aNFJ1Op/To0UPZtm2b6wIWdYJ8rkRNkM+ZcJa68tmStdiEEEIIIUqRMUhCCCGEEKVIgiSEEEIIUYokSEIIIYQQpUiCJIQQQghRiiRIQgghhBClSIIkhBBCCFGKJEhCCCGEEKVIgiSEEEIIUYokSEIIIYQQpUiCJIQQQghRiiRIQgi79O/fnylTptSaelx9DSFE/SYJkhC12MMPP4xKpUKlUuHm5kaTJk14/vnnKSgocHVo1fbdd98xa9Ysh9VXXjLk6GvcaGoqwUxISGDUqFGEh4djMBho3rw5Y8aMYf/+/U6/thCVkQRJiFpuyJAhJCUlceLECd5++20++OADZs6c6eqwqsVoNBIQEIC3t7dTr1MT13AWo9Ho6hAc5lqv5aOPPiImJgZfX1++++47Dh8+zAcffEB2djZffvllDUYpRAUUIUStNW7cOOXOO++02Xf33XcrXbp0sT43mUzK66+/rkRFRSkGg0Hp2LGj8vXXX9uck5WVpTzwwAOKh4eHEhYWpsyfP1/p16+f8vTTTyuKoiiNGzdW3n77bZtzOnXqpMycOdP6/OryiqIov/76q9K7d2/F19dXCQgIUIYNG6YcO3bMpo5+/fopkyZNUp5++mklMDBQ6d+/v009J0+eVIAyW79+/ey6xrhx48qce/LkyTKxFhQUKJMnT1aCg4MVvV6v9O7dW9mxY0eZWCdPnqw899xzir+/vxIaGmrz+stT8vomTZqk+Pj4KIGBgco///lPxWw22/0+lfce2XveU089pTz99NOKn5+fEhISonz44YdKTk6O8vDDDyteXl5Ks2bNlF9++cUm5so+LxW9p/Z8zip6LaVt2rRJUavVyn//+99yj6elpV3zfReiJkgLkhB1yL59+9iyZQs6nc66b/bs2Xz22We8//777N+/n6lTpzJmzBg2bNhgLRMXF8fmzZv56aefWLNmDZs2bWLXrl3XFUtubi5xcXH88ccfxMfHo1arueuuuzCbzTbllixZgk6nY/Pmzbz//vs2xyIjI0lKSrJuf/75J4GBgdx88812XePf//43PXv25LHHHrPWERkZWSbW559/nm+//ZYlS5awa9cumjdvzuDBg0lPTy8Tq6enJ9u3b2fu3Lm8+uqrrFmz5prvw5IlS9BqtezYsYN///vfzJ8/n48//rhK71N575G95wUFBbFjxw4mT57MxIkTuffee+nVqxe7du1i0KBBPPTQQ+Tl5VnPqezzUtF7as/nrLJ/7xJxcXH069ePiRMnlns8ICDgmu+5EDXC1RmaEKJi48aNUzQajeLp6ano9XoFUNRqtfLNN98oimJpGfHw8FC2bNlic96ECROU0aNHK4piaT1yc3Oz+Ws/IyND8fDwuK4WpNIuXLigAMrevXttzrm6teta9eTn5ysxMTHK7bffrphMpipdo3R9V+/LyclR3NzclC+++MJ63Gg0KuHh4crcuXNtzunTp49NPd27d1emTZtW4Wvu16+f0qZNG5sWo2nTpilt2rSp8JzSr6G898je866Ot7i4WPH09FQeeugh676kpCQFULZu3aooin2fl5K6S7fA2XteZa/lwIEDCmD9DAtRW2ldmJsJIexwyy238N5775Gbm8vbb7+NVqtl5MiRABw7doy8vDwGDhxoc47RaKRLly4AnDhxgqKiInr06GE97uvrS6tWra4rrqNHjzJjxgy2b9/OxYsXrS0biYmJtG/f3louOjrarvoeeeQRsrOzWbNmDWq1ukrXuJbjx49TVFRE7969rfvc3Nzo0aMHBw8etCnbsWNHm+cNGjQgNTX1mvXfdNNNqFQq6/OePXvy1ltvYTKZ0Gg0dr2G8t4je867Ol6NRkNgYCAdOnSw7gsNDQWwvgZ7Pi/lqcp5lf17l7Rc2vu5EMJVJEESopbz9PSkefPmAHzyySd06tSJRYsWMWHCBHJycgBYsWIFERERNufp9Xq7r6FWq1EUxWZfUVHRNc8ZPnw4jRs35qOPPiI8PByz2Uz79u3LDMz19PSs9PqvvfYaq1atYseOHTaDq+29hqO4ubnZPFepVGW6DKvKntdQ3ntkz3nlxXv1vpLEreQ1VPfzUpXzKvv3Lunu8/LyumY5IVxNEiQh6hC1Ws2LL75IXFwcDzzwAG3btkWv15OYmEi/fv3KPadp06a4ubmxc+dOGjVqBEBmZiZHjhyxjvUJDg4mKSnJek5WVhYnT56sMI60tDQOHz7MRx99RN++fQH4/fffq/Wavv32W1599VV+/fVXmjVrVuVr6HQ6TCZThfU3a9bMOiamcePGgCX527lzp0NuZd++fbvN823bttGiRQs0Gk213ydHvr9Xs+fzAmXfU3vPs0dJ69emTZu46667yhzPz8/H3d39uq4hhCNIgiREHXPvvffy3HPPsXDhQp599lmeffZZpk6ditlspk+fPmRmZrJ582Z8fHwYN24c3t7ejBs3jueee46AgABCQkKYOXMmarXa2sIwYMAAPv30U4YPH46fnx8zZsxAo9FUGIO/vz+BgYF8+OGHNGjQgMTERF544YUqv5Z9+/YxduxYpk2bRrt27UhOTgYsX9D2XiMqKort27dz6tQpvLy8ygzw9fT0ZOLEidbX36hRI+bOnUteXh4TJkyocsylJSYmEhcXxxNPPMGuXbt45513eOutt4Dqv0+Oen9L8/b2rvTzAuW/p/acZ4+ePXsyaNAg/va3v5GTk0PPnj0xm83s3LmT9957jw8//JC2bdte92sV4npJgiREHaPVannqqaeYO3cuEydOZNasWQQHBzN79mxOnDiBn58fXbt25cUXX7SeM3/+fJ588kluv/12fHx8eP755zlz5gwGgwGA6dOnc/LkSW6//XZ8fX2ZNWvWNVuQ1Go1S5cu5e9//zvt27enVatW/Oc//6F///5Vei1//PEHeXl5vPbaa7z22mvW/f369WP9+vV2XePZZ59l3LhxtG3blvz8/HLjnjNnDmazmYceeojs7Gy6devGqlWr8Pf3r1K85Rk7diz5+fn06NEDjUbD008/zeOPPw5U/31y1PtbHns+L+W9p/acZ68ff/yRt99+m7lz53LixAnrJJHDhw+ndevW1/0ahXAElVJ64IEQot7Lzc0lIiKCt956yyGtKDeq/v3707lzZxYsWODqUIQQDiYtSELcAP78808OHTpEjx49yMzM5NVXXwXgzjvvdHFkQghRO0mCJMQNYt68eRw+fBidTkd0dDSbNm0iKCjI1WEJIUStJF1sQgghhBClyFIjQgghhBClSIIkhBBCCFGKJEhCCCGEEKVIgiSEEEIIUYokSEIIIYQQpUiCJIQQQghRiiRIQgghhBClSIIkhBBCCFGKJEhCCCGEEKVIgiSEEEIIUYokSEIIIYQQpfw/+qEipcUXLIkAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(Cs, cv_risks, \"-.\", label=\"5-fold CV\")\n", + "plt.plot(Cs, alo_risks, \"-\", label=\"RandALO\")\n", + "plt.plot(Cs, test_risks, \"--\", label=\"True risk\")\n", + "\n", + "plt.xlabel(r\"Regularization parameter $C$\")\n", + "plt.ylabel(\"Risk\")\n", + "plt.xscale(\"log\")\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/randalo/__init__.py b/randalo/__init__.py new file mode 100644 index 0000000..82ddecd --- /dev/null +++ b/randalo/__init__.py @@ -0,0 +1,9 @@ +from . import modeling_layer +from . import randalo +from . import reductions +from . import truncnorm +from . import utils + +from .randalo import RandALO +from .modeling_layer import HyperParameter, Regularizer, SquareRegularizer, L1Regularizer, L2Regularizer, Loss, LogisticLoss, MSELoss +from .reductions import Jacobian, gen_cvxpy_jacobian diff --git a/randalo/modeling_layer.py b/randalo/modeling_layer.py new file mode 100644 index 0000000..e685526 --- /dev/null +++ b/randalo/modeling_layer.py @@ -0,0 +1,252 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + +import cvxpy as cp +import numpy as np +import torch + + +@dataclass +class HyperParameter: + parameter: cp.Parameter = field(default_factory=cp.Parameter) + scale: float = field(init=False, default=1.0) + + def __mul__(self, r): + # TODO: __mul__ should not have side-effects + if isinstance(r, float | np.float32): + return HyperParameter(parameter=self.parameter, scale=self.scale * r) + else: + return NotImplemented + + @property + def value(self): + return self.scale * self.parameter.value + + @value.setter + def value(self, val): + assert self.scale == 1.0, "Cannot set the value of a scaled parameter" + self.parameter.value = val + + +@dataclass +class Regularizer(ABC): + linear: np.ndarray | list[int] = field(default=None) + scale: float = field(init=False, default=1.0) + parameter: HyperParameter = field(init=False, default=None) + + def __mul__(self, r): + if isinstance(r, HyperParameter): + if self.parameter is not None: + raise TypeError("Cannot have multiple parameters") + out = type(self)(linear=self.linear) + out.scale = self.scale + out.parameter= r + return out + # elif isinstance(r, float | np.float32): # <- didn't allow integers or other floats... + else: + out = type(self)(linear=self.linear) + out.scale = self.scale * r + out.parameter = self.parameter + return out + # else: + # raise TypeError("Multiply must be with either a scalar or HyperParameter") + return NotImplemented + + def __rmul__(self, r): + return self * r + + def __truediv__(self, r): + return self * (1 / r) + + def __add__(self, rhs): + if rhs == 0: + return self + if isinstance(rhs, Sum): + return Sum([self] + rhs.exprs) + elif isinstance(rhs, Regularizer): + return Sum([self, rhs]) + else: + raise TypeError("Addend must be a Regularizer or Sum") + + def __radd__(self, lhs): + return self + lhs + + @abstractmethod + def to_cvxpy(self, variable, func): + expr = func(obj.linear @ variable if obj.linear is not None else variable) + if obj.parameter is None: + return obj.scale * expr + else: + return obj.scale * obj.parameter.scale * obj.parameter.parameter * expr + + @abstractmethod + def get_constraint_hessian_mask(self, beta_hat, epsilon=1e-6): + """ + Returns two matrices and a vector[bool] (or None if they would be the + 0 matrix or all true) such that + the hessian of this regularizer can be viewed as + infinity * (first matrix) + (second matrix) + infty * (third matrix) + first matrix is called the "constraint matrix" second matrix is called + the "hessian matrix" + + The third matrix is given by + diag = torch.zero(n) + diag[mask] = 1 + third matrix = torch.diag(diag) + ie any entry associated with a False in mask is held to always be zeros + """ + + def _scale(self): + if self.parameter is not None: + return self.scale * self.parameter.value + else: + return self.scale + + +class SquareRegularizer(Regularizer): + def to_cvxpy(self, variable): + super().to_cvxpy(variable, cp.sum_squares) + + def get_constraint_hessian_mask(self, beta_hat, epsilon=1e-6): + scale = self._scale() + if scale == 0.0: + return None, None, None + if self.linear is None: + return None, torch.diag( + 2 * scale * torch.ones_like(beta_hat, dtype=beta_hat.dtype)), None + elif isinstance(linear, list): + diag = torch.zeros_like(beta_hat) + diag[linear] = scale + return None, torch.diag(diag), None + else: + A = utils.to_tensor(linear) + return None, torch.diag(scale * (A.mT @ A)), None + + +class L1Regularizer(Regularizer): + def to_cvxpy(self, variable): + super().to_cvxpy(variable, cp.norm1) + + def get_constraint_hessian_mask(self, beta_hat, epsilon=1e-6): + scale = self._scale() + mask = torch.ones_like(beta_hat, dtype=bool) + if self.linear is None: + mask[torch.abs(beta_hat) <= epsilon] = False + return None, None, mask + elif isinstance(linear, list): + mask[linear][torch.abs(beta_hat[linear]) <= epsilon] = False + return None, None, mask + else: + A = utils.from_numpy(linear) + return A[torch.abs(A @ beta_hat) <= epsilon, :], None, None + + +class L2Regularizer(Regularizer): + def to_cvxpy(self, variable): + super().to_cvxpy(variable, cp.norm2) + + def get_constraint_hessian_mask(self, beta_hat, epsilon=1e-6): + linear = self.linear + if self.linear is None: + norm = torch.linalg.norm(beta_hat) + if norm <= epsilon: + mask = torch.zeros_like(beta_hat, dtype=bool) + return None, None, mask + + tilde_beta_hat_2d = torch.atleast_2d(beta_hat).T / norm + hessian = self._scale() * (torch.eye(beta_hat.shape) - beta_hat_2d @ beta_hat_2d.T) + return None, hessian, None + elif isinstance(linear, list): + norm = torch.linalg.norm(beta_hat[linear]) + if norm <= epsilon: + mask = torch.ones_like(beta_hat, dtype=bool) + mask[linear] = False + return None, None, mask + tilde_b = torch.atleast_2d(torch.zeros_like(beta_hat)).T / norm + tilde_b[linear] = beta_hat[linear] + diag = torch.zero_like(beta_hat) + diag[linear] = 1.0 + return None, self._scale() * (torch.diag(diag) - tilde_b @ tilde_b.T), None + + else: + Lb = linear @ beta_hat + norm = torch.linalg.norm(Lb) + tilde_Lb = torch.atleast_2d(Lb).T / norm + if Lb <= epsilon: + return linear, None, None + return None, self._scale() * linear.T @ ( + torch.eye(Lb.shape[0]) - Lb_2d @ Lb_2d.T) @ linear, None + + +class HuberRegularizer(Regularizer): + def to_cvxpy(self, variable): + super().to_cvxpy(variable, cp.huber) + + +@dataclass +class Sum: + exprs: list[Regularizer] + + def __mul__(self, r): + return Sum([r * expr for expr in self.exprs]) + + def __rmul__(self, r): + return self * r + + def __truediv__(self, r): + return self * (1 / r) + + def __add__(self, r): + if isinstance(r, Sum): + return Sum(r.exprs + self.exprs) + else: + return NotImplemented + + def to_cvxpy(self, variable): + return cp.sum([term.to_cvxpy(variable) for term in terms]) + + def get_constraint_hessian_mask(self, beta_hat, epsilon=1e-6): + constraints = [] + hessians = [] + mask = torch.ones_like(beta_hat, dtype=bool) + for reg in self.exprs: + cons, hess, m = reg.get_constraint_hessian_mask(beta_hat, epsilon) + if cons is not None: + constraints.append(cons) + if hess is not None: + hessians.append(hess) + if m is not None: + mask &= m + + constraints = torch.vstack(constraints) if len(constraints) > 0 else None + hessians = sum(hessians) if len(hessians) > 0 else None + return constraints, hessians, mask + + +class Loss(ABC): + def __call__(self, y, z): + return torch.mean(self.func(y, z)) + + @abstractmethod + def func(self, y, z): + pass + + @abstractmethod + def to_cvxpy(self, X, y, variable): + pass + + +class LogisticLoss(Loss): + def func(self, y, z): + return torch.log(1 + torch.exp(-y * z)) + + def to_cvxpy(self, y, z): + return cp.sum(cp.logistic(-cp.multiply(y, z))) / np.prod(y.shape) + + +class MSELoss(Loss): + def func(self, y, z): + return (y - z) ** 2 + + def to_cvxpy(self, y, z): + return cp.sum_squares(y - z) / np.prod(y.shape) diff --git a/randalo/randalo.py b/randalo/randalo.py new file mode 100644 index 0000000..777f1b5 --- /dev/null +++ b/randalo/randalo.py @@ -0,0 +1,344 @@ +from typing import Callable + +import linops as lo +import numpy as np +import sklearn.base +import torch + +from . import modeling_layer as ml +from . import sklearn_integration as ski +from . import truncnorm +from . import utils + + +class RandALO(object): + def __init__( + self, + loss: ml.Loss = None, + jac: lo.LinearOperator = None, + y: torch.Tensor | np.ndarray = None, + y_hat: torch.Tensor | np.ndarray = None, + dtype: torch.dtype = torch.float32, + device: torch.device = None, + rng: torch.Generator = None, + ): + """Initialize the RandALO object. + + Parameters + ---------- + loss : Loss + Loss function. + jac : LinearOperator + Jacobian operator. + y : torch.Tensor | np.ndarray + True labels. + y_hat : torch.Tensor | np.ndarray + Predicted values. + dtype : torch.dtype, optional + Data type for tensors, by default torch.float32. + device : torch.device, optional + Device for tensors, by default None. + rng : torch.Generator, optional + Random number generator, by default None. + """ + + if loss is None: + raise ValueError("loss function must be provided") + self._loss = loss + + if jac is None: + raise ValueError("Jacobian operator must be provided") + self._jac = jac + + if y is None: + raise ValueError("label values must be provided") + y = utils.to_tensor(y, dtype=dtype, device=device) + + if y_hat is None: + raise ValueError("predicted values must be provided") + y_hat = utils.to_tensor(y_hat, dtype=dtype, device=device) + + self._dtype = dtype + self._device = y.device + if rng is None: + rng = torch.Generator(device=device) + rng.seed() + self._rng = rng + + # check dtypes and devices + # assert self._jac.dtype == self._dtype + # assert self._jac.device == self._device + + # compute derivatives of loss function + ( + self._y, + self._y_hat, + self._dloss_dy_hat, + self._d2loss_dboth, + self._d2loss_dy_hat2, + ) = utils.compute_derivatives(self._loss, y, y_hat) + + # precompute some quantities for working with generic form + # a = dloss_dy_hat + # b = d2loss_dy_hat2 + # c = d2loss_dboth + self._a_c = self._dloss_dy_hat / self._d2loss_dy_hat2 + self._c_b = self._d2loss_dy_hat2 / self._d2loss_dboth + + # the normalized Jacobian has diagonal values in the range [0, 1] + self._normalized_jac = self._jac @ lo.DiagonalOperator(-self._c_b) + + self._n_matvecs = 0 + self._normalized_diag_jac_estims = None + self._y_tilde_exact = None + + def evaluate( + self, + risk_fun: Callable[[torch.Tensor, torch.Tensor], float], + n_matvecs: int = 100, + subsets: list[list[int]] | int = 50, + ) -> float: + """Evaluate the risk function using RandALO: + + R = 1 / n * sum_i risk(y_i, y_hat_i). + + Parameters + ---------- + risk_fun : Callable[[torch.Tensor, torch.Tensor], torch.Tensor] + The risk function to evaluate. + n_matvecs : int, optional + The number of Jacobian–vector products to compute for the RandALO + method, by default 100. + subsets : list[list[int]] | int, optional + The subsets of Jacobian–vector products to use for the debiasing + step in RandALO. If an integer is provided, the subsets are chosen + randomly with sizes uniformly taken between `n_matvecs // 2` and + `n_matvecs`. By default 50. + + Returns + ------- + float + The risk estimate. + """ + self._do_diag_jac_estims_upto(n_matvecs) + + # compute BKS estimates for subsets of Jacobian–vector products + if isinstance(subsets, int): + subsets = [ + torch.randperm(n_matvecs, generator=self._rng)[:m] + for m in torch.linspace(n_matvecs // 2, n_matvecs, subsets, dtype=int) + ] + mixing_matrix = utils.create_mixing_matrix(n_matvecs, subsets) + mus = self._normalized_diag_jac_estims[:, :n_matvecs] @ mixing_matrix + m_primes = torch.sum(mixing_matrix > 0, dim=0, keepdim=True) + normalized_diag_jacs = self._uniform_map_estimates(mus, m_primes) + y_tildes = [ + self._y_tilde_from_normalized_jac(normalized_diag_jacs[:, j]) + for j in range(len(subsets)) + ] + risks = [risk_fun(self._y, y_tilde) for y_tilde in y_tildes] + # return utils.robust_y_intercept(1 / m_primes, risks), m_primes, risks + return utils.robust_y_intercept(1 / m_primes, risks) + + def evaluate_bks( + self, + risk_fun: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], + n_matvecs: int = 100, + ) -> float: + """Evaluate the risk function using the plug-in BKS estimate. + + Parameters + ---------- + risk_fun : Callable[[torch.Tensor, torch.Tensor], float] + The risk function to evaluate. + n_matvecs : int, optional + The number of Jacobian–vector products to compute for the BKS + method, by default 100. + + Returns + ------- + float + The risk estimate. + """ + + self._do_diag_jac_estims_upto(n_matvecs) + + mus = self._normalized_diag_jac_estims[:, :n_matvecs].mean(dim=1) + normalized_diag_jac_bks = self._uniform_map_estimates( + mus, utils.unsqueeze_scalar_like(n_matvecs, mus) + ) + + y_tilde = self._y_tilde_from_normalized_jac(normalized_diag_jac_bks) + return risk_fun(self._y, y_tilde) + + def evaluate_alo( + self, risk_fun: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] + ) -> float: + """Perform exact ALO. + + Parameters + ---------- + risk_fun : Callable[[torch.Tensor, torch.Tensor], float] + The risk function to evaluate. + + Returns + ------- + float + The risk estimate. + + """ + if self._y_tilde_exact is None: + if hasattr(self._jac, "diag"): + self._y_tilde_exact = self._y_tilde_from_jac(self._jac.diag()) + else: + self._y_tilde_exact = self._y_tilde_from_normalized_jac( + torch.diag(self._normalized_jac @ torch.eye(self._y.shape[0])) + ) + + return risk_fun(self._y, self._y_tilde_exact) + + def _y_tilde_from_normalized_jac( + self, normalized_jac: torch.Tensor + ) -> torch.Tensor: + """Compute the corrected predictions using the normalized Jacobian. + + Parameters + ---------- + normalized_jac : torch.Tensor + The transformed diagonal of the Jacobian. + + Returns + ------- + torch.Tensor + The corrected predictions. + """ + normalized_jac = torch.clamp(normalized_jac, 0, 1) + return self._y_hat + self._a_c * normalized_jac / (1 - normalized_jac) + + def _y_tilde_from_jac(self, diag_jac: torch.Tensor) -> torch.Tensor: + """Convenience method for computing the corrected predictions. + + First transforms the diagonal of the Jacobian into a generic form, + then computes the corrected predictions. + + Parameters + ---------- + diag_jac : Tensor + The diagonal of the Jacobian. + + Returns + ------- + Tensor + The corrected predictions. + """ + + return self._y_tilde_from_normalized_jac(-diag_jac * self._c_b) + + def _get_matvecs(self, n_matvecs: int) -> tuple[torch.Tensor, torch.Tensor]: + """Compute random Jacobian–vector products using random Rademacher + vectors. + + Parameters + ---------- + n_matvecs : int + The number of samples to compute. + + Returns + ------- + [Tensor, Tensor] + The matrix–vector products and the random vectors. + """ + Omega = ( + torch.randint( + 0, + 2, + (self._y.shape[0], n_matvecs), + generator=self._rng, + dtype=self._dtype, + ) + * 2.0 + - 1 + ) + return self._normalized_jac @ Omega, Omega + + def _do_diag_jac_estims_upto(self, n_matvecs: int) -> None: + """Compute more diagonal Jacobian estimates. + + Parameters + ---------- + n_matvecs : int + The desired total number of samples. + """ + + if n_matvecs <= self._n_matvecs: + return + + # compute the matrix–vector products + matvecs, Omega = self._get_matvecs(n_matvecs - self._n_matvecs) + + # update the diagonal Jacobian estimates + normalized_diag_jac_estims = matvecs * Omega + if self._normalized_diag_jac_estims is None: + self._normalized_diag_jac_estims = normalized_diag_jac_estims + else: + self._normalized_diag_jac_estims = torch.cat( + (self._normalized_diag_jac_estims, normalized_diag_jac_estims), dim=1 + ) + self._n_matvecs = n_matvecs + + # compute statistics for truncated normal MAP estimation + self._normalized_diag_jac_stds = self._normalized_diag_jac_estims.std(dim=1) + + def _uniform_map_estimates(self, mus: torch.Tensor, ms: torch.Tensor): + """Compute MAP estimates with a Uniform[0, 1] prior given sample means. + Uses the standard deviations computed from all available samples, even + for sample means computed from a subset of samples. + + Parameters + ---------- + mus : torch.Tensor + The sample means of the diagonal estimates. + ms: torch.Tensor + The number of samples, broadcastable with `mus`. + + Returns + ------- + torch.Tensor + The MAP estimates. + """ + return truncnorm.truncnorm_mean( + mus, + self._normalized_diag_jac_stds.reshape(-1, *([1] * (mus.ndim - 1))) + / torch.sqrt(ms), + utils.unsqueeze_scalar_like(0.0, mus), + utils.unsqueeze_scalar_like(1.0, mus), + ) + + @classmethod + def from_sklearn( + cls, + model: sklearn.base.BaseEstimator = None, + X: torch.Tensor | np.ndarray = None, + y: torch.Tensor | np.ndarray = None, + ) -> "RandALO": + """Instantiate a RandALO object from a scikit-learn model. + + Automatically converts a fitted scikit-learn model to a RandALO object + by mapping the model and parameters to the appropriate loss and Jacobian. + + Parameters + ---------- + model : sklearn.base.BaseEstimator + The fitted scikit-learn estimator. + X : torch.Tensor | np.ndarray + The training data that the model was fitted on. + y : torch.Tensor | np.ndarray | list + The training labels that the model was fitted on. + + Returns + ------- + RandALO + The instantiated RandALO object. + """ + loss, jac, y, y_hat = ski.map_sklearn(model, X, y) + return cls(loss=loss, jac=jac, y=y, y_hat=y_hat) diff --git a/randalo/reductions.py b/randalo/reductions.py new file mode 100644 index 0000000..719699a --- /dev/null +++ b/randalo/reductions.py @@ -0,0 +1,139 @@ +import functools +from typing import Callable, Literal +from dataclasses import dataclass, field + +import numpy as np +import linops as lo +import torch + +from . import modeling_layer as ml +from . import utils + +def gen_cvxpy_jacobian(loss, regularizer, X, variable, y, inversion_method=None): + prob = transform_model_to_cvxpy(loss, regularizer, X, y, variable) + J = Jacobian(y, X, lambda: variable.value, loss, regularizer, inversion_method=None) + return prob, J + +def transform_model_to_cvxpy(loss, regularizer, X, y, variable): + import cvxpy as cp + return cp.Problem( + cp.Minimize( + loss.to_cvxpy(y, X @ regularizer) + + regularizer.to_cvxpy(variable) + ) + ) + + +class Jacobian(lo.LinearOperator): + solution_func: Callable[[], torch.Tensor] + loss: ml.Loss + regularizer: ml.Sum | ml.Regularizer + inverse_method: Literal[None, "minres", "cholesky"] + + supports_operator_matrix = True + + def __init__(self, y, X, solution_func, loss, regularizer, inverse_method=None): + self.solution_func = solution_func + self.loss = loss + self.regularizer = regularizer + self.inverse_method = inverse_method + self.y = utils.to_tensor(y) + self.X = utils.to_tensor(X) + + @property + def _shape(self): + n = self.y.shape[0] + return (n, n) + + _diag: torch.Tensor = None + + # @functools.cached_property + def diag(self): + return torch.diag(self @ torch.eye(self.shape[1])) + + def _matmul_impl(self, rhs): + rhs = utils.to_tensor(rhs) + needs_squeeze = False + if len(rhs.shape) == 1: + rhs = rhs.unsqueeze(-1) + needs_squeeze = True + beta_hat = utils.to_tensor(self.solution_func()) + y = self.y + X = self.X + _, _, _, d2loss_dboth, d2loss_dy_hat2 = utils.compute_derivatives( + self.loss, y, X @ beta_hat + ) + + constraints, hessians, mask = \ + self.regularizer.get_constraint_hessian_mask(beta_hat) + if mask is not None: + X_mask = X[:, mask] + else: + X_mask = X + rhs_scaled = -d2loss_dboth[:, None] * rhs + + if constraints is None and hessians is None: + sqrt_d2loss_dy_hat2 = torch.sqrt(d2loss_dy_hat2)[:, None] + tilde_X = sqrt_d2loss_dy_hat2 * X_mask + Q, _ = torch.linalg.qr(tilde_X) + return ( + Q @ (Q.T @ (rhs_scaled / sqrt_d2loss_dy_hat2)) + ) / sqrt_d2loss_dy_hat2 + elif constraints is None: + # TODO: double check this doesn't need additional scaling + kkt_rhs = X_mask.T @ rhs_scaled + if mask is not None: + hessians_mask = hessians[mask, :][:, mask] + else: + hessians_mask = hessians + P = X_mask.T @ (d2loss_dy_hat2[:, None] * X_mask) + hessians_mask + R = torch.linalg.cholesky(P, upper=True) + v = torch.linalg.solve_triangular( + R, torch.linalg.solve_triangular(R.T, kkt_rhs, upper=False), upper=True + ) + else: + # TODO: double check this doesn't need additional scaling + if mask is not None: + constraints_mask = constraints[:, mask] + else: + constraints_mask = constraints + n, m = constraints_mask.shape + if n >= m: + _, N = torch.linalg.qr(constraints_mask, mode="r") + else: + N = constraints_mask + + if hessians is None: + tilde_X = torch.sqrt(d2loss_dy_hat2)[:, None] * X_mask + _, P_R = torch.linalg.qr(tilde_X, mode="r") + else: + if mask is not None: + hessians_mask = hessians[mask, :][:, mask] + else: + hessian_mask = hessians + P = X_mask.T @ (d2loss_dy_hat2[:, None] * X_mask) + hessians_mask + P_R = torch.linalg.cholesky(P, upper=True) + + S = self.D_nmask @ torch.linalg.solve_triangular( + P_R, + torch.linalg.solve_triangular(P_R.T, kkt_rhs, upper=False), + upper=True, + ) + S_R = torch.linalg.cholesky(S, upper=True) + NPinvRhs = N @ torch.linalg.solve_triangular( + P_R, + torch.linalg.solve_triangular(P_R.T, kkt_rhs, upper=False), + upper=True, + ) + nu = torch.linalg.solve_triangular( + S_R, + torch.linalg.solve_triangular(S_R.T, -NPinvRhs, upper=False), + upper=True, + ) + v = torch.linalg.solve_triangular( + P_R, + torch.linalg.solve_triangular(P_R.T, kkt_rhs + N.T @ nu, upper=False), + upper=True, + ) + out = X_mask @ v + return out if not needs_squeeze else out.squeeze(-1) diff --git a/randalo/sklearn_integration.py b/randalo/sklearn_integration.py new file mode 100644 index 0000000..ece5f55 --- /dev/null +++ b/randalo/sklearn_integration.py @@ -0,0 +1,76 @@ +import numpy as np +import torch + +import sklearn.base +import sklearn.linear_model +import sklearn.utils.validation + +from . import modeling_layer as ml +from . import reductions + + +def map_sklearn( + model: sklearn.base.BaseEstimator = None, + X: torch.Tensor | np.ndarray = None, + y: torch.Tensor | np.ndarray | list = None, +) -> tuple[ml.Loss, reductions.Jacobian, torch.Tensor, torch.Tensor]: + sklearn.utils.validation.check_is_fitted(model) + n = X.shape[0] + + def solution_func(): + return model.coef_ + + match model: + case sklearn.linear_model.LinearRegression(): + loss = ml.MSELoss() + # TODO create trivial regularizer + reg = 0 * ml.SquareRegularizer() + y_hat = model.predict(X) + + case sklearn.linear_model.Ridge(): + loss = ml.MSELoss() + reg = model.alpha / n * ml.SquareRegularizer() + y_hat = model.predict(X) + + case sklearn.linear_model.Lasso() | sklearn.linear_model.LassoLars(): + loss = ml.MSELoss() + reg = 2.0 * model.alpha * ml.L1Regularizer() + y_hat = model.predict(X) + + case sklearn.linear_model.ElasticNet(): + loss = ml.MSELoss() + reg = model.alpha * ( + 2.0 * model.l1_ratio * ml.L1Regularizer() + + (1 - model.l1_ratio) * ml.SquareRegularizer() + ) + y_hat = model.predict(X) + + case sklearn.linear_model.LogisticRegression(): + # TODO: implement sample weight + if len(model.classes_) > 2: + raise ValueError("Only binary classification is supported.") + + loss = ml.LogisticLoss() + y = ((model.classes_[None, :] == y[:, None]) * np.array([[-1, 1]])).sum(1) + match model.penalty: + case None: + # TODO: create trivial regularizer + reg = 0 * ml.SquareRegularizer() + case "l1": + reg = ml.L1Regularizer() + case "l2": + reg = 0.5 * ml.SquareRegularizer() + case "elasticnet": + reg = ( + model.l1_ratio * ml.L1Regularizer() + + 0.5 * (1 - model.l1_ratio) * ml.SquareRegularizer() + ) + reg = reg * 1 / model.C / n + y_hat = model.decision_function(X) + + def solution_func(): + return model.coef_[0, :] + + jac = reductions.Jacobian(y, X, solution_func, loss, reg, inverse_method=None) + + return loss, jac, y, y_hat diff --git a/randalo/truncnorm.py b/randalo/truncnorm.py new file mode 100644 index 0000000..357b3cc --- /dev/null +++ b/randalo/truncnorm.py @@ -0,0 +1,107 @@ +import math + +import torch + +"""Implementation of truncated normal distributions in PyTorch. + +This module is based on the implementation in scipy.stats.truncnorm, which is +licensed under the BSD license. The original source code can be found at +https://github.com/scipy/scipy/blob/v1.11.3/scipy/stats/_continuous_distns.py + +Copyright (c) 2001-2002 Enthought, Inc. 2003-2023, SciPy Developers. +All rights reserved. +""" + +# Constants for the standard normal distribution +_norm_pdf_C = math.sqrt(2 * math.pi) +_norm_pdf_logC = math.log(_norm_pdf_C) + + +def _norm_logpdf(x: torch.Tensor) -> torch.Tensor: + """Log of the probability density function of the standard normal distribution.""" + return -(x**2) / 2.0 - _norm_pdf_logC + + +def _log_diff(log_a: torch.Tensor, log_b: torch.Tensor) -> torch.Tensor: + """Compute log(a - b) when log_a > log_b and both are log-space values""" + return log_a + torch.log1p(-torch.exp(log_b - log_a)) + + +def _log_gauss_mass(alpha: torch.Tensor, beta: torch.Tensor) -> torch.Tensor: + """Log of Gaussian probability mass within an interval""" + + alpha, beta = torch.broadcast_tensors(alpha, beta) + + # Calculations in right tail are inaccurate, so we'll exploit the + # symmetry and work only in the left tail + case_left = beta <= 0 + case_right = alpha > 0 + case_central = ~(case_left | case_right) + + def mass_case_left(a, b): + return _log_diff(torch.special.log_ndtr(b), torch.special.log_ndtr(a)) + + def mass_case_right(a, b): + return mass_case_left(-b, -a) + + def mass_case_central(a, b): + return torch.log1p(-torch.special.ndtr(a) - torch.special.ndtr(-b)) + + # Initialize tensor for output + out = torch.full_like(alpha, float("nan")) + out[case_left] = mass_case_left(alpha[case_left], beta[case_left]).real + out[case_right] = mass_case_right(alpha[case_right], beta[case_right]).real + out[case_central] = mass_case_central(alpha[case_central], beta[case_central]) + + return out + + +def _truncnorm_log_pdf(x, alpha, beta): + """Log of the probability density function of the truncated normal distribution.""" + return _norm_logpdf(x) - _log_gauss_mass(alpha, beta) + + +def _truncnorm_pdf(x, alpha, beta): + """Probability density function of the truncated normal distribution.""" + return torch.exp(_truncnorm_log_pdf(x, alpha, beta)) + + +def truncnorm_mean( + mu: torch.Tensor, sigma: torch.Tensor, a: torch.Tensor, b: torch.Tensor +) -> torch.Tensor: + """Calculates the mean of a truncated normal distribution given its parameters. + + Parameters + ---------- + mu : torch.Tensor + The means of the normal distributions + sigma : torch.Tensor + The standard deviations of the normal distributions + a : torch.Tensor + The lower bounds of the truncation interval + b : torch.Tensor + The upper bounds of the truncation interval + + Returns + ------- + torch.Tensor + The means of the truncated normal distributions + """ + + mu, sigma, a, b = torch.broadcast_tensors(mu, sigma, a, b) + + alpha = (a - mu) / sigma + beta = (b - mu) / sigma + + nan = torch.isnan(alpha) | torch.isnan(beta) + notnan = ~nan + alpha = alpha[notnan] + beta = beta[notnan] + alpha_beta = torch.stack([alpha, beta], dim=0) + pA, pB = _truncnorm_pdf(alpha_beta, alpha[None, :], beta[None, :]) + + out = torch.full_like(mu, float("nan")) + out[nan] = torch.clamp(mu[nan], min=a[nan], max=b[nan]) + out[notnan] = mu[notnan] - sigma[notnan] * (pB - pA) + + return out diff --git a/randalo/utils.py b/randalo/utils.py new file mode 100644 index 0000000..c6b456e --- /dev/null +++ b/randalo/utils.py @@ -0,0 +1,180 @@ +from typing import Callable, NamedTuple + +import numpy as np +import sklearn.linear_model +import torch +from torch import autograd + + +def to_tensor( + array: np.ndarray | torch.Tensor | list[float], + dtype: torch.dtype = torch.float32, + device: torch.device = None, +) -> torch.Tensor: + """Convert a numpy array or torch tensor to a torch tensor. + + Parameters + ---------- + array : np.ndarray | torch.Tensor | list[float] + Input array or tensor. + dtype : torch.dtype, optional + Data type for the output tensor, by default torch.float32. + device : torch.device, optional + Device for the output tensor, by default None. + + Returns + ------- + torch.Tensor + Output tensor. + """ + + if isinstance(array, np.ndarray): + return torch.tensor(array, dtype=dtype, device=device) + elif isinstance(array, torch.Tensor) or isinstance(array, list): + return torch.as_tensor(array, dtype=dtype, device=device) + else: + raise ValueError("Input must be a numpy array or torch tensor") + + +class LossDerivatives(NamedTuple): + y: torch.Tensor + y_hat: torch.Tensor + y: torch.Tensor + dloss_dy_hat: torch.Tensor + d2loss_dboth: torch.Tensor + d2loss_dy_hat2: torch.Tensor + + +def compute_derivatives( + loss_fun: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], + y: torch.Tensor, + y_hat: torch.Tensor, +) -> LossDerivatives: + """Compute first and second derivatives of a loss function. + + Parameters + ---------- + loss_fun : Callable[[torch.Tensor, torch.Tensor], torch.Tensor] + Loss function to compute derivatives of. The function should take the + true labels `y` and the predicted values `y_hat` as input and return + the sum or mean reduction of the element-wise losses as a singleton + `torch.Tensor`. + y : torch.Tensor + True labels. + y_hat : torch.Tensor + Predicted values. + + Returns + ------- + tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor] + The true labels, + the predicted values, + the first derivatives of the loss with respect to the predicted values, + the second derivatives of the loss with respect to both the true labels + and predicted values, + and the second derivatives of the loss with respect to the predicted values. + """ + + # detach and clone to avoid memory leaks + y = y.detach().clone().requires_grad_(True) + y_hat = y_hat.detach().clone().requires_grad_(True) + + # compute first and second derivatives of loss function + # we obtain the vector derivatives by summing and then taking the gradient + loss = loss_fun(y, y_hat) + # keep the graph for computing the second derivatives + dloss_dy_hat, *_ = autograd.grad(loss, y_hat, create_graph=True) + dloss_dy_hat_sum = dloss_dy_hat.sum() + + d2loss_dboth, d2loss_dy_hat2 = autograd.grad( + dloss_dy_hat_sum, [y, y_hat], allow_unused=True + ) + if d2loss_dboth is None: + d2loss_dboth = torch.zeros_like(y) + if d2loss_dy_hat2 is None: + d2loss_dy_hat2 = torch.zeros_like(y_hat) + + # free memory used by autograd and return + return LossDerivatives( + y.detach(), + y_hat.detach(), + dloss_dy_hat.detach(), + d2loss_dboth.detach(), + d2loss_dy_hat2.detach(), + ) + + +def unsqueeze_scalar_like(x: float, array: torch.Tensor) -> torch.Tensor: + """Expand a scalar to the number of dimensions of an array. + + Parameters + ---------- + x : float + Scalar value. + array : torch.Tensor + Array to expand the scalar to. + + Returns + ------- + torch.Tensor + Expanded scalar. + """ + + return torch.tensor(x, dtype=array.dtype, device=array.device).reshape( + *(1,) * array.ndim + ) + + +def create_mixing_matrix(m: int, subsets: list[list[int]]) -> torch.Tensor: + """Create a mixing matrix A for a given set of subsets, such that when + M @ A is computed, the columns of M corresponding to the subsets are + averaged. + + Parameters + ---------- + m : int + Number of rows of the mixing matrix. + subsets : list[list[int]] + List of subsets of [m] to average. + + Returns + ------- + torch.Tensor + Mixing matrix. + """ + + # create mixing matrix + A = torch.zeros(m, len(subsets)) + for j, subset in enumerate(subsets): + A[subset, j] = 1 / len(subset) + + return A + + +def robust_y_intercept( + x: torch.Tensor | np.ndarray | list[float], + y: torch.Tensor | np.ndarray | list[float], + epsilon: float = 1.35, +) -> float: + """Find the y-intercept of the robust linear fit. + + Parameters + ---------- + x : torch.Tensor | np.ndarray | list[float] + Input data. + y : torch.Tensor | np.ndarray | list[float] + Output data. + epsilon : float, optional + Huber loss parameter, by default 1.35. + + Returns + ------- + float + The y-intercept. + """ + x = np.asarray(x).reshape(-1, 1) + y = np.asarray(y).reshape(-1) + + huber = sklearn.linear_model.HuberRegressor(fit_intercept=True, epsilon=epsilon) + huber.fit(x, y) + return huber.intercept_ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fcc0318 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +numpy==2.1.1 +scipy +torch +scikit-learn +matplotlib +pandas +cvxpy +cvxpylayers +tqdm +git+ssh://git@github.com/cvxgrp/torch_linops.git +-e . \ No newline at end of file diff --git a/setup.py b/setup.py index fa36a69..d6a5595 100644 --- a/setup.py +++ b/setup.py @@ -3,29 +3,31 @@ from setuptools import setup, find_packages with open("README.md", "r") as fh: - long_description = fh.read() + long_description = fh.read() # get all the git tags from the cmd line that follow our versioning pattern -git_tags = subprocess.Popen( - ["git", "tag", "--list", "v*[0-9]", "--sort=version:refname"], - stdout=subprocess.PIPE, -) -tags = git_tags.stdout.read() -git_tags.stdout.close() -tags = tags.decode("utf-8").split("\n") -tags.sort() +# git_tags = subprocess.Popen( +# ["git", "tag", "--list", "v*[0-9]", "--sort=version:refname"], +# stdout=subprocess.PIPE, +# ) +# tags = git_tags.stdout.read() +# git_tags.stdout.close() +# tags = tags.decode("utf-8").split("\n") +# tags.sort() +# print("-----") +# print(tags) # PEP 440 won't accept the v in front, so here we remove it, strip the new line and decode the byte stream -VERSION_FROM_GIT_TAG = tags[-1][1:] if len(tags) > 0 else "v0.0" +# VERSION_FROM_GIT_TAG = tags[-1][1:] if len(tags) > 0 else "v0.0" setup( - name="alogcv", - version=VERSION_FROM_GIT_TAG, # Required + name="randalo", + # version=VERSION_FROM_GIT_TAG, # Required + version="0.1.0", setup_requires=["setuptools>=18.0"], packages=find_packages(exclude=["notebooks"]), # Required install_requires=[ - "surecr >= 0.1.2", "numpy >= 1.17.5", "scipy", "torch", @@ -34,7 +36,7 @@ description="", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/cvxgrp/SURE-CR", + url="https://github.com/cvxgrp/randalo", classifiers=[ "Programming Language :: Python :: 3", ], diff --git a/alogcv/__init__.py b/test/__init__.py similarity index 100% rename from alogcv/__init__.py rename to test/__init__.py diff --git a/test/test_randalo.py b/test/test_randalo.py new file mode 100644 index 0000000..1861ae8 --- /dev/null +++ b/test/test_randalo.py @@ -0,0 +1,173 @@ +import unittest + +import linops as lo +import numpy as np +import torch + +from randalo import RandALO +from randalo import modeling_layer as ml +from randalo import truncnorm +from randalo import utils + + +class TestRandALO(unittest.TestCase): + def setUp(self): + self.n = 10 + self.rng = torch.Generator().manual_seed(0) + self.diag = torch.rand(self.n, generator=self.rng) + self.jac = lo.DiagonalOperator(self.diag) + self.loss = ml.MSELoss() + self.y = torch.rand(self.n, generator=self.rng) + self.y_hat = torch.rand(self.n, generator=self.rng) + + def risk_fun(self, y, z): + return torch.mean((y - z) ** 2).item() + + def test_diagonal_jac(self): + ra = RandALO( + loss=self.loss, jac=self.jac, y=self.y, y_hat=self.y_hat, rng=self.rng + ) + risk = ra.evaluate(self.risk_fun, n_matvecs=3) + self.assertEqual(ra._n_matvecs, 3) + self.assertEqual(ra._normalized_diag_jac_estims.shape, (self.n, 3)) + self.assertTrue( + torch.allclose(ra._normalized_diag_jac_estims, self.diag[:, None]) + ) + self.assertTrue( + torch.allclose(ra._normalized_diag_jac_stds, torch.zeros(self.n, 1)) + ) + + risk_bks = ra.evaluate_bks(self.risk_fun, n_matvecs=3) + risk_alo = ra.evaluate_alo(self.risk_fun) + self.assertAlmostEqual(risk, risk_bks) + self.assertAlmostEqual(risk, risk_alo) + + self.assertTrue( + torch.allclose( + ra._y_tilde_exact, + self.y_hat + self.diag / (1 - self.diag) * (self.y_hat - self.y), + ) + ) + self.assertAlmostEqual( + risk, + torch.mean((self.y - self.y_hat) ** 2 / (1 - self.diag) ** 2).item(), + ) + + def test_psd_jac(self): + X = torch.rand(self.n, self.n, generator=self.rng) + J = X @ torch.linalg.solve(X.T @ X + torch.eye(self.n), X.T) + diag = torch.diag(J) + jac = lo.MatrixOperator(J) + + ra = RandALO(loss=self.loss, jac=jac, y=self.y, y_hat=self.y_hat, rng=self.rng) + risk = ra.evaluate(self.risk_fun, n_matvecs=200_000) + self.assertFalse(torch.allclose(ra._normalized_diag_jac_estims, diag[:, None])) + + risk_bks = ra.evaluate_bks(self.risk_fun, n_matvecs=200_000) + risk_alo = ra.evaluate_alo(self.risk_fun) + self.assertAlmostEqual(risk, risk_bks, places=3) + self.assertAlmostEqual(risk, risk_alo, places=3) + self.assertAlmostEqual( + risk_alo, + torch.mean((self.y - self.y_hat) ** 2 / (1 - diag) ** 2).item(), + ) + + def test_logistic(self): + loss = ml.LogisticLoss() + X = torch.rand(self.n, self.n, generator=self.rng) + y = torch.randint(0, 2, (self.n,), generator=self.rng).float() * 2 - 1 + y_hat = torch.randn(self.n, generator=self.rng) + + ds = utils.compute_derivatives(loss, y, y_hat) + dloss_dy_hat = ds.dloss_dy_hat + d2loss_dboth = ds.d2loss_dboth + d2loss_dy_hat2 = ds.d2loss_dy_hat2 + + jac = -lo.MatrixOperator( + X + @ torch.linalg.solve( + X.T @ (d2loss_dy_hat2[:, None] * X) + torch.eye(self.n), + X.T * d2loss_dboth[None, :], + ) + ) + jac_tilde = X @ torch.linalg.solve( + X.T @ (d2loss_dy_hat2[:, None] * X) + torch.eye(self.n), + X.T * d2loss_dy_hat2[None, :], + ) + + ra = RandALO(loss=loss, jac=jac, y=y, y_hat=y_hat, rng=self.rng) + + def risk_fun(y, z): + return torch.mean(torch.lt(y * z, 0).float()).item() + + jac_tilde_diag = torch.diag(jac_tilde) + risk = ra.evaluate_alo(risk_fun) + risk_bks = ra.evaluate_bks(risk_fun, n_matvecs=200) + self.assertAlmostEqual(risk, risk_bks) + + y_tilde_exact = y_hat + dloss_dy_hat / d2loss_dy_hat2 * jac_tilde_diag / ( + 1 - jac_tilde_diag + ) + self.assertTrue(torch.allclose(ra._y_tilde_exact, y_tilde_exact)) + self.assertAlmostEqual(risk, risk_fun(y, y_tilde_exact)) + + def test_more_matvecs(self): + ra = RandALO( + loss=self.loss, jac=self.jac, y=self.y, y_hat=self.y_hat, rng=self.rng + ) + ra.evaluate(self.risk_fun, n_matvecs=3) + self.assertEqual(ra._n_matvecs, 3) + self.assertEqual(ra._normalized_diag_jac_estims.shape, (self.n, 3)) + + ra.evaluate_bks(self.risk_fun, n_matvecs=5) + self.assertEqual(ra._n_matvecs, 5) + self.assertEqual(ra._normalized_diag_jac_estims.shape, (self.n, 5)) + + # test that the last matevecs are not used if we request fewer than we have + ra._normalized_diag_jac_estims[:, -1] = torch.nan + risk1 = ra.evaluate(self.risk_fun, n_matvecs=2) + self.assertEqual(ra._n_matvecs, 5) + self.assertEqual(ra._normalized_diag_jac_estims.shape, (self.n, 5)) + self.assertFalse(np.isnan(risk1)) + + risk2 = ra.evaluate_bks(self.risk_fun, n_matvecs=4) + self.assertEqual(ra._n_matvecs, 5) + self.assertEqual(ra._normalized_diag_jac_estims.shape, (self.n, 5)) + self.assertFalse(np.isnan(risk2)) + + # but if we do use them, we should get nan + risk3 = ra.evaluate_bks(self.risk_fun, n_matvecs=6) + self.assertEqual(ra._n_matvecs, 6) + self.assertEqual(ra._normalized_diag_jac_estims.shape, (self.n, 6)) + self.assertTrue(np.isnan(risk3)) + + def test_uniform_map_estimates(self): + ra = RandALO( + loss=self.loss, jac=self.jac, y=self.y, y_hat=self.y_hat, rng=self.rng + ) + ra.evaluate(self.risk_fun, n_matvecs=3) + + mus = ra._normalized_diag_jac_estims.mean(dim=1, keepdim=True) + stds = ra._normalized_diag_jac_stds[:, None] + ms = torch.arange(1, 4, dtype=torch.float32)[None, :] + sigmas = stds / torch.sqrt(ms) + uniform_map_estimates = ra._uniform_map_estimates(mus, ms) + truncnorm_means = truncnorm.truncnorm_mean( + mus, sigmas, torch.tensor([[0.0]]), torch.tensor([[1.0]]) + ) + self.assertTrue(torch.allclose(uniform_map_estimates, truncnorm_means)) + + # add another dimension + mus = mus[..., None] + stds = stds[..., None] + ms = ms[..., None] + sigmas = stds / torch.sqrt(ms) + uniform_map_estimates = ra._uniform_map_estimates(mus, ms) + truncnorm_means = truncnorm.truncnorm_mean( + mus, sigmas, torch.tensor([[[0.0]]]), torch.tensor([[[1.0]]]) + ) + self.assertTrue(torch.allclose(uniform_map_estimates, truncnorm_means)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_reductions.py b/test/test_reductions.py new file mode 100644 index 0000000..7b4a659 --- /dev/null +++ b/test/test_reductions.py @@ -0,0 +1,48 @@ +import unittest + +import linops as lo +import cvxpy as cp +import numpy as np +import torch +from cvxpylayers.torch import CvxpyLayer + +from randalo import modeling_layer as ml, reductions + + +class TestReductions(unittest.TestCase): + def setUp(self): + self.n = 10 + self.rng = np.random.default_rng(0x219A) + self.loss = ml.MSELoss() + self.regularizer = 0.01 * (0.5 * ml.SquareRegularizer() + ml.L1Regularizer()) + self.X = self.rng.standard_normal((self.n, 3 * self.n)) + self.y = self.X[:, 0] + 0.01 * self.rng.standard_normal(self.n) + + def test_transform_to_cvxpy_and_test_jacobian(self): + # TODO fix segemntation fault? + return + + b = cp.Variable(3 * self.n) + y = cp.Parameter(self.n) + prob = reductions.transform_model_to_cvxpy( + self.loss, self.regularizer, self.X, y, b + ) + self.layer = CvxpyLayer(prob, parameters=[y], variables=[b]) + + y_torch = torch.tensor(self.y, requires_grad=True) + J = reductions.Jacobian( + self.y, self.X, lambda: b.value, self.loss, self.regularizer + ) + + y.value = self.y + prob.solve() + y_hat_torch = torch.from_numpy(self.X) @ self.layer(y_torch)[0] + + z = self.rng.standard_normal(self.n) + Jz = J @ z + (y_hat_torch @ torch.from_numpy(z)).backward() + assert torch.allclose(Jz, y_torch.grad.to(torch.float32), atol=1e-4, rtol=1e-2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_sklearn_integration.py b/test/test_sklearn_integration.py new file mode 100644 index 0000000..3c943b1 --- /dev/null +++ b/test/test_sklearn_integration.py @@ -0,0 +1,261 @@ +import unittest + +import numpy as np +import scipy +import scipy.special +import sklearn.linear_model +import sklearn.linear_model._coordinate_descent +import torch + +from randalo import RandALO +from randalo import modeling_layer as ml +from randalo import utils + + +class TestSklearnRandALO(unittest.TestCase): + def setUp(self): + np.random.seed(0) + self.n = 10 + self.p = 8 + self.rng = np.random.default_rng(0) + self.X = self.rng.normal(0, 1, (self.n, self.p)) + self.beta = self.rng.normal(0, 1 / np.sqrt(self.p), (self.p)) + self.y = self.X @ self.beta + self.rng.normal(0, 1, (self.n)) + self.y_bin = ( + self.rng.uniform(0, 1, (self.n,)) + < scipy.special.expit(4 * self.X @ self.beta) + ).astype(int) + + def compute_dy_hat(self, model, dy): + model.fit(self.X, self.y) + y_hat = model.predict(self.X) + model.fit(self.X, self.y + dy) + y_hat2 = model.predict(self.X) + return y_hat2 - y_hat + + def assertRandomJacobianDirectionAlmostEqual( + self, model, jac, epsilon=1e-6, sim_thresh=0.99, norm_rtol=1e-3 + ): + dy = self.rng.normal(0, epsilon, (self.n,)) + dy_hat = utils.to_tensor(self.compute_dy_hat(model, dy)) + dy = utils.to_tensor(dy) + + dy_hat = dy_hat / torch.norm(dy) + djac = jac @ dy / torch.norm(dy) + self.assertTrue( + torch.allclose(torch.norm(dy_hat), torch.norm(djac), rtol=norm_rtol) + ) + + if torch.norm(dy_hat) == 0 or torch.norm(djac) == 0: + return + + sim = dy_hat / torch.norm(dy_hat) @ djac / torch.norm(djac) + self.assertGreaterEqual(sim, sim_thresh) + + def get_randalo_jac(self, model, y=None): + if y is None: + y = self.y + ra = RandALO.from_sklearn(model, self.X, y) + return ra._jac @ torch.eye(self.n) + + def test_linear_regression(self): + lr = sklearn.linear_model.LinearRegression(fit_intercept=False) + lr.fit(self.X, self.y) + + # check against theoretical ground truth + Q, _ = np.linalg.qr(self.X) + jac = utils.to_tensor(Q @ Q.T) + ra_jac = self.get_randalo_jac(lr) + self.assertTrue(torch.allclose(jac, ra_jac, atol=1e-6)) + + # check numerically + self.assertRandomJacobianDirectionAlmostEqual(lr, ra_jac) + + def test_ridge(self): + alphas = np.logspace(-1, 1, 10) + + for alpha in alphas: + ridge = sklearn.linear_model.Ridge(alpha=alpha, fit_intercept=False) + ridge.fit(self.X, self.y) + + # check against theoretical ground truth + jac = utils.to_tensor( + self.X + @ np.linalg.solve( + self.X.T @ self.X + alpha * np.eye(self.p), + self.X.T, + ) + ) + ra_jac = self.get_randalo_jac(ridge) + self.assertTrue(torch.allclose(jac, ra_jac, atol=1e-6)) + + # check against the wrong Jacobian + bad_jac = utils.to_tensor( + self.X + @ np.linalg.solve( + self.X.T @ self.X + 2 * alpha * np.eye(self.p), + self.X.T, + ) + ) + self.assertFalse(torch.allclose(bad_jac, ra_jac, atol=1e-6)) + + # check numerically + self.assertRandomJacobianDirectionAlmostEqual(ridge, ra_jac) + + def test_lasso(self): + # get a path of parameters and shrink a bit so we aren't at the max + alphas = ( + sklearn.linear_model._coordinate_descent._alpha_grid( + self.X, + self.y, + fit_intercept=False, + l1_ratio=1.0, + n_alphas=10, + ) + * 0.99 + ) + + # store the unique numbers of nonzeros over the alphas + nnzs = set() + + # fit a lasso model for each alpha and check Jacobian + for alpha in alphas: + lasso = sklearn.linear_model.Lasso( + alpha=alpha, fit_intercept=False, tol=1e-8 + ) + lasso.fit(self.X, self.y) + lasso_lars = sklearn.linear_model.LassoLars( + alpha=alpha, fit_intercept=False, max_iter=10000 + ) + lasso_lars.fit(self.X, self.y) + + mask = lasso.coef_ != 0 + X_mask = self.X[:, mask] + nnzs.add(np.sum(mask)) + + # check against theoretical ground truth + Q, _ = np.linalg.qr(X_mask) + jac = utils.to_tensor(Q @ Q.T) + ra_jac = self.get_randalo_jac(lasso) + ra_jac_lars = self.get_randalo_jac(lasso_lars) + self.assertTrue(torch.allclose(jac, ra_jac, atol=1e-6)) + self.assertTrue(torch.allclose(jac, ra_jac_lars, atol=1e-6)) + + # check numerically + self.assertRandomJacobianDirectionAlmostEqual( + lasso, ra_jac, sim_thresh=0.999, norm_rtol=1e-3 + ) + self.assertRandomJacobianDirectionAlmostEqual( + lasso_lars, ra_jac, sim_thresh=0.9999, norm_rtol=1e-2 + ) + + # make sure we've actually checked different sparsity patterns + self.assertTrue(len(nnzs) > 3) + + def test_elastic_net(self): + # get a path of parameters and shrink a bit + l1_ratio = 0.5 + alphas = ( + sklearn.linear_model._coordinate_descent._alpha_grid( + self.X, + self.y, + fit_intercept=False, + l1_ratio=l1_ratio, + n_alphas=10, + ) + * 0.99 + ) + + # store the unique numbers of nonzeros over the alphas + nnzs = set() + + # fit a lasso model for each alpha and check Jacobian + for alpha in alphas: + enet = sklearn.linear_model.ElasticNet( + alpha=alpha, l1_ratio=l1_ratio, fit_intercept=False, tol=1e-8 + ) + enet.fit(self.X, self.y) + + mask = enet.coef_ != 0 + X_mask = self.X[:, mask] + n_mask = np.sum(mask) + nnzs.add(n_mask) + + # check against theoretical ground truth + jac = utils.to_tensor( + X_mask + @ np.linalg.solve( + X_mask.T @ X_mask / self.n + + alpha * (1 - l1_ratio) * np.eye(n_mask), + X_mask.T / self.n, + ) + ) + ra_jac = self.get_randalo_jac(enet) + self.assertTrue(torch.allclose(jac, ra_jac, atol=1e-6)) + + # check numerically + self.assertRandomJacobianDirectionAlmostEqual(enet, ra_jac, norm_rtol=0.02) + + # make sure we've actually checked different sparsity patterns + self.assertTrue(len(nnzs) > 3) + + def test_logistic(self): + param_dicts = [ + {"penalty": None, "eye_scale": 0.0}, + {"penalty": "l1", "C": 1.0, "eye_scale": 0.0}, + {"penalty": "l2", "C": 1.0, "eye_scale": 1.0}, + {"penalty": "elasticnet", "C": 1.0, "l1_ratio": 0.5, "eye_scale": 0.5}, + ] + lr = sklearn.linear_model.LogisticRegression( + tol=1e-5, solver="saga", max_iter=100000, fit_intercept=False + ) + + y = utils.to_tensor(self.y_bin * 2 - 1) + y_labels = np.array(["a", "b"])[self.y_bin] + nnzs = set() + + for param_dict in param_dicts: + eye_scale = param_dict.pop("eye_scale") + lr.set_params(**param_dict) + lr.fit(self.X, y_labels) + + mask = lr.coef_[0, :] != 0 + X_mask = self.X[:, mask] + n_mask = np.sum(mask) + nnzs.add(n_mask) + + y_hat = utils.to_tensor(lr.decision_function(self.X)) + ds = utils.compute_derivatives(ml.LogisticLoss(), y, y_hat) + d2loss_dboth = ds.d2loss_dboth.numpy() + d2loss_dy_hat2 = ds.d2loss_dy_hat2.numpy() + + jac = utils.to_tensor( + -X_mask + @ np.linalg.solve( + X_mask.T @ (d2loss_dy_hat2[:, None] * X_mask) + + eye_scale / self.n * np.eye(n_mask), + X_mask.T * d2loss_dboth[None, :], + ) + ) + ra_jac = self.get_randalo_jac(lr, y=y_labels) + atol = 1e-2 if param_dict["penalty"] is None else 1e-6 + self.assertTrue(torch.allclose(jac, ra_jac, atol=atol)) + + # check that wrong y = wrong Jacobian + ds_bad = utils.compute_derivatives(ml.LogisticLoss(), -y, y_hat) + d2loss_dboth_bad = ds_bad.d2loss_dboth.numpy() + d2loss_dy_hat2_bad = ds_bad.d2loss_dy_hat2.numpy() + + jac_bad = utils.to_tensor( + -X_mask + @ np.linalg.solve( + X_mask.T @ (d2loss_dy_hat2_bad[:, None] * X_mask) + + eye_scale / self.n * np.eye(n_mask), + X_mask.T * d2loss_dboth_bad[None, :], + ) + ) + self.assertFalse(torch.allclose(jac_bad, ra_jac, atol=atol)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_truncnorm.py b/test/test_truncnorm.py new file mode 100644 index 0000000..910cd76 --- /dev/null +++ b/test/test_truncnorm.py @@ -0,0 +1,81 @@ +import unittest + +import numpy as np +import scipy +import torch + +from randalo import truncnorm + + +class TruncnormData(object): + def __init__(self, mu, sigma, a=None, b=None, alpha=None, beta=None): + self.mu = mu + self.sigma = sigma + + match (a, b, alpha, beta): + case (None, None, None, None): + self.a = -np.inf + self.b = np.inf + self.alpha = -np.inf + self.beta = np.inf + + case (a, b, None, None): + self.a = a + self.b = b + self.alpha = (a - mu) / sigma + self.beta = (b - mu) / sigma + + case (None, None, alpha, beta): + self.alpha = alpha + self.beta = beta + self.a = mu + alpha * sigma + self.b = mu + beta * sigma + + case (a, b, alpha, beta): + raise ValueError("Cannot specify both a, b and alpha, beta") + + def get_torch_tensors(self, dtype=torch.float64): + return ( + torch.tensor(x, dtype=dtype) for x in [self.mu, self.sigma, self.a, self.b] + ) + + +class TestTruncnorm(unittest.TestCase): + def test_truncnorm_mean(self): + rng = np.random.default_rng(0) + + # test basic truncnorm_mean + td = TruncnormData(mu=1.0, sigma=1.0, a=-1.0, b=1.0) + mean_scipy = scipy.stats.truncnorm.mean( + td.alpha, td.beta, loc=td.mu, scale=td.sigma + ) + mean_torch = truncnorm.truncnorm_mean(*td.get_torch_tensors()).item() + self.assertAlmostEqual(mean_scipy, mean_torch, places=10) + + # test truncnorm_mean with no truncation + td = TruncnormData(mu=1.0, sigma=1.0) + mean_scipy = scipy.stats.truncnorm.mean( + td.alpha, td.beta, loc=td.mu, scale=td.sigma + ) + mean_torch = truncnorm.truncnorm_mean(*td.get_torch_tensors()).item() + self.assertAlmostEqual(td.mu, mean_torch, places=10) + self.assertAlmostEqual(mean_scipy, mean_torch, places=10) + + # test truncnorm_mean with broadcasting + n = 5 + m = 6 + k = 7 + mu = rng.uniform(-1, 1, size=(n, m, k)) + sigma = rng.uniform(0.1, 1, size=(n, m, 1)) + a = rng.uniform(-2, -1, size=(n, 1, k)) + b = rng.uniform(1, 2, size=(1, m, 1)) + td = TruncnormData(mu=mu, sigma=sigma, a=a, b=b) + mean_scipy = scipy.stats.truncnorm.mean( + td.alpha, td.beta, loc=td.mu, scale=td.sigma + ) + mean_torch = truncnorm.truncnorm_mean(*td.get_torch_tensors()).numpy() + self.assertTrue(np.allclose(mean_scipy, mean_torch)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 0000000..74ec53c --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,97 @@ +import unittest + +import numpy as np +import torch + +from randalo import utils + + +class TestUtils(unittest.TestCase): + def test_to_tensor(self): + x = [1, 2, 3] + x_np = np.array(x) + self.assertIs(x_np.dtype, np.dtype("int64")) + x_torch = torch.tensor(x) + self.assertIs(x_torch.dtype, torch.int64) + x_torch = x_torch.to(torch.float32) + + x_to_tensor = utils.to_tensor(x) + self.assertTrue(torch.allclose(x_torch, x_to_tensor)) + self.assertIs(x_to_tensor.dtype, torch.float32) + + x_np_to_tensor = utils.to_tensor(x_np) + self.assertTrue(torch.allclose(x_torch, x_np_to_tensor)) + self.assertIs(x_np_to_tensor.dtype, torch.float32) + + x_torch_to_tensor = utils.to_tensor(x_torch) + self.assertTrue(torch.allclose(x_torch, x_torch_to_tensor)) + self.assertIs(x_torch_to_tensor.dtype, torch.float32) + + with self.assertRaises(ValueError): + utils.to_tensor(1) + + def test_compute_derivatives(self): + n = 100 + + def loss_fun(y, z): + return torch.sum((y - z) ** 4) + + rng = torch.Generator().manual_seed(0) + y = torch.randn(n, generator=rng) + z = torch.randn(n, generator=rng) + + y_, z_, dloss_dz, d2loss_dboth, d2loss_dz2 = utils.compute_derivatives( + loss_fun, y, z + ) + self.assertTrue(torch.allclose(y, y_)) + self.assertTrue(torch.allclose(z, z_)) + self.assertTrue(torch.allclose(4 * (z - y) ** 3, dloss_dz)) + self.assertTrue(torch.allclose(-12 * (y - z) ** 2, d2loss_dboth)) + self.assertTrue(torch.allclose(12 * (y - z) ** 2, d2loss_dz2)) + + def test_unsqueeze_scalar_like(self): + x = 1.5 + arr1 = torch.ones(10, dtype=torch.float32) + arr2 = torch.ones(10, 10, dtype=torch.float64) + arr3 = torch.ones(10, 10, 10, dtype=torch.int32) + + x1 = utils.unsqueeze_scalar_like(x, arr1) + self.assertEqual(x1.shape, (1,)) + self.assertIs(x1.dtype, torch.float32) + self.assertEqual(x1.item(), x) + + x2 = utils.unsqueeze_scalar_like(x, arr2) + self.assertEqual(x2.shape, (1, 1)) + self.assertIs(x2.dtype, torch.float64) + self.assertEqual(x2.item(), x) + + x3 = utils.unsqueeze_scalar_like(x, arr3) + self.assertEqual(x3.shape, (1, 1, 1)) + self.assertIs(x3.dtype, torch.int32) + self.assertNotEqual(x3.item(), x) + self.assertEqual(x3.item(), np.floor(x)) + + def test_create_mixing_matrix(self): + m = 5 + subsets = [[0, 1, 2], [2, 3], [4]] + M0 = torch.tensor( + [[1 / 3, 0, 0], [1 / 3, 0, 0], [1 / 3, 1 / 2, 0], [0, 1 / 2, 0], [0, 0, 1]] + ) + M = utils.create_mixing_matrix(m, subsets) + self.assertTrue(torch.allclose(M0, M)) + + def test_robust_y_intercept(self): + n = 100 + rng = torch.Generator().manual_seed(0) + x = torch.randn(n // 2, generator=rng) + y0 = np.pi + z = torch.randn(n // 2, generator=rng) + y = torch.concatenate([y0 + x + z, y0 + x - z]) + x = torch.concatenate([x, x]) + + y0_hat = utils.robust_y_intercept(x, y) + self.assertAlmostEqual(y0, y0_hat, places=4) + + +if __name__ == "__main__": + unittest.main()