diff --git a/src/concrete/ml/torch/lora.py b/src/concrete/ml/torch/lora.py index d80cdab0a..b4888927e 100644 --- a/src/concrete/ml/torch/lora.py +++ b/src/concrete/ml/torch/lora.py @@ -198,6 +198,7 @@ def forward(self, inputs: Tuple[Tensor, ...]) -> Tuple[Tensor, Union[Tensor, Non len(inputs) >= 2 and len(inputs) <= 3 ), "Expected at least two inputs in the tuple: inputs (x) and targets (y)" + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/4682 # Unpack depending on how many inputs we have if len(inputs) == 2: input_ids, labels = inputs diff --git a/use_case_examples/lora_finetuning/GPT2FineTuneHybrid.ipynb b/use_case_examples/lora_finetuning/GPT2FineTuneHybrid.ipynb index 51a99b21b..c5c44d65b 100644 --- a/use_case_examples/lora_finetuning/GPT2FineTuneHybrid.ipynb +++ b/use_case_examples/lora_finetuning/GPT2FineTuneHybrid.ipynb @@ -14,13 +14,18 @@ "cell_type": "code", "execution_count": 1, "id": "eca73e44", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T13:59:14.310990Z", + "iopub.status.busy": "2024-12-20T13:59:14.310782Z", + "iopub.status.idle": "2024-12-20T13:59:18.219429Z", + "shell.execute_reply": "2024-12-20T13:59:18.218929Z" + } + }, "outputs": [], "source": [ "# Import necessary libraries\n", "import math\n", - "import shutil\n", - "from pathlib import Path\n", "\n", "import matplotlib.pyplot as plt\n", "import torch\n", @@ -43,8 +48,114 @@ "cell_type": "code", "execution_count": 2, "id": "8b965a1a", - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T13:59:18.221836Z", + "iopub.status.busy": "2024-12-20T13:59:18.221449Z", + "iopub.status.idle": "2024-12-20T13:59:22.897185Z", + "shell.execute_reply": "2024-12-20T13:59:22.896409Z" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8db1a659865848c2b46921271fe692b5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "tokenizer_config.json: 0%| | 0.00/26.0 [00:00" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Avoid parallelism error from HuggingFace during training\n", + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 20/40, Loss: 4.9995, grad norm: None, lr: 0.0010285714285714284\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 50%|█████ | 20/40 [10:33<10:55, 32.76s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) allows secure, encrypted data to be accessed without exposing the data. This allows for secure computations on encrypted\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 21/40, Loss: 4.6647, grad norm: None, lr: 0.0009771428571428572\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 52%|█████▎ | 21/40 [11:05<10:22, 32.75s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? GHE is a type of Fully Homomorphic Encryption (FHE) where the ciphertext is encrypted only on the top level of the encryption process\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 22/40, Loss: 4.6801, grad norm: None, lr: 0.0009257142857142857\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 55%|█████▌ | 22/40 [11:35<09:33, 31.85s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) allows data to be processed by any computer on a single node in a encrypted environment. This makes it possible\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 23/40, Loss: 4.1295, grad norm: None, lr: 0.0008742857142857144\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 57%|█████▊ | 23/40 [12:08<09:07, 32.19s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast fast fully-homomorphic encryption that can be safely applied to encrypted data without needing to\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 24/40, Loss: 3.6438, grad norm: None, lr: 0.0008228571428571429\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 60%|██████ | 24/40 [12:41<08:39, 32.49s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? GHE is a type of Fully Homomorphic Encryption (FHE) that allows secure computations on encrypted data without needing to decrypt it. F\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 25/40, Loss: 3.7342, grad norm: None, lr: 0.0007714285714285715\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 62%|██████▎ | 25/40 [13:10<07:48, 31.26s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) involves computing a mathematical operation that updates the encrypted state on the hardware. The operation can be performed on encrypted\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 26/40, Loss: 3.3181, grad norm: None, lr: 0.0007199999999999999\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 65%|██████▌ | 26/40 [13:39<07:08, 30.60s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a growing industry, with growing demand for high-quality FH programs that can be implemented in various\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 27/40, Loss: 2.8375, grad norm: None, lr: 0.0006685714285714286\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 68%|██████▊ | 27/40 [14:10<06:41, 30.88s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a technique that allows secure, mutable data to be processed by the encryption process. This enables the\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 28/40, Loss: 3.0023, grad norm: None, lr: 0.0006171428571428571\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 70%|███████ | 28/40 [14:44<06:20, 31.71s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a technique that allows secure, immutable data to store information or perform computations on encrypted data without needing\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 29/40, Loss: 2.6809, grad norm: None, lr: 0.0005657142857142857\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 72%|███████▎ | 29/40 [15:17<05:54, 32.21s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast growing industry that is growing rapidly. Fhessian (Fast Growing Industry) has been growing\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 30/40, Loss: 2.4333, grad norm: None, lr: 0.0005142857142857142\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 75%|███████▌ | 30/40 [15:45<05:09, 30.91s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a technique that allows secure, mutable data to be processed by the program's hardware hardware drivers without\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 31/40, Loss: 2.4024, grad norm: None, lr: 0.00046285714285714284\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 78%|███████▊ | 31/40 [16:14<04:33, 30.37s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? FHE is a type of encryption algorithm that can be applied to encrypted data without needing a decryptant. It supports both G and PHE\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 32/40, Loss: 2.0085, grad norm: None, lr: 0.00041142857142857143\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 80%|████████ | 32/40 [16:44<04:01, 30.19s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? FHE is a type of encryption algorithm that can be applied to reduce the number of computations required for a ciphertext. It can also\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 33/40, Loss: 1.9126, grad norm: None, lr: 0.00035999999999999997\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 82%|████████▎ | 33/40 [17:14<03:30, 30.12s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast FH model that supports infinite computation and can handle sensitive data without needing to decrypt it.\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 34/40, Loss: 2.0537, grad norm: None, lr: 0.00030857142857142856\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 85%|████████▌ | 34/40 [17:50<03:10, 31.82s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast growing industry that is growing fast and slow. FH is an incredibly fast growth rate,\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 35/40, Loss: 2.1758, grad norm: None, lr: 0.0002571428571428571\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 88%|████████▊ | 35/40 [18:18<02:34, 30.92s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a technique that allows secure, mutable data to be processed by the application state-free. It\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 36/40, Loss: 1.7766, grad norm: None, lr: 0.00020571428571428572\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 90%|█████████ | 36/40 [18:48<02:02, 30.65s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a technique that allows secure, mutable data to be processed by the application state-free. It\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 37/40, Loss: 1.8864, grad norm: None, lr: 0.00015428571428571428\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 92%|█████████▎| 37/40 [19:22<01:34, 31.57s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a technique that allows secure, mutable data to be processed by the application state machine (PAS\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 38/40, Loss: 1.7823, grad norm: None, lr: 0.00010285714285714286\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 95%|█████████▌| 38/40 [19:51<01:01, 30.70s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast FH model that supports infinite computation and can handle sensitive data like secure computations. F\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 39/40, Loss: 1.8798, grad norm: None, lr: 5.142857142857143e-05\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 98%|█████████▊| 39/40 [20:27<00:32, 32.18s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast FH model that supports infinite computation and can handle sensitive information, making it ideal for use\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 40/40, Loss: 1.6045, grad norm: None, lr: 0.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 100%|██████████| 40/40 [21:01<00:00, 32.88s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is FHE? Fully Homomorphic Encryption (FHE) is a fast FH model that supports infinite computation and can handle sensitive information, making it ideal for use\n", + "\n", + "--------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Training Progress: 100%|██████████| 40/40 [21:01<00:00, 31.54s/it]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0kAAAIjCAYAAADWYVDIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABzM0lEQVR4nO3deVxVdf7H8fe97AhcBETAFdFUJNcCKTPLvcYlranMUqtpMlut38zUVErLWM00NVNm+zZmaYuVLZZtlqWSGrngiriDKMguiHB+fxgEsl3wcs8FXs/Hg4dy7rnnfvh6pvHt9/v9HIthGIYAAAAAAJIkq9kFAAAAAIArISQBAAAAQCWEJAAAAACohJAEAAAAAJUQkgAAAACgEkISAAAAAFRCSAIAAACASghJAAAAAFAJIQkAAAAAKiEkAQBqtGfPHlksFr3++usOve7cuXNlsVgcek1XN2zYMA0bNszsMprEmd4nFotFc+fOdWhNAHCmCEkAWq3XX39dFotF69atM7uUOpWHitq+0tPTzS6xmsLCQs2dO1ffffed2aVUUXnc3N3dFRQUpEGDBumOO+5QcnKy2eU5VH33TflXSw1vAHAm3M0uAABgnwULFsjPz6/a8cDAQOcXU4/CwkIlJCRIUrW/hN9///3629/+ZkJVp4wcOVLXXXedDMNQTk6Ofv31V73xxht67rnn9Pjjj2v27NkO/8wvv/zS4desz6RJk9S9e/eK7/Pz8zVz5kxddtllmjRpUsXx9u3bn9HndOnSRcePH5eHh0ej3n/8+HG5u/PXEQCuhf8qAUAzcfnllyskJMTsMs6Yu7u7qX8pPuusszR16tQqxx577DGNGzdOd999t3r16qVLLrnEIZ9VWFgoX19feXp6OuR6DdG3b1/17du34vujR49q5syZ6tu3b7Wfv7KioiJ5enrKarVvsYnFYpG3t3ej6zyT9wJAU2G5HQDU45dfftHYsWMVEBAgPz8/DR8+XGvWrKlyTklJiRISEtSjRw95e3srODhYQ4YM0YoVKyrOSU9P14wZM9SxY0d5eXkpPDxcEyZM0J49e864xsOHD8vd3b1i9qay7du3y2Kx6Nlnn604tnv3bl1xxRUKCgqSr6+vBg8erE8//bTez6ltb8306dPVtWtXSaf2qLRr106SlJCQULGsq3zfSU17kk6ePKmHH35YUVFR8vLyUteuXXXfffepuLi4ynldu3bVH/7wB61atUqxsbHy9vZWt27d9Oabb9Zbe12Cg4P1zjvvyN3dXY8++mjF8fIlmaf/GX333XeyWCxVlhMOGzZMMTExWr9+vYYOHSpfX1/dd999Fa9VHrfy9y9ZskSPPvqoOnbsKG9vbw0fPly7du2qVt/8+fPVrVs3+fj4KDY2Vj/88IND9jmV1/HOO+/o/vvvV4cOHeTr66vc3FxlZWXpnnvu0dlnny0/Pz8FBARo7Nix+vXXX6tco6Y9SdOnT5efn58OHjyoiRMnys/PT+3atdM999yj0tLSKu8/fU9S+f2xa9cuTZ8+XYGBgbLZbJoxY4YKCwurvPf48eO6/fbbFRISIn9/f40fP14HDx5knxOAM8ZMEgDUYcuWLbrgggsUEBCgv/zlL/Lw8NALL7ygYcOGaeXKlYqLi5N06i928+bN04033qjY2Fjl5uZq3bp12rBhg0aOHClJmjx5srZs2aLbbrtNXbt2VUZGhlasWKF9+/ZVBIy6ZGVlVTvm7u6uwMBAtW/fXhdeeKGWLFmiOXPmVDln8eLFcnNz0xVXXCHpVKA677zzVFhYqNtvv13BwcF64403NH78eL333nu67LLLzmjM2rVrpwULFlRb2lV5VuN0N954o9544w1dfvnluvvuu7V27VrNmzdPW7du1dKlS6ucu2vXLl1++eW64YYbNG3aNL366quaPn26Bg0apD59+jS67s6dO+vCCy/Ut99+q9zcXAUEBDT4GpmZmRo7dqyuuuoqTZ06td6lbI899pisVqvuuece5eTk6IknntA111yjtWvXVpyzYMEC3Xrrrbrgggt01113ac+ePZo4caLatm2rjh07NrjGmjz88MPy9PTUPffco+LiYnl6eio5OVkffvihrrjiCkVGRurw4cN64YUXdOGFFyo5OVkRERF1XrO0tFSjR49WXFyc/vWvf+mrr77Sk08+qaioKM2cObPemv74xz8qMjJS8+bN04YNG/Tyyy8rNDRUjz/+eMU506dP15IlS3Tttddq8ODBWrlypS699NIzHg8AkAEArdRrr71mSDJ+/vnnWs+ZOHGi4enpaaSkpFQcO3TokOHv728MHTq04li/fv2MSy+9tNbrHDt2zJBk/POf/2xwnXPmzDEk1fjVs2fPivNeeOEFQ5KxadOmKu+Pjo42Lr744orv77zzTkOS8cMPP1Qcy8vLMyIjI42uXbsapaWlhmEYRmpqqiHJeO211yrOu/DCC40LL7ywWo3Tpk0zunTpUvH9kSNHDEnGnDlzav15yiUlJRmSjBtvvLHKeffcc48hyfjmm28qjnXp0sWQZHz//fcVxzIyMgwvLy/j7rvvrvZZp5NkzJo1q9bX77jjDkOS8euvvxqG8fs9kpqaWuW8b7/91pBkfPvttxXHLrzwQkOS8fzzz1e77unjVv7+3r17G8XFxRXH//Of/1T5MywuLjaCg4ONc8891ygpKak47/XXXzck1fhnUZua/kzK6+jWrZtRWFhY5fyioqKKe6Fcamqq4eXlZTz00ENVjp1+n0ybNs2QVOU8wzCMAQMGGIMGDapy7PSayu+P66+/vsp5l112mREcHFzx/fr16w1Jxp133lnlvOnTp9d67wGAvVhuBwC1KC0t1ZdffqmJEyeqW7duFcfDw8M1ZcoUrVq1Srm5uZJONU/YsmWLdu7cWeO1fHx85Onpqe+++07Hjh1rVD3vv/++VqxYUeXrtddeq3h90qRJcnd31+LFiyuObd68WcnJybryyisrjn322WeKjY3VkCFDKo75+fnppptu0p49e5ze5e2zzz6TpGoNE+6++25JqrYMMDo6WhdccEHF9+3atVPPnj21e/fuM66lvDFGXl5eo97v5eWlGTNm2H3+jBkzquxXKv+5yn+WdevWKTMzU3/605+q7OO65ppr1LZt20bVWJNp06bJx8enyjEvL6+KfUmlpaXKzMyUn5+fevbsqQ0bNth13ZtvvrnK9xdccIHdf041vTczM7Pif3PLly+XJN1yyy1Vzrvtttvsuj4A1IWQBAC1OHLkiAoLC9WzZ89qr/Xu3VtlZWXav3+/JOmhhx5Sdna2zjrrLJ199tn6v//7P23cuLHifC8vLz3++OP6/PPP1b59ew0dOlRPPPFEg9p3Dx06VCNGjKjyFR8fX/F6SEiIhg8friVLllQcW7x4sdzd3at0M9u7d2+tP1P56860d+9eWa3WKp3YJCksLEyBgYHV6uncuXO1a7Rt27bR4bOy/Px8SZK/v3+j3t+hQ4cGNWk4/WcpDz7lP0v5z3762Li7u9u1RNNekZGR1Y6VlZXpqaeeUo8ePeTl5aWQkBC1a9dOGzduVE5OTr3X9Pb2rtibVq4hf072jI3Vaq1W++ljBQCNQUgCAAcYOnSoUlJS9OqrryomJkYvv/yyBg4cqJdffrninDvvvFM7duzQvHnz5O3trQceeEC9e/fWL7/84rA6rrrqKu3YsUNJSUmSpCVLlmj48OEO64pX20NgT9+M78hrn87Nza3G44ZhnHENmzdvlpubW8VfvBv6854+G1OfpvxZGqKmuv/xj39o9uzZGjp0qBYuXKgvvvhCK1asUJ8+fVRWVlbvNWv72ezlKmMDoHUiJAFALdq1aydfX19t37692mvbtm2T1WpVp06dKo4FBQVpxowZevvtt7V//3717du3WoetqKgo3X333fryyy+1efNmnThxQk8++aTDap44caI8PT21ePFiJSUlaceOHbrqqquqnNOlS5daf6by12vTtm1bZWdnVzt++myPvYGn/PPKysqqLVU8fPiwsrOz66zHkfbt26eVK1cqPj6+YiapfPbi9J/ZWbNt5T/76R3vTp486ZCuiHV57733dNFFF+mVV17RVVddpVGjRmnEiBE1/vmbofy+SU1NrXK8pu6AANBQhCQAqIWbm5tGjRqljz76qMpfSA8fPqxFixZpyJAhFR3QMjMzq7zXz89P3bt3r2hhXVhYqKKioirnREVFyd/fv1qb6zMRGBio0aNHa8mSJXrnnXfk6empiRMnVjnnkksuUWJiolavXl1xrKCgQC+++KK6du2q6OjoWq8fFRWlbdu26ciRIxXHfv31V/34449VzvP19ZVUPVzUpPyZRE8//XSV4//+978lySndyrKysnT11VertLRUf//73yuOR0VFSZK+//77imOlpaV68cUXm7wmSTrnnHMUHBysl156SSdPnqw4/tZbbzlkeWFd3Nzcqs3avPvuuzp48GCTfq69Ro8eLUl67rnnqhx/5plnzCgHQAtDC3AArd6rr75asQm8sjvuuEOPPPKIVqxYoSFDhuiWW26Ru7u7XnjhBRUXF+uJJ56oODc6OlrDhg3ToEGDFBQUpHXr1um9997TrbfeKknasWOHhg8frj/+8Y+Kjo6Wu7u7li5dqsOHD1eb6anNe++9V9FYoLKRI0dWaTV95ZVXaurUqXruuec0evRoBQYGVjn/b3/7m95++22NHTtWt99+u4KCgvTGG28oNTVV77//fp0PEb3++uv173//W6NHj9YNN9ygjIwMPf/88+rTp0/Fhnrp1PKt6OhoLV68WGeddZaCgoIUExOjmJiYatfs16+fpk2bphdffFHZ2dm68MILlZiYqDfeeEMTJ07URRddZNf42GvHjh1auHChDMNQbm6ufv31V7377rvKz8/Xv//9b40ZM6bi3D59+mjw4MG69957lZWVpaCgIL3zzjtVAktT8vT01Ny5c3Xbbbfp4osv1h//+Eft2bNHr7/+uqKioho0Y9dQf/jDH/TQQw9pxowZOu+887Rp0ya99dZbVZqYmGnQoEGaPHmynn76aWVmZla0AN+xY4ekhs1mAsDpCEkAWr0FCxbUeHz69Onq06ePfvjhB917772aN2+eysrKFBcXp4ULF1Y8I0mSbr/9dn388cf68ssvVVxcrC5duuiRRx7R//3f/0mSOnXqpKuvvlpff/21/ve//8nd3V29evXSkiVLNHnyZLvqrO3ZMt9++22VkDR+/Hj5+PgoLy+vSle7cu3bt9dPP/2kv/71r3rmmWdUVFSkvn37atmyZfXO2vTu3VtvvvmmHnzwQc2ePVvR0dH63//+p0WLFlV5sKokvfzyy7rtttt011136cSJE5ozZ06NIan83G7duun111/X0qVLFRYWpnvvvbfaM58cobwzoNVqVUBAgCIjIzVt2jTddNNNNc6ivfXWW/rzn/+sxx57TIGBgbrhhht00UUXVTz/qqndeuutMgxDTz75pO655x7169dPH3/8sW6//XZ5e3s32efed999Kigo0KJFi7R48WINHDhQn376qf72t7812Wc21JtvvqmwsDC9/fbbWrp0qUaMGKHFixerZ8+eTTo2AFo+i8EOSAAAmpWysjK1a9dOkyZN0ksvvWR2OS4lKSlJAwYM0MKFC3XNNdeYXQ6AZoo9SQAAuLCioqJqe4PefPNNZWVladiwYeYU5SKOHz9e7djTTz8tq9WqoUOHmlARgJaC5XYAALiwNWvW6K677tIVV1yh4OBgbdiwQa+88opiYmJ0xRVXmF2eqZ544gmtX79eF110kdzd3fX555/r888/10033VSl8yQANBTL7QAAcGF79uzR7bffrsTExIrmEZdccokee+wxhYaGml2eqVasWKGEhAQlJycrPz9fnTt31rXXXqu///3vcnfn34EBNB4hCQAAAAAqYU8SAAAAAFRCSAIAAACASlr8gt2ysjIdOnRI/v7+PFgOAAAAaMUMw1BeXp4iIiLqfHh6iw9Jhw4dosMNAAAAgAr79+9Xx44da329xYckf39/SacGIiAgoFHXKCkp0ZdffqlRo0bJw8PDkeWhEsbZeRhr52CcnYNxdh7G2jkYZ+dgnJ3HlcY6NzdXnTp1qsgItWnxIal8iV1AQMAZhSRfX18FBASY/gfbkjHOzsNYOwfj7ByMs/Mw1s7BODsH4+w8rjjW9W3DoXEDAAAAAFRCSAIAAACASghJAAAAAFAJIQkAAAAAKiEkAQAAAEAlhCQAAAAAqISQBAAAAACVEJIAAAAAoBJCEgAAAABUQkgCAAAAgEoISQAAAABQCSEJAAAAACohJAEAAABAJe5mF9BalJYZSkzNUkZekUL9vRUbGSQ3q8XssgAAAACchpDkBMs3pylhWbLScooqjoXbvDVnXLTGxISbWBkAAACA07Hcrokt35ymmQs3VAlIkpSeU6SZCzdo+eY0kyoDAAAAUBNCUhMqLTOUsCxZRg2vlR9LWJas0rKazgAAAABgBkJSE0pMzao2g1SZISktp0iJqVnOKwoAAABAnQhJTSgjr/aA1JjzAAAAADQ9QlITCvX3duh5AAAAAJoeIakJxUYGKdzmrdoafVt0qstdbGSQM8sCAAAAUAdCUhNys1o0Z1y0JNUalOaMi+Z5SQAAAIALISQ1sTEx4VowdaDCbFWX1Nl8PLRg6kCekwQAAAC4GB4m6wRjYsI1MjpMialZenXVbq3YmqHhvUIJSAAAAIALIiQ5iZvVovioYJ0oLdOKrRlK3EPbbwAAAMAVsdzOyQZ1aSs3q0UHjh3XwezjZpcDAAAA4DSEJCfz83JXTESAJGnt7kyTqwEAAABwOkKSCeK6BUuS1u5myR0AAADgaghJJoj77blIa1OZSQIAAABcDSHJBOd0DZLFIu3JLNTh3CKzywEAAABQCSHJBDYfD0WH/7YvKZUldwAAAIArISSZJC6yfF8SS+4AAAAAV0JIMklct/J9ScwkAQAAAK6EkGSS2K6nQtKujHwdzS82uRoAAAAA5QhJJmnbxlO9wvwlSYnMJgEAAAAug5BkoopW4OxLAgAAAFwGIclEFQ+VZSYJAAAAcBmEJBPF/jaTtC09T8cKTphcDQAAAADJ5JC0YMEC9e3bVwEBAQoICFB8fLw+//zziteLioo0a9YsBQcHy8/PT5MnT9bhw4dNrNixQvy81D3UT5KUuIfZJAAAAMAVmBqSOnbsqMcee0zr16/XunXrdPHFF2vChAnasmWLJOmuu+7SsmXL9O6772rlypU6dOiQJk2aZGbJDvf7viRCEgAAAOAK3M388HHjxlX5/tFHH9WCBQu0Zs0adezYUa+88ooWLVqkiy++WJL02muvqXfv3lqzZo0GDx5sRskOF9ctWG+t3ae1qTRvAAAAAFyBqSGpstLSUr377rsqKChQfHy81q9fr5KSEo0YMaLinF69eqlz585avXp1rSGpuLhYxcW/P3coNzdXklRSUqKSkpJG1Vb+vsa+vy4DO55qA56clqvM3EIF+Hg4/DOai6YcZ1TFWDsH4+wcjLPzMNbOwTg7B+PsPK401vbWYDEMw2jiWuq0adMmxcfHq6ioSH5+flq0aJEuueQSLVq0SDNmzKgSeCQpNjZWF110kR5//PEarzd37lwlJCRUO75o0SL5+vo2yc9wph7+xU1Hiyz6U69SxbQ19Y8DAAAAaLEKCws1ZcoU5eTkKCAgoNbzTJ9J6tmzp5KSkpSTk6P33ntP06ZN08qVKxt9vXvvvVezZ8+u+D43N1edOnXSqFGj6hyIupSUlGjFihUaOXKkPDwcP9Oz6sQWvbv+oIyQKF0y+iyHX7+5aOpxxu8Ya+dgnJ2DcXYexto5GGfnYJydx5XGunyVWX1MD0menp7q3r27JGnQoEH6+eef9Z///EdXXnmlTpw4oezsbAUGBlacf/jwYYWFhdV6PS8vL3l5eVU77uHhccZ/KI64Rk3io0L07vqD+nlvtuk3jitoqnFGdYy1czDOzsE4Ow9j7RyMs3Mwzs7jCmNt7+e73HOSysrKVFxcrEGDBsnDw0Nff/11xWvbt2/Xvn37FB8fb2KFjlf+UNnNB3OUX3zS5GoAAACA1s3UmaR7771XY8eOVefOnZWXl6dFixbpu+++0xdffCGbzaYbbrhBs2fPVlBQkAICAnTbbbcpPj6+xXS2K9ch0Ecd2/rowLHjWr/3mC48q53ZJQEAAACtlqkhKSMjQ9ddd53S0tJks9nUt29fffHFFxo5cqQk6amnnpLVatXkyZNVXFys0aNH67nnnjOz5CYTFxmsA8cOaO3uTEISAAAAYCJTQ9Irr7xS5+ve3t6aP3++5s+f76SKzBPXLUjvbzigtak8VBYAAAAwk8vtSWqtBkee2pe08UC2jp8oNbkaAAAAoPUiJLmITkE+Crd5q6TU0IZ9x8wuBwAAAGi1CEkuwmKxKC4ySJK0dnemydUAAAAArRchyYWUtwJfw74kAAAAwDSEJBdSPpOUtD9bRSXsSwIAAADMQEhyIZEhbdTO30snTpYpaX+22eUAAAAArRIhyYVU3ZfEkjsAAADADIQkF1O+L2ltKs0bAAAAADMQklzM4N9mkjbsO6YTJ8tMrgYAAABofQhJLqZ7qJ+C2niqqKRMmw5mm10OAAAA0OoQklyMxWJRbNdTs0lr2JcEAAAAOB0hyQXFdfuteQPPSwIAAACcjpDkguIiTzVvWL8nSydL2ZcEAAAAOBMhyQX1CvOXzcdDBSdKtflQrtnlAAAAAK0KIckFWa0Wndu1/HlJtAIHAAAAnImQ5KIGsy8JAAAAMAUhyUWV70v6OTVLpWWGydUAAAAArQchyUVFRwTI38tdecUntTWNfUkAAACAsxCSXJSb1aJzuraVJK1hXxIAAADgNIQkFxbX7dSSO/YlAQAAAM5DSHJhcZGnmjf8vCdLZexLAgAAAJyCkOTCYjrY5OvppuzCEm0/nGd2OQAAAECrQEhyYR5uVg3qcmpfUiJL7gAAAACnICS5uMEV+5Jo3gAAAAA4AyHJxZXvS0pMzZJhsC8JAAAAaGqEJBfXt2OgvNytOpp/QilH8s0uBwAAAGjxCEkuztPdqoGdy5+XxL4kAAAAoKkRkpqBuG6nltzxvCQAAACg6RGSmoG4yN+aN+zOZF8SAAAA0MQISc3AgM6B8nSzKiOvWHsyC80uBwAAAGjRCEnNgLeHm/p3CpR0ajYJAAAAQNMhJDUT7EsCAAAAnIOQ1EywLwkAAABwDkJSMzGwS6DcrRYdyinSgWPHzS4HAAAAaLEISc2Er6e7+na0SZLWsC8JAAAAaDKEpGYkrttvS+7YlwQAAAA0GUJSMxIXeap5QyIhCQAAAGgyhKRm5JyuQbJapH1ZhXrjp1StTslUaRlNHAAAAABHcje7ANhv1c4jcrNaVFZqaM7HyZKkcJu35oyL1piYcJOrAwAAAFoGZpKaieWb0zRz4QaVlFadOUrPKdLMhRu0fHOaSZUBAAAALQshqRkoLTOUsCxZNS2sKz+WsCyZpXcAAACAAxCSmoHE1Cyl5RTV+rohKS2niIYOAAAAgAMQkpqBjLzaA1JjzgMAAABQO0JSMxDq7+3Q8wAAAADUjpDUDMRGBinc5i1LLa9bdKrLXexvz1ECAAAA0HiEpGbAzWrRnHHRklRrUJozLlpu1tpeBQAAAGAvQlIzMSYmXAumDlSYrfqSun9e3pfnJAEAAAAOwsNkm5ExMeEaGR2mxNQsZeQW6d8rdmhvVqFyik6aXRoAAADQYjCT1My4WS2KjwrWhAEd9Keh3SRJi9bulWHwjCQAAADAEQhJzdiE/hHy9XRTypECnpEEAAAAOAghqRnz9/bQ+H4RkqS3E/eZXA0AAADQMhCSmrkpcZ0lSZ9tTtexghMmVwMAAAA0f4SkZu7sDjb1iQjQiZNlen/DAbPLAQAAAJo9QlIzZ7FYKmaTFiXuo4EDAAAAcIYISS3A+H6nGjjspoEDAAAAcMYISS2Av7eHJvQ/1cBhEQ0cAAAAgDNCSGohro49teTu803pyqKBAwAAANBohKQWom/HQMV0CNCJ0jJ9QAMHAAAAoNEISS3IlNgukmjgAAAAAJwJU0PSvHnzdO6558rf31+hoaGaOHGitm/fXuWcYcOGyWKxVPm6+eabTarYtY3vH6E2vzVwWEsDBwAAAKBRTA1JK1eu1KxZs7RmzRqtWLFCJSUlGjVqlAoKCqqc96c//UlpaWkVX0888YRJFbs2Py93je/fQZK0aC0NHAAAAIDGcDfzw5cvX17l+9dff12hoaFav369hg4dWnHc19dXYWFhzi6vWZoS21lvJ+7T8s2nGjgEtfE0uyQAAACgWTE1JJ0uJydHkhQUFFTl+FtvvaWFCxcqLCxM48aN0wMPPCBfX98ar1FcXKzi4uKK73NzcyVJJSUlKikpaVRd5e9r7PudqVd7X8VEBGjzoVwt+Xmvbji/q9kl2a05jXNzx1g7B+PsHIyz8zDWzsE4Owfj7DyuNNb21mAxXGSHf1lZmcaPH6/s7GytWrWq4viLL76oLl26KCIiQhs3btRf//pXxcbG6oMPPqjxOnPnzlVCQkK144sWLao1WLU0Px22aPFuN4V6G7qvf6ksFrMrAgAAAMxXWFioKVOmKCcnRwEBAbWe5zIhaebMmfr888+1atUqdezYsdbzvvnmGw0fPly7du1SVFRUtddrmknq1KmTjh49WudA1KWkpEQrVqzQyJEj5eHh0ahrOFN+8UkNeWKlCk6UauH15yguMqj+N7mA5jbOzRlj7RyMs3Mwzs7DWDsH4+wcjLPzuNJY5+bmKiQkpN6Q5BLL7W699VZ98skn+v777+sMSJIUFxcnSbWGJC8vL3l5eVU77uHhccZ/KI64hjO09fDQhAEdtGjtPi1ef0hDzmpvdkkN0lzGuSVgrJ2DcXYOxtl5GGvnYJydg3F2HlcYa3s/39TudoZh6NZbb9XSpUv1zTffKDIyst73JCUlSZLCw8ObuLrmbUpsZ0nSF5vTlZlfXM/ZAAAAAMqZGpJmzZqlhQsXatGiRfL391d6errS09N1/PhxSVJKSooefvhhrV+/Xnv27NHHH3+s6667TkOHDlXfvn3NLN3lxXSwqW9Hm06Ulun9DQfMLgcAAABoNkwNSQsWLFBOTo6GDRum8PDwiq/FixdLkjw9PfXVV19p1KhR6tWrl+6++25NnjxZy5YtM7PsZuPq32aT3k7cLxfZegYAAAC4PFP3JNX3F/dOnTpp5cqVTqqm5RnfL0KPfJKs1KMFWr07U+dFhZhdEgAAAODyTJ1JQtNq4+WuCQM6SJIWrd1ncjUAAABA80BIauEqGjhsoYEDAAAAYA9CUgsX08Gmfh1tKik19N56GjgAAAAA9SEktQK/N3DYRwMHAAAAoB6EpFZgXL8I+Xm5a09moVanZJpdDgAAAODSCEmtQBsvd03oHyFJWpRIAwcAAACgLoSkVmJK3O8NHI7SwAEAAACoFSGplegTQQMHAAAAwB6EpFakfDbpncR9KiujgQMAAABQE0JSK1KlgcNuGjgAAAAANSEktSK+nu6aOIAGDgAAAEBdCEmtzJTYLpKkL2ngAAAAANSIkNTKREcEqF+nQJWUGnryy+36KOmgVqdkqpQ9SgAAAIAkyd3sAuB8fTvY9Ov+bL2duF9vJ+6XJIXbvDVnXLTGxISbXB0AAABgLmaSWpnlm9O0cM3easfTc4o0c+EGLd+cZkJVAAAAgOsgJLUipWWGEpYlq6aFdeXHEpYls/QOAAAArRohqRVJTM1SWk5Rra8bktJyipSYmuW8ogAAAAAXQ0hqRTLyag9IjTkPAAAAaIkISa1IqL+3Q88DAAAAWiJCUisSGxmkcJu3LLW8btGpLnexkUHOLAsAAABwKYSkVsTNatGccdGSVGtQmjMuWm7W2l4FAAAAWj5CUiszJiZcC6YOVJit+pK6ywZ04DlJAAAAaPV4mGwrNCYmXCOjw5SYmqWMvCJtTcvT8ytT9P3OIzp+olQ+nm5mlwgAAACYhpmkVsrNalF8VLAm9O+gu0edpY5tfXQ0/4Te+Xmf2aUBAAAApiIkQR5uVs0cFiVJemHlbhWfLDW5IgAAAMA8hCRIki4f1FFhAd5Kzy3S++sPml0OAAAAYBpCEiRJXu5u+vOF3SRJz323SyWlZSZXBAAAAJiDkIQKV53bWSF+njpw7Lg+SjpkdjkAAACAKQhJqODj6aYbL/htNunbXSotM0yuCAAAAHA+QhKqmDq4iwJ9PbT7aIE+25RmdjkAAACA0xGSUIWfl7uuPz9SkvTsN7tUxmwSAAAAWhlCEqqZdl5X+Xu5a/vhPH219bDZ5QAAAABORUhCNTYfD113XhdJ0jPf7JJhMJsEAACA1oOQhBpdf36kfDzctOlgjlbuOGJ2OQAAAIDTEJJQo2A/L10T11kSs0kAAABoXQhJqNVNQ7vJ092q9XuPac3uLLPLAQAAAJyCkIRahQZ466pzO0mSnv12p8nVAAAAAM5BSEKd/nxhlNytFv24K1Pr9x4zuxwAAACgyRGSUKcOgT6aPLCjJOnZb5hNAgAAQMtHSEK9Zg6LktUifbv9iDYfzDG7HAAAAKBJEZJQr64hbTShfwdJ0rPf7DK5GgAAAKBpEZJgl1uGRclikZZvSdeOw3lmlwMAAAA0GUIS7NKjvb/GxoRJYjYJAAAALRshCXabdVF3SdInGw9p95F8k6sBAAAAmgYhCXbrE2HTiN6hKjOkBd+lmF0OAAAA0CQISWiQ8tmkpb8c1P6sQpOrAQAAAByPkIQGGdC5rS7oEaKTZYZe+J7ZJAAAALQ8hCQ02K2/zSYt+fmADucWmVwNAAAA4FiEJDRYXLdgxXYN0onSMr2wcrfZ5QAAAAAORUhCo9w2/NRs0qLEvTqaX2xyNQAAAIDjEJLQKEO6h6hfp0AVlZTpoWXJ+ijpoFanZKq0zDC7NAAAAOCMuJtdAJoni8Wi87oF69f92fr410P6+NdDkqRwm7fmjIvWmJhwkysEAAAAGoeZJDTK8s1pen5l9e526TlFmrlwg5ZvTjOhKgAAAODMEZLQYKVlhhKWJaumhXXlxxKWJbP0DgAAAM0SIQkNlpiapbSc2lt/G5LScoqUmJrlvKIAAAAAByEkocEy8ux7NpK95wEAAACuhJCEBgv193boeQAAAIArISShwWIjgxRu85aljnPcrRZ1CfZ1Wk0AAACAoxCS0GBuVovmjIuWpFqD0skyQ1e/tEYHjhU6rzAAAADAAUwNSfPmzdO5554rf39/hYaGauLEidq+fXuVc4qKijRr1iwFBwfLz89PkydP1uHDh02qGOXGxIRrwdSBCrNVXVIXbvPWIxNj1CnIR3szC/XH51cr9WiBSVUCAAAADWfqw2RXrlypWbNm6dxzz9XJkyd13333adSoUUpOTlabNm0kSXfddZc+/fRTvfvuu7LZbLr11ls1adIk/fjjj2aWDp0KSiOjw5SYmqWMvCKF+nsrNjJIblaLRvRurykvr9HuIwX64wur9daNcTqrvb/ZJQMAAAD1MjUkLV++vMr3r7/+ukJDQ7V+/XoNHTpUOTk5euWVV7Ro0SJdfPHFkqTXXntNvXv31po1azR48GAzykYlblaL4qOCqx0Ps3lr8U3xuvaVtdqWnqcrX1it/90Qp5gONhOqBAAAAOxnakg6XU5OjiQpKChIkrR+/XqVlJRoxIgRFef06tVLnTt31urVq2sMScXFxSouLq74Pjc3V5JUUlKikpKSRtVV/r7Gvr+1CvS26n8zztENb67XxoO5uvqlNXrluoEa0CmwxvMZZ+dhrJ2DcXYOxtl5GGvnYJydg3F2Hlcaa3trsBiGYTRxLXYpKyvT+PHjlZ2drVWrVkmSFi1apBkzZlQJPZIUGxuriy66SI8//ni168ydO1cJCQnVji9atEi+vnRbM0PRSemFbW7anWeRl9XQTb1K1Z0JJQAAADhZYWGhpkyZopycHAUEBNR6nsvMJM2aNUubN2+uCEiNde+992r27NkV3+fm5qpTp04aNWpUnQNRl5KSEq1YsUIjR46Uh4fHGdXXWo0Zc1IzFyXpp5QsvbjDUwum9NcFPUKqnMM4Ow9j7RyMs3Mwzs7DWDsH4+wcjLPzuNJYl68yq49LhKRbb71Vn3zyib7//nt17Nix4nhYWJhOnDih7OxsBQYGVhw/fPiwwsLCaryWl5eXvLy8qh338PA44z8UR1yjtbJ5eOjV6bGa9dYGfb0tQze/laRnpgzQ6D7V/xwZZ+dhrJ2DcXYOxtl5GGvnYJydg3F2HlcYa3s/39QW4IZh6NZbb9XSpUv1zTffKDIyssrrgwYNkoeHh77++uuKY9u3b9e+ffsUHx/v7HJxhrw93LRg6iBdena4TpSW6Za3NuijpINmlwUAAABUYepM0qxZs7Ro0SJ99NFH8vf3V3p6uiTJZrPJx8dHNptNN9xwg2bPnq2goCAFBATotttuU3x8PJ3tmilPd6v+c1V/eblb9cEvB3Xn4iQVl5Rp8qCOWpuapfVHLQpOzVJ891C5WWt7VC0AAADQdEwNSQsWLJAkDRs2rMrx1157TdOnT5ckPfXUU7JarZo8ebKKi4s1evRoPffcc06uFI7k7mbVv67oJx9PN721dp/+8v5GPfJpsnKLTkpy05s71ync5q0546I1Jibc7HIBAADQypgakuxprOft7a358+dr/vz5TqgIzmK1WvTIxBil5xTp620ZvwWk36XnFGnmwg1aMHUgQQkAAABOZeqeJLRuZYa0Ja3mDiPl8TlhWbJKy1yiSz0AAABaCUISTJOYmqX0nKJaXzckpeUUKTE1y3lFAQAAoNUjJME0GXm1B6TGnAcAAAA4AiEJpgn193boeQAAAIAjEJJgmtjIIIXbvFVXo+9wm7diI4OcVhMAAABASIJp3KwWzRkXLUm1BqUHLo3meUkAAABwKkISTDUmJlwLpg5UmK3qkrryWHS0oNj5RQEAAKBVa3BI2r9/vw4cOFDxfWJiou688069+OKLDi0MrceYmHCt+uvFWnj9ObquR6kWXn+O5o4/NcP0zy+262g+QQkAAADO0+CQNGXKFH377beSpPT0dI0cOVKJiYn6+9//roceesjhBaJ1cLNaFBcZpEEhhuIigzR1cFfFdAhQXtFJPfb5NrPLAwAAQCvS4JC0efNmxcbGSpKWLFmimJgY/fTTT3rrrbf0+uuvO7o+tFJuVosenhAjSXpv/QGt28OzkgAAAOAcDQ5JJSUl8vLykiR99dVXGj9+vCSpV69eSktLc2x1aNUGdG6rq87tJEm6/8PNOllaZnJFAAAAaA0aHJL69Omj559/Xj/88INWrFihMWPGSJIOHTqk4OBghxeI1u0vY3rJ5uOhbel5+t+avWaXAwAAgFagwSHp8ccf1wsvvKBhw4bp6quvVr9+/SRJH3/8ccUyPMBRgtp46i9jekqS/v3lDmXkFZlcEQAAAFo694a+YdiwYTp69Khyc3PVtm3biuM33XSTfH19HVocIElXndtZS37er18P5GjeZ9v01JX9zS4JAAAALViDZ5KOHz+u4uLiioC0d+9ePf3009q+fbtCQ0MdXiDgZrXooQkxslikpb8c1NrdmWaXBAAAgBaswSFpwoQJevPNNyVJ2dnZiouL05NPPqmJEydqwYIFDi8QkKR+nQJ1dWxnSdKDH21RCU0cAAAA0EQaHJI2bNigCy64QJL03nvvqX379tq7d6/efPNN/fe//3V4gUC5v4zuqba+Htp+OE9v/LTH7HIAAADQQjU4JBUWFsrf31+S9OWXX2rSpEmyWq0aPHiw9u6l+xiaTqCvp/42tpck6akVO3Q4lyYOAAAAcLwGh6Tu3bvrww8/1P79+/XFF19o1KhRkqSMjAwFBAQ4vECgsisGdVL/ToEqOFGqRz/danY5AAAAaIEaHJIefPBB3XPPPeratatiY2MVHx8v6dSs0oABAxxeIFCZ1WrRIxNjZLVIH/96SD+lHDW7JAAAALQwDQ5Jl19+ufbt26d169bpiy++qDg+fPhwPfXUUw4tDqhJTAebpg7uIulUE4cTJ2niAAAAAMdpcEiSpLCwMA0YMECHDh3SgQMHJEmxsbHq1auXQ4sDanP3yJ4KbuOpXRn5eu3HVLPLAQAAQAvS4JBUVlamhx56SDabTV26dFGXLl0UGBiohx9+WGVl/Is+nMPm61HRxOE/X+9UWs5xkysCAABAS9HgkPT3v/9dzz77rB577DH98ssv+uWXX/SPf/xDzzzzjB544IGmqBGo0eSBHXVOl7YqPFGqRz6hiQMAAAAco8Eh6Y033tDLL7+smTNnqm/fvurbt69uueUWvfTSS3r99deboESgZlarRQ9NONXE4dNNafph5xGzSwIAAEAL0OCQlJWVVePeo169eikrK8shRQH2io4I0HXxXSVJcz7aouKTpeYWBAAAgGavwSGpX79+evbZZ6sdf/bZZ9WvXz+HFAU0xOxRZynEz0u7jxbolVU0cQAAAMCZcW/oG5544gldeuml+uqrryqekbR69Wrt379fn332mcMLBOoT4O2hv1/aS3ct/lX//WqnOrX1VZlhKNTfW7GRQXKzWswuEQAAAM1Ig0PShRdeqB07dmj+/Pnatm2bJGnSpEm65ZZbFBER4fACAXtM7N9B879J0a4j+brt7V8qjofbvDVnXLTGxISbWB0AAACakwaHJEmKiIjQo48+WuXYgQMHdNNNN+nFF190SGFAQ3yxJV27juRXO56eU6SZCzdowdSBBCUAAADYpVEPk61JZmamXnnlFUddDrBbaZmhhGXJNb5m/PZrwrJklZYZNZ4DAAAAVOawkASYJTE1S2k5RbW+bkhKyylSYirdFwEAAFA/QhKavYy82gNSY84DAABA60ZIQrMX6u/t0PMAAADQutnduGHSpEl1vp6dnX2mtQCNEhsZpHCbt9JzilTbrqNw26l24AAAAEB97A5JNput3tevu+66My4IaCg3q0VzxkVr5sINskg1BqU546J5XhIAAADsYndIeu2115qyDuCMjIkJ14KpA5WwLLnGJg4+no3qdg8AAIBWiL85osUYExOukdFhSkzNUkZekUL9vbUiOV2v/rhHj36arPOjLpC7G9vwAAAAUDdCEloUN6tF8VHBFd9Hhwfog18OasfhfC1Zd0BT4jqbWB0AAACaA/5ZHS2azddDdwzvIUn694rtyi8+aXJFAAAAcHWEJLR418R1UWRIGx3NP6EF3+0yuxwAAAC4OEISWjxPd6vuHdtLkvTyD6k6mH3c5IoAAADgyuzak/Txxx/bfcHx48c3uhigqYyMbq+4yCCtTc3SP5dv09NXDTC7JAAAALgou0LSxIkT7bqYxWJRaWnpmdQDNAmLxaIH/hCtcc+u0odJhzTj/Ej16xRodlkAAABwQXYttysrK7Pri4AEVxbTwabLBnSQJD3yabIMo6bHzgIAAKC1Y08SWpX/G91T3h5W/bznmL7Ykm52OQAAAHBBjXpOUkFBgVauXKl9+/bpxIkTVV67/fbbHVIY0BTCbT666YJu+u83uzTv8226uFd7ebrzbwUAAAD4XYND0i+//KJLLrlEhYWFKigoUFBQkI4ePSpfX1+FhoYSkuDy/nxhlN7+eb/2ZhbqzdV7dOMF3cwuCQAAAC6kwf+Eftddd2ncuHE6duyYfHx8tGbNGu3du1eDBg3Sv/71r6aoEXCoNl7uumfUWZKk/369U8cKTtTzDgAAALQmDQ5JSUlJuvvuu2W1WuXm5qbi4mJ16tRJTzzxhO67776mqBFwuMsHdVKvMH/lFp3Uf7/ZaXY5AAAAcCENDkkeHh6yWk+9LTQ0VPv27ZMk2Ww27d+/37HVAU3EzWrR/ZdGS5L+t3qvdh/JN7kiAAAAuIoGh6QBAwbo559/liRdeOGFevDBB/XWW2/pzjvvVExMjMMLBJrKkB4huqhnO50sM/TY59vMLgcAAAAuosEh6R//+IfCw8MlSY8++qjatm2rmTNn6siRI3rhhRccXiDQlO67pLfcrBZ9mXxYa3Znml0OAAAAXECDu9udc845Fb8PDQ3V8uXLHVoQ4Ew92vvr6thOWrhmnx75NFkfzxoiq9VidlkAAAAwUYNnki6++GJlZ2dXO56bm6uLL77YETUBTnXniLPk7+WuzQdz9WHSQbPLAQAAgMkaHJK+++67ag+QlaSioiL98MMPDikKcKYQPy/dclF3SdITy7fr+IlSkysCAACAmexebrdx48aK3ycnJys9Pb3i+9LSUi1fvlwdOnRwbHWAk8w4v6sWrtmrg9nH9fIPu3Xb8B5mlwQAAACT2B2S+vfvL4vFIovFUuOyOh8fHz3zzDMOLQ5wFm8PN/11bC/d/vYvWrAyRVee20mhAd5mlwUAAAAT2L3cLjU1VSkpKTIMQ4mJiUpNTa34OnjwoHJzc3X99dc36MO///57jRs3ThEREbJYLPrwww+rvD59+vSKYFb+NWbMmAZ9BmCvcX3DNaBzoApPlOrfK3aYXQ4AAABMYvdMUpcuXSRJZWVlDvvwgoIC9evXT9dff70mTZpU4zljxozRa6+9VvG9l5eXwz4fqMxisej+S3tr8oLVWrxuv6YO7qK8opPKyCtSqL+3YiOD5EbnOwAAgBavwS3AJSklJUVPP/20tm7dKkmKjo7WHXfcoaioqAZdZ+zYsRo7dmyd53h5eSksLKwxZQINNqhLkC7tG65PN6Zp0nM/6UTp7/8oEG7z1pxx0RoTE25ihQAAAGhqDQ5JX3zxhcaPH6/+/fvr/PPPlyT9+OOP6tOnj5YtW6aRI0c6tMDvvvtOoaGhatu2rS6++GI98sgjCg4OrvX84uJiFRcXV3yfm5srSSopKVFJSUmjaih/X2PfD/u4yjif28WmTzemVQlIkpSeU6SZCzfomav6aXSf9iZV5xiuMtYtHePsHIyz8zDWzsE4Owfj7DyuNNb21mAxDMNoyIUHDBig0aNH67HHHqty/G9/+5u+/PJLbdiwoSGX+70Qi0VLly7VxIkTK46988478vX1VWRkpFJSUnTffffJz89Pq1evlpubW43XmTt3rhISEqodX7RokXx9fRtVG1qPMkNK2OCm7BOSVNPSOkOBntKcgaVi5R0AAEDzUlhYqClTpignJ0cBAQG1ntfgkOTt7a1NmzapR4+qLZJ37Nihvn37qqioqFEF1xSSTrd7925FRUXpq6++0vDhw2s8p6aZpE6dOuno0aN1DkRdSkpKtGLFCo0cOVIeHh6Nugbq5wrjvDY1S1NfXVfveQuvP0dxkUFOqKhpuMJYtwaMs3Mwzs7DWDsH4+wcjLPzuNJY5+bmKiQkpN6Q1ODldu3atVNSUlK1kJSUlKTQ0NCGV9oA3bp1U0hIiHbt2lVrSPLy8qqxuYOHh8cZ/6E44hqon5njnFl40u7zWsK9wD3tHIyzczDOzsNYOwfj7ByMs/O4wljb+/l2h6SHHnpI99xzj/70pz/ppptu0u7du3XeeedJOrUn6fHHH9fs2bMbV62dDhw4oMzMTIWHs3EeTSPU375nI73z8361D/BWXGSQLBbW3QEAALQkdoekhIQE3XzzzXrggQfk7++vJ598Uvfee68kKSIiQnPnztXtt9/eoA/Pz8/Xrl27Kr5PTU1VUlKSgoKCFBQUpISEBE2ePFlhYWFKSUnRX/7yF3Xv3l2jR49u0OcA9oqNDFK4zVvpOUWqax3q6pRMrU7JVFS7NromrosmD+wom2/N/zJRWmYoMTWLVuIAAADNhN0hqXzrksVi0V133aW77rpLeXl5kiR/f/9Gffi6det00UUXVXxfPhM1bdo0LViwQBs3btQbb7yh7OxsRUREaNSoUXr44Yd5VhKajJvVojnjojVz4QZZpCpBqTzW3Du2l1IzC/VR0kGlHCnQQ58k6/Hl2zSuX4Suieus/p0CK2aXlm9OU8KyZKXl/L5Xj1biAAAArq1Be5JOX1bU2HBUbtiwYaqrb8QXX3xxRtcHGmNMTLgWTB1YLdyEnRZu7ruklz5MOqS31uzVtvQ8vbf+gN5bf0DR4QG6ZnBn+Xq6a/bipGozUuWtxBdMHUhQAgAAcEENCklnnXVWvfsvsrKyzqggwBWMiQnXyOiwOpfJ+Xt76NrBXTQ1rrM27MvWW2v36pONaUpOy9Xfl26uNhNVztCpWamEZckaGR3G0jsAAAAX06CQlJCQIJvN1lS1AC7FzWpRfFTtDy4uZ7FYNKhLWw3q0lYP/iFa760/oFd+SFVabu3t8A1JaTlFSkzNsuszAAAA4DwNCklXXXVVk7f5BpqzQF9P3XhBN7Xz89Idi5PqPT8jr3HPFQMAAEDTsdp7Im2OAfuFBtjXStzeluMAAABwHrtDUl0NFgBUVd5KvK5/Wgi3ndrnBAAAANdid0gqKytjqR1gp/JW4pJqDUohfp4qPlnqvKIAAABgF7tDEoCGKW8lHmaruqSura+H3K0WbTqYqytfWMO+JAAAABfToMYNABqmtlbivx7I1o1vrNOmgzm6bP5Pen3GuerR/syeOwYAAADHYCYJaGLlrcQn9O+g+KhguVktGti5rZbecp4iQ9roYPZxTVrwk35KOWp2qQAAABAhCTBNl+A2en/meTqnS1vlFZ3UtFcTtfSXA2aXBQAA0OoRkgATBbXx1MIb43Tp2eEqKTV01+Jf9czXO+kmCQAAYCJCEmAybw83PXP1AP15aDdJ0pMrduhv729SSWmZyZUBAAC0ToQkwAVYrRbde0lvPTyhj6wWafG6/br+9Z+VV1RidmkAAACtDiEJcCHXxnfVS9edIx8PN/2w86iueH610nKOS5JKywytTsnUR0kHtTolU6VlLMkDAABoCrQAB1zM8N7ttfjPg3X96+u0LT1Pl83/STdeEKlXVqUqLef3ZyqF27w1Z1y0xsSEm1gtAABAy8NMEuCC+nYM1NJbzlP3UD+l5xbpkU+3VglIkpSeU6SZCzdo+eY0k6oEAABomQhJgIvqFOSrJTfFy9PNUuPr5YvtEpYls/QOAADAgQhJgAvbfjhPJ0prD0CGpLScIiWmZjmvKAAAgBaOkAS4sIy8ovpPasB5AAAAqB8hCXBhof7eDj0PAAAA9SMkAS4sNjJI4TZv1bwr6XeJqZkqPlnqlJoAAABaOkIS4MLcrBbNGRctSXUGpae+2qmx//lBP6UcdU5hAAAALRghCXBxY2LCtWDqQIXZqi6pC7d5a8E1A/Wfq/orxM9Tu48UaMpLazV7cZKO5hebVC0AAEDzx8NkgWZgTEy4RkaHKTE1Sxl5RQr191ZsZJDcrKfml4b1DNW/vtiuhWv36oNfDuqrrYf1t7G9ddW5nWS11rdYDwAAAJUxkwQ0E25Wi+KjgjWhfwfFRwVXBCRJsvl46OGJMVp6y/mKDg9QbtFJ3bd0kyY//5OSD+VWuU5pmaG1qVlaf9SitalZPGMJAADgNMwkAS1I/06B+vjW8/Xm6r168svt+mVftsY9u0ozzuuqu0aepR92HlHCsmSl5RRJctObO9cp3OatOeOiNSYm3OzyAQAAXAIzSUAL4+5m1fVDIvX13cN06dnhKi0z9PKqVJ3/+De6eeGG3wLS79JzijRz4QYt35xmUsUAAACuhZAEtFBhNm/Nv2agXptxrjq29VZ2YUmN55UvtktYlszSOwAAABGSgBbvop6henTi2XWeY0hKyylSYmqWc4oCAABwYYQkoBXIPl7zLNLpMvKK6j8JAACghSMkAa1AqL93/Sc14DwAAICWjJAEtAKxkUEKt3mrricmhdtOPXsJAACgtSMkAa2Am9WiOeOiJanWoPS3sb2qPHsJAACgtSIkAa3EmJhwLZg6UGG2qkvqLL/louWb01VGdzsAAAAeJgu0JmNiwjUyOkyrd2Xoyx/WatQFcXJ3c9e1r67V55vT9c8vt+uvY3qZXSYAAICpmEkCWhk3q0VxkUEaFGIoLjJIg6OC9fjkvpKkBd+laMm6/SZXCAAAYC5CEgBNGthRt13cXZJ03web9FPKUZMrAgAAMA8hCYAk6a4RZ+kPfcN1sszQzIUbtPtIvtklAQAAmIKQBECSZLVa9K8r+mlA50DlHC/R9a//rGMFJ8wuCwAAwOkISQAqeHu46cVrz1GHQB/tySzUn/+3XsUnS80uCwAAwKkISQCqaOfvpddmnCt/L3cl7snSvR9skmHQGhwAALQehCQA1ZzV3l/PXjNQblaLPthwUM99l2J2SQAAAE5DSAJQowvPaqe54/tIkv75xXZ9svGQyRUBAAA4ByEJQK2uHdxF158fKUm6e8mv+mXfMZMrAgAAaHqEJAB1+vulvTW8V6iKT5bpT2+u0/6sQrNLAgAAaFKEJAB1crNa9N+rByg6PEBH80/ohjd+Vm5RidllAQAANBlCEoB6tfFy1yvTz1Gov5d2HM7XrYt+UXFJqVanZOqjpINanZKp0jI64AEAgJbB3ewCADQP4TYfvTLtXP3xhdX6fscRDXh4hQpPlFZ63VtzxkVrTEy4iVUCAACcOWaSANjt7I42XXdeF0mqEpAkKT2nSDMXbtDyzWlmlAYAAOAwhCQAdistM/RxUs2twMsX2yUsS2bpHQAAaNYISQDslpiapbScolpfNySl5RQpMTXLeUUBAAA4GCEJgN0y8moPSI05DwAAwBURkgDYLdTf26HnAQAAuCJCEgC7xUYGKdzmLUs9521Lz5VhsC8JAAA0T4QkAHZzs1o0Z1y0JFULSpW/T1iWrNve/kX5xSedVhsAAICjEJIANMiYmHAtmDpQYbaqS+rCbN5acM1A3X9pb7lbLfpkY5rGP7tK29PzTKoUAACgcXiYLIAGGxMTrpHRYUpMzVJGXpFC/b0VGxkkN+up+aQBnQM1661ftPtIgSbMX6V/XHa2Jg3saHLVAAAA9mEmCUCjuFktio8K1oT+HRQfFVwRkCRpUJcgfXr7EF3QI0RFJWWaveRX3fvBJhWVlNZxRQAAANdASALQJIL9vPT6jFjdOaKHLBbp7cR9mrzgJ+3LLDS7NAAAgDqZGpK+//57jRs3ThEREbJYLPrwww+rvG4Yhh588EGFh4fLx8dHI0aM0M6dO80pFkCDuVktunPEWXpjRqyC2nhqy6FcXfrMD/pyS3rFOaVlhlanZOqjpINanZKp0jK64gEAAHOZGpIKCgrUr18/zZ8/v8bXn3jiCf33v//V888/r7Vr16pNmzYaPXq0iop4UCXQnAw9q50+vX2IBnYOVF7RSd30v/Wa99lWfbLxkIY8/o2ufmmN7ngnSVe/tEZDHv9GyzenmV0yAABoxUxt3DB27FiNHTu2xtcMw9DTTz+t+++/XxMmTJAkvfnmm2rfvr0+/PBDXXXVVc4sFcAZCrf5aPGf4/XY59v0yqpUvfD97hrPS88p0syFG7Rg6kCNiQl3cpUAAAAu3N0uNTVV6enpGjFiRMUxm82muLg4rV69utaQVFxcrOLi4orvc3NzJUklJSUqKSlpVC3l72vs+2Efxtl5zBzrv43uob4R/rpzyUbVtLDO0KlnLiUs26JhPao2hGhuuKedg3F2HsbaORhn52CcnceVxtreGiyGYbjEBgCLxaKlS5dq4sSJkqSffvpJ559/vg4dOqTw8N//NfmPf/yjLBaLFi9eXON15s6dq4SEhGrHFy1aJF9f3yapHUDD7Myx6Nlkt3rPuzW6VD1sLvGfKAAA0AIUFhZqypQpysnJUUBAQK3nuexMUmPde++9mj17dsX3ubm56tSpk0aNGlXnQNSlpKREK1as0MiRI+Xh4eGoUnEaxtl5zB7rZRvTpORN9Z7XrU9/XdK3+S65M3ucWwvG2XkYa+dgnJ2DcXYeVxrr8lVm9XHZkBQWFiZJOnz4cJWZpMOHD6t///61vs/Ly0teXl7Vjnt4eJzxH4ojroH6Mc7OY9ZYhwe2sfu8lnAvcE87B+PsPIy1czDOzsE4O48rjLW9n++yz0mKjIxUWFiYvv7664pjubm5Wrt2reLj402sDMCZio0MUrjNW/XtNvp8c5pyCs1fvwwAAFoXU0NSfn6+kpKSlJSUJOlUs4akpCTt27dPFotFd955px555BF9/PHH2rRpk6677jpFRERU7FsC0Dy5WS2aMy5akuoMSm+u3qth//pWi9bu4/lJAADAaUwNSevWrdOAAQM0YMAASdLs2bM1YMAAPfjgg5Kkv/zlL7rtttt000036dxzz1V+fr6WL18ub29vM8sG4ABjYsK1YOpAhdmq/u853Oat56cO1MIb4tQj1E/HCkt039JNGv/sKq3bk2VStQAAoDUxdU/SsGHDVFdzPYvFooceekgPPfSQE6sC4CxjYsI1MjpMialZysgrUqi/t2Ijgyrafn92xwX63+q9euqrHdpyKFeXP79aE/pH6N6xvauFKwAAAEdx2cYNAFoHN6tF8VHBNb7m4WbV9UMiNaF/hP715Xa98/N+fZR0SCuSD2vWRd11w5BIeXucaiVeWmbUGrYAAAAagpAEwOUF+3lp3qS+mhLbRXM+3qwN+7L1zy+2a8m6/Xrg0miVlJbpoU+SlZZTVPGecJu35oyL1piY5ttCHAAAmMNlu9sBwOnO7mjT+zPP01NX9lOov5f2ZhbqxjfXaeZbG6oEJElKzynSzIUbtHxzmknVAgCA5oqQBKBZsVgsumxAR31zzzD9+cJutZ5XvtsxYVkynfEAAECDEJIANEt+Xu4adlZonecYktJyipSYSlc8AABgP0ISgGYrI6+o/pMacB4AAIBESALQjIX629cG/KXvd+u77Rl1PnIAAACgHCEJQLMVGxmkcJu36mv0vflQrqa/9rNGP/29lvy8X8UnS51SHwAAaJ4ISQCaLTerRXPGRUtStaBk+e3rkYkxuv78SLXxdNOOw/n6y/sbdf5j3+qZr3fqWMGJatcsLTO0OiVTHyUd1OqUTJo+AADQCvGcJADN2piYcC2YOlAJy6o+JynstOck3TGih95J3KfXftyj9NwiPblih+Z/t0uXD+qoG4Z0U2RIGy3fnFbtOjxvCQCA1oeQBKDZGxMTrpHRYUpMzVJGXpFC/b0VGxkkN+vv80s2Hw/9+cIoXT8kUp9uTNNLP+zWlkO5Wrhmn95au09nd7Bp44Gcatcuf97SgqkDCUoAALQShCQALYKb1aL4qOB6z/Nws2rigA6a0D9Cq3dn6uUfUvXNtowaA5J0qo24RaeetzQyOqxK8KpLaZmhtalZWn/UouDULMV3D7X7vQAAwFyEJACtksVi0XlRITovKkTvrd+ve97dWOu55c9bejtxn8b3j1CAt0ed1666bM9Nb+5cx7I9AACaEUISgFbPw82+Hjb3f7hZ93+4WSF+nuoW4qfIkDaKbNdG3ULaqFu7NuoU5Ktvt2Vo5sINOr3dA8v2AABoPghJAFo9e5+3ZPPxUM7xEh3NP6Gj+VlK3JNV5XWLJKvVUi0gSY1ftgcAAJyPkASg1St/3lJ6TlGNAceiU93yVv31YhWeOKk9Rwu1+2i+dh8pUOrRAu0+mq/UIwUqOFFaZ8vw8mV7ialZdu2fAgAA5iAkAWj1yp+3NHPhBlmkKkGpfL5nzrhouVkt8vf20NkdbTq7o63KNQzD0MI1e/XAR1vq/byMvKJ6zwEAAObhYbIAoN+ftxRmq7r0Lszmbdc+IovFou6h/nZ9loWVdgAAuDRmkgDgN/Y8b6ku9S3bK/eXdzfqcE6xZpzfVe52No0AAADOw/87A0Al5c9bmtC/g+KjghvUYKF82Z70+zK9cuXfdw/1U9HJMj362VZNmP+jNh7IdkjdAADAcQhJAOBAdS3be37qQH1551A9Pvls2Xw8tOVQribO/1EJy7Yov/ikSRUDAIDTsdwOABysfNne6l0Z+vKHtRp1QZziu4dWzEpdeW5nDe/dXg9/kqyPkg7ptR/3aPnmdCWM76NRfcJMrh4AADCTBABNwM1qUVxkkAaFGIqrYV9TiJ+X/nPVAL15faw6B/kqLadIN/1vvf78v3VKz/m9+11pmaHVKZn6KOmgVqdk1tliHAAAOAYzSQBgoqFntdMXdw7Vf7/ZqZe+360vthzWj7sydc+osxTq762HP01WWqXQFG7z1pxx0fV22wMAAI3HTBIAmMzH001/HdNLn9w+RAM6Byq/+KTmLkvWLYs2VAlIkpSeU6SZCzdo+eY0k6oFAKDlIyQBgIvoFRag928+Twnjo6t1xytXvtguYVkyS+8AAGgihCQAcCFWq0VntQ+o8zlLhqS0nCIlpmY5qywAAFoVQhIAuJiMvKL6T2rAeQAAoGEISQDgYkL9ves/qQHnAQCAhiEkAYCLiY0MUrjNu9Z9SZLk4WZRl2Bfp9UEAEBrQkgCABfjZrVozrhoSao1KJWUGpow/0f9vId9SQAAOBohCQBc0JiYcC2YOlBhtqpL6sJt3koY30dntffTkbxiXf3iGr25eo8Mg053AAA4Cg+TBQAXNSYmXCOjw5SYmqWMvCKF+nsrNjJIblaLLh/UUX95f6M+3ZimBz/aoqT92frHZWfL28PN7LIBAGj2CEkA4MLcrBbFRwVXO97Gy13PXj1A/TsGat7nW/XBhoPalpanF64dpE5B7FUCAOBMsNwOAJopi8WiPw3tpoU3xCmojaeS03I17tlV+n7HEbNLAwCgWSMkAUAzd173EH1y2xD162hTdmGJpr2WqPnf7mKfEgAAjURIAoAWICLQR4v/HK8rz+kkw5D++cV2/fl/65VXVCJJKi0ztDolUx8lHdTqlEyVlhGgAACoDXuSAKCF8PZw0+OX91X/zoGa89EWfZl8WBPm/6hrB3fRi9/vVlpOUcW54TZvzRkXrTEx4SZWDACAa2ImCQBamKtjO2vxnwcrLMBbu48UKGFZcpWAJEnpOUWauXCDlm9OM6lKAABcFyEJAFqgAZ3b6sNZ58vTrebH0ZYvtktYlszSOwAATkNIAoAWKvVogU6U1h6ADElpOUVKTM1yXlEAADQDhCQAaKEy8orqP6kB5wEA0FoQkgCghQr197brvEBfjyauBACA5oWQBAAtVGxkkMJt3qp5V9Lv/vbeRn2w4YDKnLg3iZbkAABXRgtwAGih3KwWzRkXrZkLN8ii35s1SKr4PtDXQ2m5xZq95Fe9+mOq7rukt86LCmnSupZvTqvWcY+W5AAAV8JMEgC0YGNiwrVg6kCF2aouvQuzeev5qQO15t7h+r/RPeXn5a7NB3M15aW1uuH1n7XzcF6T1LN8c5pmLtxAS3IAgEtjJgkAWrgxMeEaGR2mxNQsZeQVKdTfW7GRQXKznlqIN+ui7rry3E7679c79dbaffp6W4a+3Z6hq2I7684RParsbSotM2q9Tn1KywwlLEtWTQvrDJ2a3UpYlqyR0WF2XxMAgKZASAKAVsDNalF8VHCtr4f4eemhCTGadl5XPf75Nn2ZfFiL1u7Th78c1M0XRunGCyL1/Y4jZ7RMLjE1q9oMUmWVW5LXVSsAAE2NkAQAqBDVzk8vXneO1u7O1D8+26pfD+To3yt26JVVu5Vz/GS188uXyS2YOrBKUMo5XqJdGflKycjXriP52pWRr6T9x+yqgZbkAACzEZIAANXEdQvW0lvO1yeb0vTYZ1t1qJYZoPKlc395f6NW7Tqq3UcKtCsjXxl5xY3+bHtblwMA0FQISQCAGlmtFo3vFyGbj7umvfpznefmHj+phWv2VTkWFuCt7qF+6h7qp6hQP0WFtNGdi5N0JK+4xn1JkuTjYVVMhwAH/QQAADQOIQkAUKfswhK7zhveK1RjYsIqQlGAd/WH1D40oU+NLcnLHS8p0+ULVmvB1IHq1s7vzAoHAKCRaAEOAKiTvcvfbrygm644p5MGdG5bY0CSam9JHm7z1t0jz1I7fy9tP5yn8c/+SDtwAIBpmEkCANQpNjJI4TZvpecU1Tj7Y9Gp5y7FRgbZdb26WpJfeW4n3broFyXuydLNCzfoz0O76f9G95S7G/+mBwBwHv5fBwBQJzerRXPGRUs6FYgqK/9+zrjoBj3bqLwl+YT+HRQfFVzx3tAAb731pzjdOCRSkvTC97s19ZW1OnIGjSAAAGgoQhIAoF61LZMLs3lXa/99pjzcrLr/D9GaP2Wg2ni6ac3uLF363x+0bk+Wwz4DAIC6sNwOAGCXupbJNYVL+4arZ5i/bl64Xrsy8nXVi2t03yW9NeP8rrJYmuYzAQCQmEkCADRAbcvkmkr3UD99NOt8/aFvuE6WGXrok2Td/k6SCopPPdi2tMzQ2tQsrT9q0drULJWW1dZcHAAA+zGTBABwaW283PXM1QM0sHNb/eOzrVr26yFtS8vVlLjOevH73UrLKZLkpjd3rlO4zVtzxkU7dPkfAKD1cemZpLlz58pisVT56tWrl9llAQCczGKx6PohkXrnpsFqH+ClnRn5SliW/FtA+l16TpFmLtxA+3AAwBlx6ZAkSX369FFaWlrF16pVq8wuCQBgknO6BumjWUPk6VbzMr/yxXYJy5JZegcAaDSXX27n7u6usLAwu88vLi5WcfHvrWJzc3MlSSUlJSopse+p8acrf19j3w/7MM7Ow1g7B+PcNHYdztGJ0toDkCEpLadIq3dlKM7OZzfBPtzTzsE4Owfj7DyuNNb21mAxDMNl/6lt7ty5+uc//ymbzSZvb2/Fx8dr3rx56ty5c53vSUhIqHZ80aJF8vX1bcpyAQBOsP6oRW/udKv3vIvCyzSqY5l87fjnwDJDSsm1KLdECvCQogIMNbYnhSOvBQBwrMLCQk2ZMkU5OTkKCAio9TyXDkmff/658vPz1bNnT6WlpSkhIUEHDx7U5s2b5e/vX+N7appJ6tSpk44ePVrnQNSlpKREK1as0MiRI+Xh4dGoa6B+jLPzMNbOwTg3jbWpWZr66jq7zrVapJiIAA3uFqTB3YI0qHOgfD2rpqYvthzWI59tU3ru7//fERbgpfsv6aXRfdo3qDZHXssVcU87B+PsHIyz87jSWOfm5iokJKTekOTSy+3Gjh1b8fu+ffsqLi5OXbp00ZIlS3TDDTfU+B4vLy95eXlVO+7h4XHGfyiOuAbqxzg7D2PtHIyzY8V3D1W4zVvpOUWq7V/5fD3d1D7AS6lHC7XxYK42HszViz/skYebRQM6tVV8VLDOiwpWRl6Rbn/n12rXOZxbrNve+bVBD8pdvjlNtznoWq6Oe9o5GGfnYJydxxXG2t7Pd+mQdLrAwECdddZZ2rVrl9mlAABM4ma1aM64aM1cuEEWqUooKV/V9u8/9tOYmHCl5xRp9e6j+mlXpn5KydTB7ONK3JOlxD1Z+s/XO2v9DOO3ayUsS9bI6DBZJJUahkrLDJX99mvFl2Go5KShBz7aUmNoO/1aTf1sKQDAmWtWISk/P18pKSm69tprzS4FAGCiMTHhWjB1YLU24GGnPScpzOatywZ01GUDOsowDO3POq6fUo7qx5RMrdyeodyik7V+RnkDiKj7PjvjesuvlZiapfio4DO+HgCgabl0SLrnnns0btw4denSRYcOHdKcOXPk5uamq6++2uzSAAAmGxMTrpHRYVq9K0Nf/rBWoy6IU3z30FpnaiwWizoH+6pzcGddFdtZH/1yUHcsTnJILVbLqYYN9cnIK6r/JACA6Vw6JB04cEBXX321MjMz1a5dOw0ZMkRr1qxRu3btzC4NAOAC3KwWxUUGKXOrobjIoAYtZQsN8LbrvOenDlRsZLDcLBZZrac+02qxyM1q+e2YRatTMnX1S2vqvdYbP+1RUBtPDekeIouFZXcA4KpcOiS98847ZpcAAGihYiOD6mwAYdGp5Xr27COq71rlNuzL1rWvJKpbuzaaFt9Vkwd1lJ+XS/9fMQC0SlazCwAAwAzlDSCk3xs+lCv/fs64aLtmp+q7lkXS3PHRmn5eV/l5uWv3kQLN+XiLBv/ja839eItSjuRXu2ZpmaHVKZn6KOmgVqdkqtSe9XwAAIfgn68AAK2WvQ0gHHmtu0edpQ82HNQbq/do95ECvf7THr3+0x5d0CNE08/rqmE9Q7UiOb3adcIbURMAoHEISQCAVq28AURiapYy8ooU6u+t2Abub2rItfy9PTTtvK66Lr6LVu06qjd+2quvtx3WDzuP6oedRxXi56mj+SeqXTs9p0gzF25oUc9bAgBXRUgCALR6blaLw1pz23sti8WiC3q00wU92ml/VqH+t2av3kncV2NAknjeEgA4E3uSAAAwWacgX913SW89c/WAOs+r/LwlAEDTISQBAOAiso+X2HXel1vSVVRS2sTVVFVaZmhtapbWH7VobWoWjSQAtGgstwMAwEWE+tv37KbXftqjD345qPH9InT5oI7q29FW63OXSsuMM95vtXxzWqVGEm56c+c6GkkAaNEISQAAuAh7nrfk5+UmPy93pecW639r9up/a/aqe6ifLh/UUZcN6KD2lR6SWzXcnNLQcLN8c5pmLtxQrR4aSQBoyVhuBwCAi7DneUv/uqKffvzbcC28IU4T+0fIy92qXRn5euzzbYqf97Wmv5aoTzYe0sdJBzVz4YYqAUn6Pdws35xWbz2lZYYSliXXGNjKjyUsS2bpHYAWh5kkAABciL3PWxrSI0RDeoTooaISfbYxTe+tP6B1e4/pu+1H9N32I7JItYYbi6Q5H29Rz/YByi8+qezjJ5RdWKLs4yXKKTyhnOMlyi4s0e6jBdVC1unXKm8k4ajugADgCghJAAC4mIY8uynA20NXxXbWVbGdlXq0QO+vP6BFiXuVVVB7EwhD0uHcYl305HcOqXfjgWwN7hZU674oAGhuCEkAALigxjy7KTKkje4Z3VNRoX66a3FSved7uFkU4uclm4+HAn09Tv3q43nq974eyso/oZdXpdZ7nXmfb9Pin/drdEyYxvQJa/JGEgDQ1AhJAAC0MGEB9nXJe/P6uDqDWGmZoU83pdXZSMLL3aqyMkO7jxZowXcpWvBdiiJs3hrVJ0xjY8J0TtffQ5AjGkkAgDMQkgAAaGHq65Jn0ak9TrGRQXVep7yRxMyFG6rtcSqf+/nPVf01pEc7fbstQ8u3pOvbbRk6lFOk13/ao9d/2qPgNp4a1ae9gtt4av63KXTJA9As0N0OAIAWpr4ueZI0Z1y0XcvcyhtJhNmqzk6F2bwrgo2fl7vG9YvQ/CkDteGBkXr5unN0+aCOsvl4KLPghN5O3K9nawhIEl3yALgmZpIAAGiB7O2SZ++1RkaHafWuDH35w1qNuiBO8d1DawxZ3h5uGhHdXiOi26uktExrd2fp9Z9S9dXWjFqvb2aXPPZIAagJIQkAgBaqIV3y6uNmtSguMkiZWw3F2XkNDzerhvQIUWZBcZ0hqdzWtFynhiT2SAGoDSEJAIAWrDFd8hwt1N++RhIPfZKs5ZvTNXlQB11ydrj8vT2arKblm9M0c+EG9kgBqBF7kgAAQJMqbyRR19yTl/upv5Ik7snSX9/fpHMf/Up3vPOLvt9xpMa9SqVlhlanZOqjpINanZLZoP1MpWWGEpYls0cKQK2YSQIAAE3K3i55/ToFaukvB/X++gNKOVKgj5IO6aOkQ2of4KWJAzro8oEd1aO9f6OXyZWWGcrIK9LyzelV3ns6M/dIAXANhCQAANDk7G0kccuw7pp5YZR+PZCjDzYc0Me/HtLh3GK9sHK3Xli5W12CfbU3s7Da9cuXyT02+Wx1D/XXgWOFOnDsuA4cK9T+rFO/Hsw+rpJS+2eH3l2/XzYfD/UK85e1nj1YNIAAWhZCEgAAcAp7G0lYLBb17xSo/p0C9fdLe+vbbRl6b/1BfbvtcI0BSfp9duqv72+qswZ3q0VBbTyVkVdcb70fbDioDzYcVKCvh87tGqTB3YIVFxmk3uEBVWqmAQTQ8hCSAACA0zS0kYSXu5vGxIRrTEy4lm9O080LN9T7nuA2HooK9VfHtj7q1Nb31K9BvuoU5Kv2/l6yWCwa8vg3tT5sV5L8vd3Vv1Og1u89puzCEq1IPqwVyYclSQHe7oqNDFJcZLDKDEOPfb6NBhBAC0NIAgAAzULxyTK7zntwXB9N6N+hznPq2yP1z8v7akxMuEpKy7T5YI7WpmZpze5MrdtzTLlFJ/XV1ox6n/1k0akGECOjw+xeeldaZmhtapbWH7UoODWr1udRAWhahCQAANAs2NtK3J7z7N0j5eFm1YDObTWgc1vdfGGUTpaWacuhXK1NzdTyTenasD+71s8obwDx8g+7dfmgjgr286qzpqrL9tz05s51jV62xx4p4MwQkgAAQLNQ3kq8tmVyFp0KObGRQXZdrzEP23V3s6pfp0D16xSo9gHe2vBOUr2fM+/zbZr3+TaFBXirT0SAoiMC1CciQH0ibOrY1kcWi8Whz21ijxRw5ghJAACgWbCnlficcdENmjE5k4ft2juzFRbgrfTcooqvr7f9vkwvwNtdvcP9tflgbq3PbWrIsj0ekgs4BiEJAAA0G/Yuk3MGe2e2Vv31Yh0vKdW2tFxtOZSrLYdytOVQrnYczlNu0UmtTT1W5+eUL9ub8Owqhfh7ydPNKg9366lf3SzycLPKw80qdzeL3k7c55Cw1dKx9wv1ISQBAIBmpTHL5JpCQ2a2/LzcdU7XIJ3T9felgCdOlmlnRp4WrtmrtxP31/t5mw/lnlG9LeEhuY7Ya+XIvV9ouQhJAACg2TmTZXKOdCYzW57uVvWJsGl8vw52haRbL4pSl+A2Kik1VFJappLSMp0oLVPJSUMnSku1NS1P32yrveNeuTsX/6LLBnTUyOhQ9e/Uts6Q4UoNIByx14rliLAXIQkAAOAMnOnMlr3L9u4a2bPOa65OybQrJB3OLdbzK1P0/MoUBbfx1EW9QjWid3td0CNEbbx+/6uhIxtAnGnYckS4KS0zlLAsmeWIsAshCQAA4AydycyWoxpS2BO2QgO8dN/Y3vpme4a+3ZahzIITem/9Ab23/oA83a06LypYI3q3l5vVovs+2OQS3fbsDTfndAlSVuEJHckr/v0r//ff780sqFJDTdcyazmiK83YuXJNzkRIAgAAMJkjGlLYE7YSxvfRmJhwTRjQQSWlZfp5T5a+3pqhFcmHtS+rUN9tP6Lvth+p9TOc3W3PMAx9uz3DrnBzzqNf1VmLvR78aLOuju2skdHt1SnIt85zHb9H6pQz2SPlijU1R4QkAAAAF1C+bG/1rgx9+cNajbogrsFd1xoStjzcrDovKkTnRYXo/kt7a1dGvlZsPawPfzmoHYfza/2M8lBy26INio4IUKCvpwJ9PdS20q9tfT3l6W6tdwZozsdbFG7z0ZG8YqXlFik957jScoqUXv6VW6TCE6V2//yBvh5q5+eldv6/fVX6/dH8Yv3js231XmNnRr4e+iRZD32SrF5h/hoZ3V4jo9vr7A42WSy//1m44h4pV6ypuSIkAQAAuAg3q0VxkUHK3GoorpHLmxqzR8pisahHe3/1aO+vDoE+usOOh+R+tjldn21Or/V1D6tFJWU1RaRTDJ3aHzVh/o/1fpY93rz+XA09K7TW10vLDL324546lyO28/fSjRdE6uutGfp5T5a2pedpW3qenvlml9oHeGlE71OBKa+oRLe/neRSe6TYt+VYhCQAAIAWxhkPyR3XL1y+Hu46VnhC2YUlOlZ4QscKS5RdeEIny4w6A1Jl/l7u6hrSRmE2b4XbvH//NcBHYTZvtfPz0sinVtbb2OL87u3q/Bx7liM+NOHUcsSbhkbpWMEJfbv91FLElTuO6HBusd5au09vrd1X7f3lKs+QdWvnp+zCEmXmFyuz4IQy808os6D898Xan1Vo1zLCIY99o4i2Pgpq46ngNp5qW/6rr6eC/DwV5Ospm4+H5ny8pdaaJOmBD7cowNtDOcdLdOy3P6/swhPKKjj1Z3as8ITSsouUlmvPvq1MxUeF1HpeZc31mVSEJAAAAFSwt9ve01cOqPEvu4ZhqOBEqb7Zeli32zEj9eJ159Qb6BzR2EJq2HLEtm08NWlgR00a2FFFJaVanZKpFVsP67ONaco+XlLrZ5TPkI166vt667FHWm7dwcVeR/KLNeXltQ6oSLrhjXU6t2uQ+nUKVP9ONvXrGKhgP69q5zXnZ1IRkgAAAFDhTLvtWSynHp57ad8Izft8W71hKzYyqIZXq3JEY4vK12ro3i9vDzdd1CtUF/UKVWzXIN25OKnez/F2tyrM5q1gPy8FtfFUiJ+ngtuc+n2wn6cycov16Gdb673OA3/orQibjzILTuhYwYlTvxaeUFbB718ZecUqtWPmLtTfS52CfNW2fO9Ym8r7yDx0KPu4Hvqk/poKT5Rq5Y4jWrnj9yYfnYJ81K9joPp3OvV1MPu47nznzJYkmomQBAAAgCqc1W3P3hmg8prO5HlUp9fW2L1f7QPsW4742ozYOmfISssMvfpjar0hcvp5kfXWtzrlqK5+qf5Zov9cNaDeml76of6anrtmoDYdzFHS/mz9uj9bKUcKtD/ruPZnHdcnG9PqrKG57G0iJAEAAKAaR4QSR84ASWe218pR7F2OWN8MmSNDZGxksFNrGtC5rQZ0bqvr4k8dyzleok0HcvTrgWwl7c9WYmqWcupZkmjWM6nsRUgCAABAjRwRShw5A+QKHBluHBUiza7J5uOhIT1CNKTHqWYOH/1yUHfYsSQxI+/M91o1FUISAAAAmpQrzAA5UlPskTrTEOlKNYXauSTR3k6KZiAkAQAAAA3k6D1SjgiRrlKTo5YkmomQBAAAADSCK86QuUJNjm7aYQar2QUAAAAAaFnKl/+F2aouqQuzebt8+2+JmSQAAAAATaAxz6RyFYQkAAAAAE3iTJ5JZSaW2wEAAABAJYQkAAAAAKiEkAQAAAAAlRCSAAAAAKASQhIAAAAAVEJIAgAAAIBKCEkAAAAAUAkhCQAAAAAqISQBAAAAQCWEJAAAAACohJAEAAAAAJUQkgAAAACgEkISAAAAAFTibnYBTc0wDElSbm5uo69RUlKiwsJC5ebmysPDw1Gl4TSMs/Mw1s7BODsH4+w8jLVzMM7OwTg7jyuNdXkmKM8ItWnxISkvL0+S1KlTJ5MrAQAAAOAK8vLyZLPZan3dYtQXo5q5srIyHTp0SP7+/rJYLI26Rm5urjp16qT9+/crICDAwRWiHOPsPIy1czDOzsE4Ow9j7RyMs3Mwzs7jSmNtGIby8vIUEREhq7X2nUctfibJarWqY8eODrlWQECA6X+wrQHj7DyMtXMwzs7BODsPY+0cjLNzMM7O4ypjXdcMUjkaNwAAAABAJYQkAAAAAKiEkGQHLy8vzZkzR15eXmaX0qIxzs7DWDsH4+wcjLPzMNbOwTg7B+PsPM1xrFt84wYAAAAAaAhmkgAAAACgEkISAAAAAFRCSAIAAACASghJAAAAAFAJIckO8+fPV9euXeXt7a24uDglJiaaXVKLMnfuXFkslipfvXr1MrusZu/777/XuHHjFBERIYvFog8//LDK64Zh6MEHH1R4eLh8fHw0YsQI7dy505xim7n6xnr69OnV7vExY8aYU2wzNm/ePJ177rny9/dXaGioJk6cqO3bt1c5p6ioSLNmzVJwcLD8/Pw0efJkHT582KSKmyd7xnnYsGHV7umbb77ZpIqbpwULFqhv374VD9eMj4/X559/XvE697Lj1DfW3M9N47HHHpPFYtGdd95Zcaw53deEpHosXrxYs2fP1pw5c7Rhwwb169dPo0ePVkZGhtmltSh9+vRRWlpaxdeqVavMLqnZKygoUL9+/TR//vwaX3/iiSf03//+V88//7zWrl2rNm3aaPTo0SoqKnJypc1ffWMtSWPGjKlyj7/99ttOrLBlWLlypWbNmqU1a9ZoxYoVKikp0ahRo1RQUFBxzl133aVly5bp3Xff1cqVK3Xo0CFNmjTJxKqbH3vGWZL+9Kc/Vbmnn3jiCZMqbp46duyoxx57TOvXr9e6det08cUXa8KECdqyZYsk7mVHqm+sJe5nR/v555/1wgsvqG/fvlWON6v72kCdYmNjjVmzZlV8X1paakRERBjz5s0zsaqWZc6cOUa/fv3MLqNFk2QsXbq04vuysjIjLCzM+Oc//1lxLDs72/Dy8jLefvttEypsOU4fa8MwjGnTphkTJkwwpZ6WLCMjw5BkrFy50jCMU/ewh4eH8e6771acs3XrVkOSsXr1arPKbPZOH2fDMIwLL7zQuOOOO8wrqoVq27at8fLLL3MvO0H5WBsG97Oj5eXlGT169DBWrFhRZWyb233NTFIdTpw4ofXr12vEiBEVx6xWq0aMGKHVq1ebWFnLs3PnTkVERKhbt2665pprtG/fPrNLatFSU1OVnp5e5d622WyKi4vj3m4i3333nUJDQ9WzZ0/NnDlTmZmZZpfU7OXk5EiSgoKCJEnr169XSUlJlfu6V69e6ty5M/f1GTh9nMu99dZbCgkJUUxMjO69914VFhaaUV6LUFpaqnfeeUcFBQWKj4/nXm5Cp491Oe5nx5k1a5YuvfTSKvev1Pz+G+1udgGu7OjRoyotLVX79u2rHG/fvr22bdtmUlUtT1xcnF5//XX17NlTaWlpSkhI0AUXXKDNmzfL39/f7PJapPT0dEmq8d4ufw2OM2bMGE2aNEmRkZFKSUnRfffdp7Fjx2r16tVyc3Mzu7xmqaysTHfeeafOP/98xcTESDp1X3t6eiowMLDKudzXjVfTOEvSlClT1KVLF0VERGjjxo3661//qu3bt+uDDz4wsdrmZ9OmTYqPj1dRUZH8/Py0dOlSRUdHKykpiXvZwWoba4n72ZHeeecdbdiwQT///HO115rbf6MJSTDd2LFjK37ft29fxcXFqUuXLlqyZIluuOEGEysDHOOqq66q+P3ZZ5+tvn37KioqSt99952GDx9uYmXN16xZs7R582b2Lzax2sb5pptuqvj92WefrfDwcA0fPlwpKSmKiopydpnNVs+ePZWUlKScnBy99957mjZtmlauXGl2WS1SbWMdHR3N/ewg+/fv1x133KEVK1bI29vb7HLOGMvt6hASEiI3N7dqXTcOHz6ssLAwk6pq+QIDA3XWWWdp165dZpfSYpXfv9zb5ujWrZtCQkK4xxvp1ltv1SeffKJvv/1WHTt2rDgeFhamEydOKDs7u8r53NeNU9s41yQuLk6SuKcbyNPTU927d9egQYM0b9489evXT//5z3+4l5tAbWNdE+7nxlm/fr0yMjI0cOBAubu7y93dXStXrtR///tfubu7q3379s3qviYk1cHT01ODBg3S119/XXGsrKxMX3/9dZV1rHCs/Px8paSkKDw83OxSWqzIyEiFhYVVubdzc3O1du1a7m0nOHDggDIzM7nHG8gwDN16661aunSpvvnmG0VGRlZ5fdCgQfLw8KhyX2/fvl379u3jvm6A+sa5JklJSZLEPX2GysrKVFxczL3sBOVjXRPu58YZPny4Nm3apKSkpIqvc845R9dcc03F75vTfc1yu3rMnj1b06ZN0znnnKPY2Fg9/fTTKigo0IwZM8wurcW45557NG7cOHXp0kWHDh3SnDlz5Obmpquvvtrs0pq1/Pz8Kv8KlpqaqqSkJAUFBalz586688479cgjj6hHjx6KjIzUAw88oIiICE2cONG8opupusY6KChICQkJmjx5ssLCwpSSkqK//OUv6t69u0aPHm1i1c3PrFmztGjRIn300Ufy9/evWMNus9nk4+Mjm82mG264QbNnz1ZQUJACAgJ02223KT4+XoMHDza5+uajvnFOSUnRokWLdMkllyg4OFgbN27UXXfdpaFDh1Zr94va3XvvvRo7dqw6d+6svLw8LVq0SN99952++OIL7mUHq2usuZ8dx9/fv8reRUlq06aNgoODK443q/va7PZ6zcEzzzxjdO7c2fD09DRiY2ONNWvWmF1Si3LllVca4eHhhqenp9GhQwfjyiuvNHbt2mV2Wc3et99+a0iq9jVt2jTDME61AX/ggQeM9u3bG15eXsbw4cON7du3m1t0M1XXWBcWFhqjRo0y2rVrZ3h4eBhdunQx/vSnPxnp6elml93s1DTGkozXXnut4pzjx48bt9xyi9G2bVvD19fXuOyyy4y0tDTzim6G6hvnffv2GUOHDjWCgoIMLy8vo3v37sb//d//GTk5OeYW3sxcf/31RpcuXQxPT0+jXbt2xvDhw40vv/yy4nXuZcepa6y5n5vW6e3Vm9N9bTEMw3BmKAMAAAAAV8aeJAAAAACohJAEAAAAAJUQkgAAAACgEkISAAAAAFRCSAIAAACASghJAAAAAFAJIQkAAAAAKiEkAQAAAEAlhCQAAOpgsVj04Ycfml0GAMCJCEkAAJc1ffp0WSyWal9jxowxuzQAQAvmbnYBAADUZcyYMXrttdeqHPPy8jKpGgBAa8BMEgDApXl5eSksLKzKV9u2bSWdWgq3YMECjR07Vj4+PurWrZvee++9Ku/ftGmTLr74Yvn4+Cg4OFg33XST8vPzq5zz6quvqk+fPvLy8lJ4eLhuvfXWKq8fPXpUl112mXx9fdWjRw99/PHHTftDAwBMRUgCADRrDzzwgCZPnqxff/1V11xzja666ipt3bpVklRQUKDRo0erbdu2+vnnn/Xuu+/qq6++qhKCFixYoFmzZummm27Spk2b9PHHH6t79+5VPiMhIUF//OMftXHjRl1yySW65pprlJWV5dSfEwDgPBbDMAyziwAAoCbTp0/XwoUL5e3tXeX4fffdp/vuu08Wi0U333yzFixYUPHa4MGDNXDgQD333HN66aWX9Ne//lX79+9XmzZtJEmfffaZxo0bp0OHDql9+/bq0KGDZsyYoUceeaTGGiwWi+6//349/PDDkk4FLz8/P33++efsjQKAFoo9SQAAl3bRRRdVCUGSFBQUVPH7+Pj4Kq/Fx8crKSlJkrR161b169evIiBJ0vnnn6+ysjJt375dFotFhw4d0vDhw+usoW/fvhW/b9OmjQICApSRkdHYHwkA4OIISQAAl9amTZtqy98cxcfHx67zPDw8qnxvsVhUVlbWFCUBAFwAe5IAAM3amjVrqn3fu3dvSVLv3r3166+/qqCgoOL1H3/8UVarVT179pS/v7+6du2qr7/+2qk1AwBcGzNJAACXVlxcrPT09CrH3N3dFRISIkl69913dc4552jIkCF66623lJiYqFdeeUWSdM0112jOnDmaNm2a5s6dqyNHjui2227Ttddeq/bt20uS5s6dq5tvvlmhoaEaO3as8vLy9OOPP+q2225z7g8KAHAZhCQAgEtbvny5wsPDqxzr2bOntm3bJulU57l33nlHt9xyi8LDw/X2228rOjpakuTr66svvvhCd9xxh84991z5+vpq8uTJ+ve//11xrWnTpqmoqEhPPfWU7rnnHoWEhOjyyy933g8IAHA5dLcDADRbFotFS5cu1cSJE80uBQDQgrAnCQAAAAAqISQBAAAAQCXsSQIANFusGAcANAVmkgAAAACgEkISAAAAAFRCSAIAAACASghJAAAAAFAJIQkAAAAAKiEkAQAAAEAlhCQAAAAAqISQBAAAAACV/D9J5r3b2HZgqAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Avoid parallelism error from HuggingFace during training\n", "tokenizer.parallelism = False\n", "\n", "# Train the model using FHE simulation\n", @@ -876,9 +1690,16 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "bd666f38", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T14:20:40.555186Z", + "iopub.status.busy": "2024-12-20T14:20:40.555015Z", + "iopub.status.idle": "2024-12-20T14:20:40.558305Z", + "shell.execute_reply": "2024-12-20T14:20:40.557887Z" + } + }, "outputs": [], "source": [ "# Get the fine-tuned model\n", @@ -890,16 +1711,22 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "id": "3e91ad0b", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T14:20:40.560754Z", + "iopub.status.busy": "2024-12-20T14:20:40.560484Z", + "iopub.status.idle": "2024-12-20T14:20:58.755422Z", + "shell.execute_reply": "2024-12-20T14:20:58.754531Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "who invented FHE?\n", - "FHE was first proposed by Craig Gentry in 2009, making it easier to use plaintexts on encrypted data. His breakthrough demonstrated ability to\n" + "Who invented Fully Homomorphic Encryption? Fully Homomorphized Encrypted Data (FHE) was first proposed by Craig Gentry in 2009. His breakthrough demonstrated how to perform\n" ] } ], @@ -909,7 +1736,7 @@ "torch.manual_seed(SEED)\n", "\n", "fine_tuned_model.enable_adapter_layers()\n", - "prompt = \"who invented FHE?\"\n", + "prompt = \"Who invented Fully Homomorphic Encryption?\"\n", "generate_and_print(prompt, fine_tuned_model, tokenizer, SEED)" ] }, @@ -917,14 +1744,20 @@ "cell_type": "code", "execution_count": 17, "id": "21e2a1d1", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T14:20:58.757983Z", + "iopub.status.busy": "2024-12-20T14:20:58.757705Z", + "iopub.status.idle": "2024-12-20T14:21:02.964246Z", + "shell.execute_reply": "2024-12-20T14:21:02.963505Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "What is FHE?\n", - "FHE is a new type of energy storage that is designed to be used in a variety of applications. It is used to store energy in\n" + "Who invented Fully Homomorphic Encryption? The first time I saw a computer fully fully homomorphic encryption, I was in the middle of a conversation with a friend of mine who was\n" ] } ], @@ -935,7 +1768,7 @@ "\n", "peft_model.disable_adapter_layers()\n", "\n", - "prompt = \"What is FHE?\"\n", + "prompt = \"Who invented Fully Homomorphic Encryption?\"\n", "generate_and_print(prompt, peft_model, tokenizer, SEED)" ] }, @@ -943,7 +1776,14 @@ "cell_type": "code", "execution_count": 18, "id": "c97425ee", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T14:21:02.966353Z", + "iopub.status.busy": "2024-12-20T14:21:02.966016Z", + "iopub.status.idle": "2024-12-20T14:21:02.970827Z", + "shell.execute_reply": "2024-12-20T14:21:02.970462Z" + } + }, "outputs": [ { "name": "stdout", @@ -964,65 +1804,15 @@ { "cell_type": "code", "execution_count": 19, - "id": "31367ff5", - "metadata": {}, - "outputs": [], - "source": [ - "# Save the model\n", - "path = Path(\"deployment/gpt2_lora_finetuned\")\n", - "path.mkdir(parents=True, exist_ok=True)\n", - "\n", - "if path.is_dir() and any(path.iterdir()):\n", - " shutil.rmtree(path)\n", - "\n", - "hybrid_model.save_and_clear_private_info(path)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "a1dda636", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total number of weights: 39717120\n", - "Total number of LoRA weights: 294912\n" - ] - } - ], - "source": [ - "# Print weights and size after saving\n", - "total_weights_size_private = print_weights_and_size(hybrid_model.model)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "506ad2f5", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total weights removed: 68.16 %\n" - ] - } - ], - "source": [ - "# Calculate and print the percentage of weights removed\n", - "percentage_removed = (total_weights_size - total_weights_size_private) / total_weights_size * 100\n", - "print(f\"Total weights removed: {percentage_removed:.2f} %\")" - ] - }, - { - "cell_type": "code", - "execution_count": 22, "id": "465cb18b", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-20T14:21:02.972555Z", + "iopub.status.busy": "2024-12-20T14:21:02.972274Z", + "iopub.status.idle": "2024-12-20T14:21:02.974349Z", + "shell.execute_reply": "2024-12-20T14:21:02.974004Z" + } + }, "outputs": [], "source": [ "# Note: Around 95% of the remaining weights are from the embedding layers (wpe and wte)\n", @@ -1033,6 +1823,2906 @@ "metadata": { "execution": { "timeout": 10800 + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "0adf10ae647546d6989008471a65d39b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "0c9f6a843d354d28bacc120cce97624c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_465d9474a385479f9f948458dc50f2ac", + "IPY_MODEL_2a0ec303845545aa9912edb0bde6744c", + "IPY_MODEL_5e8bf461747349f6a1fbd6e0bfd7eb38" + ], + "layout": "IPY_MODEL_95fa3634b4194c029c6fb51dbf2c8889", + "tabbable": null, + "tooltip": null + } + }, + "0ead2d47a82148bcb57cb9fb5b171393": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6c93854116f642809310c15db4bf7991", + "max": 1042301.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_a8ae9e08e03e4576a81c32bfb342de5a", + "tabbable": null, + "tooltip": null, + "value": 1042301.0 + } + }, + "10d275c7398d4b86bb3c890fcdd8f19d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "157295a52782495b8b25916d323ad6aa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1663f12e5b424b86aee98530acc5aa44": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "16813f2927a74de5bece9ebbd0540a84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "1bb396154c024ee184fe1eecae87b61c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_af58d39a9b5a407689565f48cd45df66", + "placeholder": "​", + "style": "IPY_MODEL_1663f12e5b424b86aee98530acc5aa44", + "tabbable": null, + "tooltip": null, + "value": "tokenizer_config.json: 100%" + } + }, + "1cf8a219626d49a59de2fff93be58ad0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "1d3f2713a88c46a78f9afa36fd1d7369": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "23f54ed9cf9c482ea575381dd1c5ec14": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2603500a791f4b11b987a96fdd414b6e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "27d91cea0f614d1086c88daf9a9295f0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "288959f7965241668f701bf4af330392": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2a0ec303845545aa9912edb0bde6744c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_1d3f2713a88c46a78f9afa36fd1d7369", + "max": 456318.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_4341cdf64f5e4d078454a9e023119129", + "tabbable": null, + "tooltip": null, + "value": 456318.0 + } + }, + "2b53704dc5564771b06f2a8275329db9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_5e2034ec91454a85ba48a868c3c49a22", + "placeholder": "​", + "style": "IPY_MODEL_f318e322bafd4beab305bb3794f8a03d", + "tabbable": null, + "tooltip": null, + "value": "generation_config.json: 100%" + } + }, + "32f9f3bdb2c141bc965cef9340574999": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "349b300275fd4ef988f794440181dda8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "3708f06d09734edf9bd173a60a4a5d50": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3a0e7119c58d41c8aafec1e1955d6b81": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2603500a791f4b11b987a96fdd414b6e", + "placeholder": "​", + "style": "IPY_MODEL_32f9f3bdb2c141bc965cef9340574999", + "tabbable": null, + "tooltip": null, + "value": " 1.36M/1.36M [00:00<00:00, 19.3MB/s]" + } + }, + "3cc394ed09114282b5fe092dc304c5d6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f669f19154934ed083f69e723ecfb9c9", + "IPY_MODEL_6767ce3c9961495689bf673044092d6d", + "IPY_MODEL_d2122595fd8845f693525b99425da740" + ], + "layout": "IPY_MODEL_23f54ed9cf9c482ea575381dd1c5ec14", + "tabbable": null, + "tooltip": null + } + }, + "3cd6d02f984945f9904d1b375e075174": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_d3dfcccbf7354490895ff88e4c7d2e6d", + "placeholder": "​", + "style": "IPY_MODEL_c8e17af3650245c9822266443ce4bd8a", + "tabbable": null, + "tooltip": null, + "value": " 1.04M/1.04M [00:00<00:00, 45.2MB/s]" + } + }, + "408db993ae4f4919a1afe63cdb3c07dc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "409983fc10c447159011fb3ce28db6c3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4341cdf64f5e4d078454a9e023119129": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "43d315831dda4ccf8eb2c8f135e3961e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4483e50d9ffe47ec8920266b5c1c1c77": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4513a70e7736481aaee00e70b26536ff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "463e799d79d44e9a979cf13baea8a5e2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "465d9474a385479f9f948458dc50f2ac": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_4483e50d9ffe47ec8920266b5c1c1c77", + "placeholder": "​", + "style": "IPY_MODEL_9f1fcc2ab7eb4f1eb6f45e033de73282", + "tabbable": null, + "tooltip": null, + "value": "merges.txt: 100%" + } + }, + "4772978d1696423cabd743b6a3361d0b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "482478c1c44045088ddae5a0ee1a438f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4a0d2523bb934f82a2598ab3f797cce5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "51acc9ceb7e1492caf9c52963fd02ee5": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "52a548321bee4c6badf06a4482c6a2b2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_51acc9ceb7e1492caf9c52963fd02ee5", + "placeholder": "​", + "style": "IPY_MODEL_ecffe998f077426bb5721161ac9c42df", + "tabbable": null, + "tooltip": null, + "value": "Map: 100%" + } + }, + "5443bd8507524c63b1511d9ec77256b6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "573410e227ba4671ab427fa5203f947b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "59bb015644ea474bb3053e4c63b98699": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e2034ec91454a85ba48a868c3c49a22": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e8bf461747349f6a1fbd6e0bfd7eb38": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_76fe203a1e5c4af1afa3e7c286b6fb6d", + "placeholder": "​", + "style": "IPY_MODEL_1cf8a219626d49a59de2fff93be58ad0", + "tabbable": null, + "tooltip": null, + "value": " 456k/456k [00:00<00:00, 3.39MB/s]" + } + }, + "5eb5c310d6e0411abfff79074661abcc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "604e7817602c4b579689f34ac54c32ec": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6f58f49e70d44af5a94da4a8ade1c1f6", + "IPY_MODEL_b017e9a0f0ae407aa34d40d6a3fb0412", + "IPY_MODEL_7f48cecc2ee3486195083de43886e3d7" + ], + "layout": "IPY_MODEL_59bb015644ea474bb3053e4c63b98699", + "tabbable": null, + "tooltip": null + } + }, + "61786c8c8202424693e8820844ad7e32": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "62cf1b68e27f4ccea5dc999ebfa6ab05": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "64d7b916acf548c7ab97a7cbf781783b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6767ce3c9961495689bf673044092d6d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_64d7b916acf548c7ab97a7cbf781783b", + "max": 548105171.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_4a0d2523bb934f82a2598ab3f797cce5", + "tabbable": null, + "tooltip": null, + "value": 548105171.0 + } + }, + "6c93854116f642809310c15db4bf7991": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6f58f49e70d44af5a94da4a8ade1c1f6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_8144bf7441e44b39b95fd95d8a18a9d6", + "placeholder": "​", + "style": "IPY_MODEL_e550aa1f09cf47f78e39cd25d47fb93c", + "tabbable": null, + "tooltip": null, + "value": "config.json: 100%" + } + }, + "76fe203a1e5c4af1afa3e7c286b6fb6d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "798e01281d8e4e55b735400f46386ac3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7dc3ba510db14d8592c45d1a31aeb192": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7f48cecc2ee3486195083de43886e3d7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_a4936b6670df463faa83e50edefefd10", + "placeholder": "​", + "style": "IPY_MODEL_0adf10ae647546d6989008471a65d39b", + "tabbable": null, + "tooltip": null, + "value": " 665/665 [00:00<00:00, 90.4kB/s]" + } + }, + "8144bf7441e44b39b95fd95d8a18a9d6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "82016ec452f34489be85ed400d1fb535": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "82cfdae5debf46329916c5df2e5b6f85": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "83ff09a399de4db5803b79d31528a043": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8db1a659865848c2b46921271fe692b5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1bb396154c024ee184fe1eecae87b61c", + "IPY_MODEL_d2dee91e2dfd4852bb31ca091bac9c57", + "IPY_MODEL_9caad0d8334b4ab79ca33a2079f72be3" + ], + "layout": "IPY_MODEL_3708f06d09734edf9bd173a60a4a5d50", + "tabbable": null, + "tooltip": null + } + }, + "953814d77d5c42f3869422b244a0b263": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_7dc3ba510db14d8592c45d1a31aeb192", + "placeholder": "​", + "style": "IPY_MODEL_573410e227ba4671ab427fa5203f947b", + "tabbable": null, + "tooltip": null, + "value": "tokenizer.json: 100%" + } + }, + "95fa3634b4194c029c6fb51dbf2c8889": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "96a4e6e67f7849cebc56788c87a4b30c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_953814d77d5c42f3869422b244a0b263", + "IPY_MODEL_f953447532c44168b6afec9ecae0aeba", + "IPY_MODEL_3a0e7119c58d41c8aafec1e1955d6b81" + ], + "layout": "IPY_MODEL_82cfdae5debf46329916c5df2e5b6f85", + "tabbable": null, + "tooltip": null + } + }, + "9968636136124d26b3220e9038ffe9f1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_798e01281d8e4e55b735400f46386ac3", + "placeholder": "​", + "style": "IPY_MODEL_62cf1b68e27f4ccea5dc999ebfa6ab05", + "tabbable": null, + "tooltip": null, + "value": " 34/34 [00:00<00:00, 1969.84 examples/s]" + } + }, + "9caad0d8334b4ab79ca33a2079f72be3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_c24c967674824717b3e045d87c9415ce", + "placeholder": "​", + "style": "IPY_MODEL_463e799d79d44e9a979cf13baea8a5e2", + "tabbable": null, + "tooltip": null, + "value": " 26.0/26.0 [00:00<00:00, 2.62kB/s]" + } + }, + "9cd2984c321045449f717490c8a34c30": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_10d275c7398d4b86bb3c890fcdd8f19d", + "max": 124.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_cc8dae5bcc0f4d8082403ba0e0af8b9e", + "tabbable": null, + "tooltip": null, + "value": 124.0 + } + }, + "9f1fcc2ab7eb4f1eb6f45e033de73282": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "a3b26ac89f1945608a8e87754a1088c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_52a548321bee4c6badf06a4482c6a2b2", + "IPY_MODEL_c71425b24b8f45cdb914382eb28413f5", + "IPY_MODEL_9968636136124d26b3220e9038ffe9f1" + ], + "layout": "IPY_MODEL_c968d4dcfc03490c93f293e346f4bc5e", + "tabbable": null, + "tooltip": null + } + }, + "a4936b6670df463faa83e50edefefd10": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8ae9e08e03e4576a81c32bfb342de5a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "af58d39a9b5a407689565f48cd45df66": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b017e9a0f0ae407aa34d40d6a3fb0412": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_288959f7965241668f701bf4af330392", + "max": 665.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_4772978d1696423cabd743b6a3361d0b", + "tabbable": null, + "tooltip": null, + "value": 665.0 + } + }, + "b27ac9c1532e4e6facf1073e4d9a2e86": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2b53704dc5564771b06f2a8275329db9", + "IPY_MODEL_9cd2984c321045449f717490c8a34c30", + "IPY_MODEL_ff52abd944a942e29aee3620135caf29" + ], + "layout": "IPY_MODEL_157295a52782495b8b25916d323ad6aa", + "tabbable": null, + "tooltip": null + } + }, + "c24c967674824717b3e045d87c9415ce": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c71425b24b8f45cdb914382eb28413f5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_61786c8c8202424693e8820844ad7e32", + "max": 34.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_cd47c84fcbbd4283b4877837294b9fd3", + "tabbable": null, + "tooltip": null, + "value": 34.0 + } + }, + "c8e17af3650245c9822266443ce4bd8a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "c968d4dcfc03490c93f293e346f4bc5e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cc8dae5bcc0f4d8082403ba0e0af8b9e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cd47c84fcbbd4283b4877837294b9fd3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d2122595fd8845f693525b99425da740": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_409983fc10c447159011fb3ce28db6c3", + "placeholder": "​", + "style": "IPY_MODEL_82016ec452f34489be85ed400d1fb535", + "tabbable": null, + "tooltip": null, + "value": " 548M/548M [00:02<00:00, 211MB/s]" + } + }, + "d2dee91e2dfd4852bb31ca091bac9c57": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_4513a70e7736481aaee00e70b26536ff", + "max": 26.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_349b300275fd4ef988f794440181dda8", + "tabbable": null, + "tooltip": null, + "value": 26.0 + } + }, + "d3dfcccbf7354490895ff88e4c7d2e6d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d901496892864e8ba8fe86a817b45813": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_5443bd8507524c63b1511d9ec77256b6", + "placeholder": "​", + "style": "IPY_MODEL_fe3f669aebe645fe99d7f33a86babbfe", + "tabbable": null, + "tooltip": null, + "value": "vocab.json: 100%" + } + }, + "df25df5de1f14096b84f361647178fce": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d901496892864e8ba8fe86a817b45813", + "IPY_MODEL_0ead2d47a82148bcb57cb9fb5b171393", + "IPY_MODEL_3cd6d02f984945f9904d1b375e075174" + ], + "layout": "IPY_MODEL_408db993ae4f4919a1afe63cdb3c07dc", + "tabbable": null, + "tooltip": null + } + }, + "e550aa1f09cf47f78e39cd25d47fb93c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "ecffe998f077426bb5721161ac9c42df": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "f318e322bafd4beab305bb3794f8a03d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "f669f19154934ed083f69e723ecfb9c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_83ff09a399de4db5803b79d31528a043", + "placeholder": "​", + "style": "IPY_MODEL_27d91cea0f614d1086c88daf9a9295f0", + "tabbable": null, + "tooltip": null, + "value": "model.safetensors: 100%" + } + }, + "f953447532c44168b6afec9ecae0aeba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_5eb5c310d6e0411abfff79074661abcc", + "max": 1355256.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_482478c1c44045088ddae5a0ee1a438f", + "tabbable": null, + "tooltip": null, + "value": 1355256.0 + } + }, + "fe3f669aebe645fe99d7f33a86babbfe": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "ff52abd944a942e29aee3620135caf29": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_43d315831dda4ccf8eb2c8f135e3961e", + "placeholder": "​", + "style": "IPY_MODEL_16813f2927a74de5bece9ebbd0540a84", + "tabbable": null, + "tooltip": null, + "value": " 124/124 [00:00<00:00, 13.6kB/s]" + } + } + }, + "version_major": 2, + "version_minor": 0 + } } }, "nbformat": 4, diff --git a/use_case_examples/lora_finetuning/LLamaFineTuning.ipynb b/use_case_examples/lora_finetuning/LLamaFineTuning.ipynb index 7ee8a6810..e86cd43a2 100644 --- a/use_case_examples/lora_finetuning/LLamaFineTuning.ipynb +++ b/use_case_examples/lora_finetuning/LLamaFineTuning.ipynb @@ -140,7 +140,20 @@ "\n", "\n", "tokenized_dataset = dataset.map(tokenize_function, batched=True)\n", - "data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)" + "\n", + "\n", + "class LoraTrainerDataCollator(DataCollatorForLanguageModeling):\n", + " def __call__(self, features):\n", + " batch = super().__call__(features)\n", + " # Reorder keys for LoraTrainer\n", + " return {\n", + " \"input_ids\": batch[\"input_ids\"],\n", + " \"labels\": batch[\"labels\"],\n", + " \"attention_mask\": batch[\"attention_mask\"],\n", + " }\n", + "\n", + "\n", + "data_collator = LoraTrainerDataCollator(tokenizer, mlm=False)" ] }, { @@ -162,7 +175,6 @@ " learning_rate=2e-4,\n", " lr_scheduler_type=\"linear\",\n", " seed=SEED,\n", - " data_seed=SEED,\n", " warmup_steps=10,\n", " weight_decay=0.01,\n", " prediction_loss_only=True,\n", @@ -338,6 +350,15 @@ "metadata": { "execution": { "timeout": 10800 + }, + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.11" } }, "nbformat": 4, diff --git a/use_case_examples/lora_finetuning/README.md b/use_case_examples/lora_finetuning/README.md index a1513298f..a4b0872ef 100644 --- a/use_case_examples/lora_finetuning/README.md +++ b/use_case_examples/lora_finetuning/README.md @@ -19,6 +19,8 @@ Fine-tuning large language models typically requires access to sensitive data, w ### Installation +This project requires Python 3.9 or higher. + Install the required packages: diff --git a/use_case_examples/lora_finetuning/requirements.txt b/use_case_examples/lora_finetuning/requirements.txt index da6495fef..86d35cb02 100644 --- a/use_case_examples/lora_finetuning/requirements.txt +++ b/use_case_examples/lora_finetuning/requirements.txt @@ -4,6 +4,6 @@ peft==0.12.0 Jinja2==3.1.4 matplotlib==3.7.5 datasets==3.1.0 -accelerate==1.2.0 +accelerate==1.0.1 jupyter==1.1.1 tqdm==4.67.1 \ No newline at end of file