From 468b98b9522d6b6c9cd874944f25a41245c79b5f Mon Sep 17 00:00:00 2001 From: Fabian Grob Date: Tue, 13 Feb 2024 09:51:23 -0800 Subject: [PATCH] Fix (notebooks): makes notebooks deterministic and adds explicit assert statements --- ...1_quant_tensor_quant_conv2d_overview.ipynb | 617 ++++++------ notebooks/02_quant_activation_overview.ipynb | 266 +++--- notebooks/03_anatomy_of_a_quantizer.ipynb | 561 ++++++----- notebooks/Brevitas_TVMCon2021.ipynb | 883 ++++++++++++++++-- notebooks/ONNX_export_tutorial.ipynb | 203 +++- notebooks/quantized_recurrent.ipynb | 565 ++++++----- 6 files changed, 2058 insertions(+), 1037 deletions(-) diff --git a/notebooks/01_quant_tensor_quant_conv2d_overview.ipynb b/notebooks/01_quant_tensor_quant_conv2d_overview.ipynb index 2e9ef9179..8f3e6eb75 100644 --- a/notebooks/01_quant_tensor_quant_conv2d_overview.ipynb +++ b/notebooks/01_quant_tensor_quant_conv2d_overview.ipynb @@ -3,7 +3,10 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } }, "source": [ "# An overview of QuantTensor and QuantConv2d\n", @@ -18,14 +21,6 @@ "execution_count": 1, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/user/.local/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - }, { "data": { "text/markdown": [ @@ -39,14 +34,22 @@ " padding: Union[int, Tuple[int, int]] = 0,\n", " dilation: Union[int, Tuple[int, int]] = 1,\n", " groups: int = 1,\n", - " bias: bool = True,\n", - " padding_type: str = 'standard',\n", + " padding_mode: str = 'zeros',\n", + " bias: Optional[bool] = True,\n", " weight_quant: Optional[WeightQuantType] = Int8WeightPerTensorFloat,\n", " bias_quant: Optional[BiasQuantType] = None,\n", " input_quant: Optional[ActQuantType] = None,\n", " output_quant: Optional[ActQuantType] = None,\n", " return_quant_tensor: bool = False,\n", + " device: Optional[torch.device] = None,\n", + " dtype: Optional[torch.dtype] = None,\n", " **kwargs) -> None:\n", + " # avoid an init error in the super class by setting padding to 0\n", + " if padding_mode == 'zeros' and padding == 'same' and stride > 1:\n", + " padding = 0\n", + " is_same_padded_strided = True\n", + " else:\n", + " is_same_padded_strided = False\n", " Conv2d.__init__(\n", " self,\n", " in_channels=in_channels,\n", @@ -54,9 +57,12 @@ " kernel_size=kernel_size,\n", " stride=stride,\n", " padding=padding,\n", + " padding_mode=padding_mode,\n", " dilation=dilation,\n", " groups=groups,\n", - " bias=bias)\n", + " bias=bias,\n", + " device=device,\n", + " dtype=dtype)\n", " QuantWBIOL.__init__(\n", " self,\n", " weight_quant=weight_quant,\n", @@ -65,9 +71,7 @@ " output_quant=output_quant,\n", " return_quant_tensor=return_quant_tensor,\n", " **kwargs)\n", - " assert self.padding_mode == 'zeros'\n", - " assert not (padding_type == 'same' and padding != 0)\n", - " self.padding_type = padding_type\n", + " self.is_same_padded_strided = is_same_padded_strided\n", "\n", "```" ], @@ -84,9 +88,18 @@ "from brevitas.nn import QuantConv2d\n", "from brevitas.nn import QuantIdentity\n", "from IPython.display import Markdown, display\n", + "import torch\n", + "\n", + "# helpers\n", + "def assert_with_message(condition):\n", + " assert condition\n", + " print(condition)\n", "\n", "def pretty_print_source(source):\n", " display(Markdown('```python\\n' + source + '\\n```'))\n", + "\n", + "# set manual seed for the notebook\n", + "torch.manual_seed(0)\n", " \n", "source = inspect.getsource(QuantConv2d.__init__) \n", "pretty_print_source(source)" @@ -149,20 +162,28 @@ "scrolled": true }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/_tensor.py:1362: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/c10/core/TensorImpl.h:1900.)\n", + " return super().rename(names)\n" + ] + }, { "data": { "text/plain": [ - "tensor([[[[-0.2594, 0.5392, 0.5916],\n", - " [ 0.3493, 0.6813, 0.2499],\n", - " [ 1.3732, 0.1229, -0.0084]],\n", + "tensor([[[[ 0.2908, -0.1793, -0.9610],\n", + " [-0.6542, -0.3532, 0.6361],\n", + " [ 1.0290, 0.2730, 0.0969]],\n", "\n", - " [[ 0.0031, -0.1702, 0.1069],\n", - " [-0.8181, -0.8056, 0.0385],\n", - " [-0.4738, 0.0589, 0.1278]],\n", + " [[-0.3479, 0.6030, 0.4900],\n", + " [ 0.1607, 0.3547, -0.4283],\n", + " [-0.6696, 0.0652, 0.7300]],\n", "\n", - " [[-0.1718, -0.1162, -0.1526],\n", - " [-0.9903, -0.3541, 0.1645],\n", - " [ 0.0557, -0.4458, -0.2080]]]], grad_fn=)" + " [[-0.0769, -0.2424, 0.1860],\n", + " [ 0.1740, -0.1182, -0.7017],\n", + " [ 0.0963, 0.2375, -0.9439]]]], grad_fn=)" ] }, "execution_count": 4, @@ -195,7 +216,15 @@ "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ "from torch.nn import Conv2d\n", "\n", @@ -206,7 +235,7 @@ "float_conv = Conv2d(\n", " in_channels=2, out_channels=3, kernel_size=(3,3), bias=False)\n", "inp = torch.randn(1, 2, 5, 5)\n", - "assert torch.isclose(disabled_quant_conv(inp), float_conv(inp)).all().item()" + "assert_with_message(torch.isclose(disabled_quant_conv(inp), float_conv(inp)).all().item())" ] }, { @@ -234,31 +263,31 @@ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[-0.0790, 0.0503, -0.0934],\n", - " [-0.1149, -0.1903, -0.1329],\n", - " [-0.1813, 0.0108, 0.0593]],\n", + "QuantTensor(value=tensor([[[[-0.0018, 0.1273, -0.1937],\n", + " [-0.1734, -0.0904, 0.0627],\n", + " [-0.0055, 0.1863, -0.0203]],\n", "\n", - " [[ 0.0970, -0.0215, -0.0144],\n", - " [ 0.2280, 0.1239, -0.0090],\n", - " [ 0.1957, -0.2011, -0.0108]]],\n", + " [[ 0.0627, -0.0720, -0.0461],\n", + " [-0.2251, -0.1568, -0.0978],\n", + " [ 0.0092, 0.0941, 0.1421]]],\n", "\n", "\n", - " [[[-0.0018, -0.1957, 0.1993],\n", - " [-0.0359, 0.1778, -0.1400],\n", - " [ 0.0916, 0.1059, 0.2173]],\n", + " [[[-0.1605, -0.1033, 0.0849],\n", + " [ 0.1956, -0.0480, 0.1771],\n", + " [-0.0387, 0.0258, 0.2140]],\n", "\n", - " [[-0.1670, 0.1939, -0.2191],\n", - " [-0.0215, 0.1688, -0.1383],\n", - " [-0.0449, -0.1185, 0.1742]]],\n", + " [[-0.2196, -0.1476, -0.0590],\n", + " [-0.0923, 0.2030, -0.1531],\n", + " [-0.1089, -0.1642, -0.2214]]],\n", "\n", "\n", - " [[[-0.0808, -0.1652, -0.0233],\n", - " [-0.0700, 0.0467, -0.0485],\n", - " [ 0.1059, 0.1418, 0.1077]],\n", + " [[[-0.1384, 0.2030, 0.1052],\n", + " [ 0.1144, 0.0129, -0.1199],\n", + " [ 0.0406, -0.2196, -0.1697]],\n", "\n", - " [[-0.0593, 0.0108, 0.0036],\n", - " [-0.1508, 0.0808, 0.1616],\n", - " [ 0.0144, -0.0287, -0.1365]]]], grad_fn=), scale=tensor(0.0018, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" + " [[-0.1218, 0.1494, 0.1384],\n", + " [-0.1052, -0.0092, 0.1513],\n", + " [ 0.2343, 0.0941, 0.0314]]]], grad_fn=), scale=tensor(0.0018, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" ] }, "execution_count": 6, @@ -288,14 +317,22 @@ "cell_type": "code", "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ "int_weight = default_quant_conv.int_weight()\n", "zero_point = default_quant_conv.quant_weight_zero_point()\n", "scale = default_quant_conv.quant_weight_scale()\n", "quant_weight_manually = (int_weight - zero_point) * scale\n", "\n", - "assert default_quant_conv.quant_weight().value.isclose(quant_weight_manually).all().item()" + "assert_with_message(default_quant_conv.quant_weight().value.isclose(quant_weight_manually).all().item())" ] }, { @@ -310,9 +347,17 @@ "cell_type": "code", "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ - "assert default_quant_conv.quant_weight().is_valid" + "assert_with_message(default_quant_conv.quant_weight().is_valid)" ] }, { @@ -325,15 +370,17 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "tensor(0.0173, grad_fn=)\n", - "tensor(0.0307, grad_fn=)\n" + "True\n", + "True\n", + "tensor(0.0211, grad_fn=)\n", + "tensor(0.0162, grad_fn=)\n" ] } ], @@ -345,8 +392,8 @@ "out_tensor_0 = quant_act(torch.randn(1,2,5,5))\n", "out_tensor_1 = quant_act(torch.randn(1,2,5,5))\n", "\n", - "assert out_tensor_0.is_valid\n", - "assert out_tensor_1.is_valid\n", + "assert_with_message(out_tensor_0.is_valid)\n", + "assert_with_message(out_tensor_1.is_valid)\n", "print(out_tensor_0.scale)\n", "print(out_tensor_1.scale)" ] @@ -361,27 +408,27 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.9489, -0.9111, -0.0536, 0.5788, 0.3645],\n", - " [ 0.3401, 1.4325, 0.6498, 0.6411, -1.4390],\n", - " [-1.9029, 0.7012, 0.1591, 1.9235, 0.5883],\n", - " [-2.7258, 2.5330, 0.9165, -0.0820, 3.4148],\n", - " [-0.3651, 1.0164, 0.9567, -0.2758, -1.1376]],\n", + "QuantTensor(value=tensor([[[[-0.1106, 1.1945, -0.4972, -2.0968, 0.7175],\n", + " [-2.5901, 0.0588, -0.2014, 2.1486, 1.6435],\n", + " [ 0.9067, -2.5212, 2.2193, 0.2352, -0.8395],\n", + " [-0.8351, 0.6341, -0.5551, 0.1040, -3.3151],\n", + " [-0.8979, -0.7092, 3.8232, 1.0875, 0.3954]],\n", "\n", - " [[-0.2414, 2.2111, -1.9124, -2.3814, -0.8805],\n", - " [ 1.3191, -0.8965, -0.2048, -3.8113, 1.1142],\n", - " [-0.3381, -0.2238, 1.2661, 0.0068, 0.2567],\n", - " [ 0.0731, -0.4280, 0.0909, 0.0875, -1.6851],\n", - " [-0.7744, -1.4127, -0.8143, 1.3557, -0.2802]]]],\n", - " grad_fn=), scale=tensor(0.0240, grad_fn=), zero_point=tensor(0.), bit_width=tensor(9.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 1.4363, -1.3973, 1.3249, 2.6914, 0.3660],\n", + " [ 1.5057, 1.8094, 0.5100, -1.6874, 1.9981],\n", + " [ 1.2472, -1.7813, 0.0334, -1.2880, -2.9333],\n", + " [ 0.0180, -1.4298, -2.9978, 0.5494, -1.4548],\n", + " [ 1.6738, -0.3177, -0.3721, -0.1650, -1.1871]]]],\n", + " grad_fn=), scale=tensor(0.0187, grad_fn=), zero_point=tensor(0.), bit_width=tensor(9.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 11, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -401,11 +448,19 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ - "assert not out_tensor.is_valid" + "assert_with_message(not out_tensor.is_valid)" ] }, { @@ -417,23 +472,23 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[1.5800, 1.0157],\n", - " [1.4445, 0.8577]],\n", + "QuantTensor(value=tensor([[[[0.5191, 0.6402],\n", + " [2.1455, 0.5883]],\n", "\n", - " [[0.5643, 1.2414],\n", - " [1.0383, 0.9028]],\n", + " [[2.0417, 0.5883],\n", + " [1.2631, 0.3980]],\n", "\n", - " [[0.5191, 0.6546],\n", - " [2.1442, 0.5868]]]], grad_fn=), scale=tensor(0.0226, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" + " [[0.7959, 0.5191],\n", + " [0.8132, 1.3496]]]], grad_fn=), scale=tensor(0.0173, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 108, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -455,29 +510,37 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 13, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_59908/1377665000.py:1: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/torch/csrc/utils/python_arg_parser.cpp:368.)\n", + " torch.tanh(quant_tensor)\n" + ] + }, { "data": { "text/plain": [ - "tensor([[[[-0.4943, -0.9938, -0.9073, 0.7681],\n", - " [-0.3262, 0.9186, 0.1786, 0.3659],\n", - " [ 0.7489, 0.8946, -0.0451, -0.5594],\n", - " [-0.1346, -0.4943, -0.4770, 0.6951]],\n", + "tensor([[[[ 0.4770, 0.2212, 0.0691, 0.5650],\n", + " [-0.0346, -0.6618, -0.4635, -0.3482],\n", + " [ 0.9730, -0.7245, -0.5881, -0.5287],\n", + " [-0.0863, 0.8857, 0.5287, -0.4498]],\n", "\n", - " [[ 0.0676, 0.5111, 0.4943, 0.8459],\n", - " [-0.8990, -0.9426, 0.0676, -0.7945],\n", - " [-0.9220, 0.0676, -0.5594, 0.6321],\n", - " [-0.0676, 0.7772, 0.7177, -0.4414]],\n", + " [[ 0.9669, 0.5650, -0.6211, -0.4498],\n", + " [-0.2376, 0.6103, 0.5287, 0.2700],\n", + " [-0.6808, 0.8519, 0.2700, -0.5531],\n", + " [-0.0173, 0.8264, 0.3782, -0.1881]],\n", "\n", - " [[ 0.4770, 0.2220, 0.0676, 0.5747],\n", - " [-0.0451, -0.6710, -0.4594, -0.3462],\n", - " [ 0.9729, -0.7177, -0.5896, -0.5276],\n", - " [-0.0900, 0.8852, 0.5276, -0.4414]]]], grad_fn=)" + " [[-0.6211, -0.9764, -0.5993, 0.4770],\n", + " [ 0.5033, 0.6618, -0.1881, -0.6211],\n", + " [-0.8031, 0.1375, 0.5287, 0.8740],\n", + " [-0.6714, 0.6714, -0.5650, 0.8611]]]], grad_fn=)" ] }, - "execution_count": 109, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -497,26 +560,26 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.9693, -0.9431, 0.2459],\n", - " [ 0.5416, 0.9037, -0.5278],\n", - " [-0.6207, -1.3578, -0.4815]],\n", + "QuantTensor(value=tensor([[[[-0.3568, -0.1883, 0.3589],\n", + " [-0.4470, 0.1039, -0.3945],\n", + " [-0.4190, 0.3723, 0.8384]],\n", "\n", - " [[ 0.4551, -1.4065, 0.8889],\n", - " [-0.3393, 0.0803, -0.1748],\n", - " [-0.0977, 0.6284, -0.7193]],\n", + " [[-0.0510, 0.5514, -0.2751],\n", + " [-0.5668, 0.5824, 0.2328],\n", + " [ 0.1316, -0.2518, 1.0418]],\n", "\n", - " [[ 0.3655, 0.7626, -0.2634],\n", - " [-0.3453, 0.3349, 0.1923],\n", - " [ 0.5993, -0.9579, 0.3557]]]], grad_fn=), scale=tensor([[[[3.2208e-05]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 0.2734, 0.7268, -0.0249],\n", + " [-0.1732, 0.5197, 1.1158],\n", + " [ 0.3771, -0.3810, 0.2008]]]], grad_fn=), scale=tensor([[[[3.1958e-05]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 110, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -533,22 +596,19 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 15, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 111, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert out_tensor.is_valid" + "assert_with_message(out_tensor.is_valid)" ] }, { @@ -569,26 +629,26 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 5.7000e-03, 2.5000e-03, -1.2400e-02, -7.2000e-03, 3.7000e-03],\n", - " [-2.3000e-03, 7.0000e-04, -1.2700e-02, 5.2000e-03, 4.0000e-04],\n", - " [-7.9000e-03, 9.5000e-03, 6.6000e-03, 5.4000e-03, 2.5000e-03],\n", - " [ 1.1100e-02, 2.4000e-03, 1.0000e-02, -3.7000e-03, 7.2000e-03],\n", - " [-1.1500e-02, -5.8000e-03, -9.3000e-03, 1.0000e-02, 3.5000e-03]],\n", + "QuantTensor(value=tensor([[[[ 7.2000e-03, -3.7000e-03, 7.7000e-03, -2.4000e-03, -8.9000e-03],\n", + " [-1.2000e-02, -8.1000e-03, 7.2000e-03, -1.1300e-02, -9.7000e-03],\n", + " [-1.0000e-03, 1.0100e-02, 3.8000e-03, -1.1900e-02, 6.9000e-03],\n", + " [ 8.3000e-03, 1.0000e-04, -6.9000e-03, 3.9000e-03, -5.4000e-03],\n", + " [ 1.1300e-02, -6.0000e-03, 9.7000e-03, 0.0000e+00, 1.0900e-02]],\n", "\n", - " [[-6.8000e-03, 1.1500e-02, -1.0600e-02, -1.5000e-03, -1.9000e-03],\n", - " [ 2.9000e-03, 9.5000e-03, 7.2000e-03, -3.7000e-03, 7.7000e-03],\n", - " [-2.4000e-03, -8.9000e-03, -1.2000e-02, -8.1000e-03, 7.2000e-03],\n", - " [-1.1300e-02, -9.7000e-03, -1.0000e-03, 1.0100e-02, 3.8000e-03],\n", - " [-1.1900e-02, 6.9000e-03, 8.3000e-03, 1.0000e-04, -6.9000e-03]]]]), scale=tensor(1.0000e-04), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" + " [[-1.0900e-02, 1.1400e-02, -6.4000e-03, 9.2000e-03, 7.1000e-03],\n", + " [-6.0000e-04, 9.2000e-03, -8.5000e-03, 5.0000e-03, 6.5000e-03],\n", + " [-8.3000e-03, -1.2000e-03, 7.4000e-03, 9.2000e-03, -6.0000e-04],\n", + " [-2.1000e-03, 9.5000e-03, 3.0000e-04, -2.9000e-03, -6.5000e-03],\n", + " [-1.1800e-02, -4.8000e-03, 5.4000e-03, -2.5000e-03, 9.0000e-04]]]]), scale=tensor(1.0000e-04), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 112, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -613,22 +673,19 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 17, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 113, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert quant_tensor_input.is_valid" + "assert_with_message(quant_tensor_input.is_valid)" ] }, { @@ -642,26 +699,26 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.0085, 0.0066, 0.0050],\n", - " [-0.0038, -0.0009, -0.0115],\n", - " [-0.0055, -0.0037, 0.0009]],\n", + "QuantTensor(value=tensor([[[[-0.0019, 0.0049, -0.0012],\n", + " [-0.0012, 0.0050, -0.0074],\n", + " [-0.0023, -0.0035, -0.0033]],\n", "\n", - " [[ 0.0015, -0.0027, -0.0079],\n", - " [-0.0034, -0.0060, 0.0043],\n", - " [-0.0008, 0.0052, -0.0033]],\n", + " [[-0.0031, 0.0028, 0.0116],\n", + " [ 0.0079, 0.0046, 0.0022],\n", + " [ 0.0021, -0.0004, 0.0011]],\n", "\n", - " [[-0.0015, 0.0082, -0.0038],\n", - " [-0.0021, 0.0004, -0.0054],\n", - " [-0.0021, -0.0079, 0.0013]]]], grad_fn=), scale=tensor([[[[1.8448e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" + " [[-0.0045, -0.0010, 0.0002],\n", + " [-0.0044, 0.0027, 0.0025],\n", + " [-0.0009, 0.0040, -0.0044]]]], grad_fn=), scale=tensor([[[[1.8307e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 114, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -675,22 +732,19 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 19, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 115, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert out_tensor.is_valid" + "assert_with_message(out_tensor.is_valid)" ] }, { @@ -702,26 +756,26 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[-0.0035, -0.0037, -0.0050],\n", - " [ 0.0010, -0.0051, -0.0027],\n", - " [-0.0010, 0.0047, 0.0017]],\n", + "QuantTensor(value=tensor([[[[-0.0073, 0.0040, -0.0011],\n", + " [-0.0033, 0.0078, -0.0028],\n", + " [ 0.0005, -0.0025, -0.0008]],\n", "\n", - " [[ 0.0021, 0.0002, 0.0027],\n", - " [ 0.0028, 0.0002, -0.0044],\n", - " [ 0.0008, -0.0052, -0.0024]],\n", + " [[ 0.0021, -0.0021, 0.0035],\n", + " [ 0.0012, -0.0016, -0.0023],\n", + " [-0.0010, -0.0015, 0.0040]],\n", "\n", - " [[ 0.0010, -0.0052, -0.0011],\n", - " [-0.0018, 0.0024, 0.0011],\n", - " [-0.0001, 0.0039, 0.0035]]]], grad_fn=), scale=tensor([[[[1.7410e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" + " [[-0.0010, 0.0047, 0.0025],\n", + " [-0.0014, 0.0021, -0.0039],\n", + " [ 0.0036, -0.0003, 0.0026]]]], grad_fn=), scale=tensor([[[[1.7393e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 116, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -741,26 +795,26 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.2111, 0.4060, 0.3654],\n", - " [-0.7876, 0.8119, -0.9825],\n", - " [-0.5115, 0.3979, -0.3248]],\n", + "QuantTensor(value=tensor([[[[-0.2117, -0.4811, 0.0385],\n", + " [-0.5100, -0.2502, -0.2213],\n", + " [-0.5773, 0.0192, -0.5485]],\n", "\n", - " [[ 0.3816, 0.0568, -0.0812],\n", - " [ 1.0312, -0.7876, 0.8038],\n", - " [-0.3491, -0.4141, 0.0650]],\n", + " [[ 0.1347, 0.8179, -1.2316],\n", + " [-0.6062, 0.4426, -0.3849],\n", + " [ 0.1732, -0.5100, -0.1251]],\n", "\n", - " [[-0.5846, -0.4222, -0.0731],\n", - " [-0.7389, 0.5034, -0.2517],\n", - " [-0.1624, -0.4385, 0.7308]]]], grad_fn=), scale=tensor(0.0081, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 1.0873, 0.2406, -0.2887],\n", + " [-0.4330, -0.4907, -0.2021],\n", + " [ 0.6447, 0.4811, 0.1347]]]], grad_fn=), scale=tensor(0.0096, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 117, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -777,22 +831,19 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 22, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 118, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert out_tensor.is_valid" + "assert_with_message(out_tensor.is_valid)" ] }, { @@ -816,7 +867,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 23, "metadata": { "tags": [ "raises-exception" @@ -830,12 +881,14 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_48365/2280634207.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0min_channels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_channels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m bias_quant=Int8Bias, return_quant_tensor=True)\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mbias_quant_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/opt/conda/envs/torch_1.10/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1103\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/workspace/scratch/git/fork_brevitas/src/brevitas/nn/quant_conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQuantTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQuantTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 192\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minner_forward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquant_weight\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquant_bias\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/workspace/scratch/git/fork_brevitas/src/brevitas/nn/quant_layer.py\u001b[0m in \u001b[0;36mforward_impl\u001b[0;34m(self, inp)\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m \u001b[0mquant_bias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_quant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_scale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_bit_width\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_inference_quant_bias\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cached_bias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_CachedIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquant_bias\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/envs/torch_1.10/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1103\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/workspace/scratch/git/fork_brevitas/src/brevitas/proxy/parameter_quant.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, input_scale, input_bit_width)\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0mimpl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport_handler\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport_mode\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor_quant\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_input_scale\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minput_scale\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Input scale required\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_input_bit_width\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minput_bit_width\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Input bit-width required\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "Cell \u001b[0;32mIn[23], line 6\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbrevitas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mquant\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mscaled_int\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Int8Bias\n\u001b[1;32m 3\u001b[0m bias_quant_conv \u001b[38;5;241m=\u001b[39m QuantConv2d(\n\u001b[1;32m 4\u001b[0m in_channels\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, out_channels\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m, kernel_size\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m3\u001b[39m,\u001b[38;5;241m3\u001b[39m), bias\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 5\u001b[0m bias_quant\u001b[38;5;241m=\u001b[39mInt8Bias, return_quant_tensor\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 6\u001b[0m bias_quant_conv(torch\u001b[38;5;241m.\u001b[39mrandn(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m5\u001b[39m, \u001b[38;5;241m5\u001b[39m))\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_conv.py:198\u001b[0m, in \u001b[0;36mQuantConv2d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Union[Tensor, QuantTensor]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[Tensor, QuantTensor]:\n\u001b[0;32m--> 198\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_impl(\u001b[38;5;28minput\u001b[39m)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_layer.py:326\u001b[0m, in \u001b[0;36mQuantWeightBiasInputOutputLayer.forward_impl\u001b[0;34m(self, inp)\u001b[0m\n\u001b[1;32m 323\u001b[0m output_signed \u001b[38;5;241m=\u001b[39m inp\u001b[38;5;241m.\u001b[39msigned \u001b[38;5;129;01mor\u001b[39;00m quant_weight\u001b[38;5;241m.\u001b[39msigned\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 326\u001b[0m quant_bias \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias_quant(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias, output_scale, output_bit_width)\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcache_inference_quant_bias:\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cached_bias \u001b[38;5;241m=\u001b[39m _CachedIO(quant_bias\u001b[38;5;241m.\u001b[39mdetach(), metadata_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/parameter_quant.py:206\u001b[0m, in \u001b[0;36mBiasQuantProxyFromInjector.forward\u001b[0;34m(self, x, input_scale, input_bit_width)\u001b[0m\n\u001b[1;32m 204\u001b[0m impl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexport_handler \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexport_mode \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtensor_quant\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_input_scale \u001b[38;5;129;01mand\u001b[39;00m input_scale \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInput scale required\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 207\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_input_bit_width \u001b[38;5;129;01mand\u001b[39;00m input_bit_width \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInput bit-width required\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "\u001b[0;31mRuntimeError\u001b[0m: Input scale required" ] } @@ -858,26 +911,27 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.0005, 0.0043, -0.0004],\n", - " [ 0.0005, 0.0106, 0.0012],\n", - " [ 0.0021, 0.0007, -0.0050]],\n", + "QuantTensor(value=tensor([[[[-2.4238e-03, -5.6598e-03, 5.1882e-03],\n", + " [-6.5582e-03, 8.9274e-03, 4.9640e-04],\n", + " [ 9.6283e-03, -1.7466e-03, -4.8311e-03]],\n", "\n", - " [[-0.0067, -0.0035, -0.0059],\n", - " [-0.0050, -0.0015, -0.0039],\n", - " [ 0.0015, 0.0028, -0.0008]],\n", + " [[ 2.9322e-03, -3.1358e-03, -6.2727e-04],\n", + " [ 2.8722e-06, -3.7981e-03, 1.0973e-02],\n", + " [-4.1031e-03, 6.5909e-03, -4.2369e-03]],\n", "\n", - " [[-0.0051, -0.0050, 0.0060],\n", - " [-0.0015, 0.0037, 0.0071],\n", - " [ 0.0067, 0.0035, -0.0071]]]], grad_fn=), scale=tensor([[[[1.8108e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(22.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 4.1967e-03, -7.0733e-03, 1.6456e-03],\n", + " [ 1.8197e-03, -3.1683e-03, 4.8200e-03],\n", + " [-3.2585e-04, 3.1055e-03, 1.9703e-03]]]],\n", + " grad_fn=), scale=tensor([[[[1.7953e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(22.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 120, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -895,26 +949,26 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[-0.3825, 0.1371, 0.9135],\n", - " [-0.2016, 0.7495, -0.4071],\n", - " [-0.0755, 0.5283, 0.2388]],\n", + "QuantTensor(value=tensor([[[[-0.2816, -0.5271, -0.1748],\n", + " [-0.4247, -0.1575, 0.0681],\n", + " [ 0.6528, -0.5346, -0.0657]],\n", "\n", - " [[ 0.0788, -0.3802, -0.2234],\n", - " [ 0.8678, -0.5546, 0.4408],\n", - " [-0.6788, 0.4422, 0.3007]],\n", + " [[ 0.2993, -0.3383, 0.3035],\n", + " [-0.4595, -0.6796, -0.9720],\n", + " [-0.1948, -0.5169, -0.2175]],\n", "\n", - " [[ 0.4412, -0.3205, 1.0033],\n", - " [-0.0083, -0.3295, -0.2076],\n", - " [ 0.4417, -0.1046, -0.3493]]]], grad_fn=), scale=tensor([[[[3.8610e-05]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(22.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 0.5586, 0.0665, -0.5807],\n", + " [ 0.5565, 0.1780, -0.0555],\n", + " [-0.1080, 0.0791, -0.2262]]]], grad_fn=), scale=tensor([[[[4.2009e-05]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(22.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 121, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -928,26 +982,26 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.0036, 0.0024, -0.0033],\n", - " [ 0.0050, 0.0080, -0.0014],\n", - " [-0.0036, -0.0080, -0.0029]],\n", + "QuantTensor(value=tensor([[[[-0.0058, 0.0030, 0.0030],\n", + " [-0.0013, -0.0002, 0.0043],\n", + " [-0.0061, 0.0033, -0.0001]],\n", "\n", - " [[ 0.0083, -0.0093, 0.0048],\n", - " [ 0.0035, 0.0015, -0.0011],\n", - " [-0.0003, 0.0067, 0.0013]],\n", + " [[ 0.0013, -0.0008, -0.0015],\n", + " [ 0.0011, 0.0012, -0.0012],\n", + " [-0.0013, -0.0020, 0.0002]],\n", "\n", - " [[-0.0009, -0.0019, 0.0039],\n", - " [ 0.0010, 0.0056, -0.0037],\n", - " [ 0.0091, -0.0095, 0.0054]]]], grad_fn=), scale=tensor([[[[1.8384e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(22.), signed_t=tensor(True), training_t=tensor(True))" + " [[-0.0061, 0.0053, -0.0004],\n", + " [ 0.0028, 0.0031, -0.0037],\n", + " [ 0.0027, -0.0048, -0.0044]]]], grad_fn=), scale=tensor([[[[1.7370e-07]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(22.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 122, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -967,7 +1021,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 27, "metadata": { "tags": [ "raises-exception" @@ -981,12 +1035,14 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_48365/2990591641.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0min_channels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_channels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkernel_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m output_quant=Int8ActPerTensorFloat, bias_quant=Int8Bias, return_quant_tensor=True)\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0moutput_bias_quant_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/opt/conda/envs/torch_1.10/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1103\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/workspace/scratch/git/fork_brevitas/src/brevitas/nn/quant_conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQuantTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQuantTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 192\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minner_forward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquant_weight\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquant_bias\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/workspace/scratch/git/fork_brevitas/src/brevitas/nn/quant_layer.py\u001b[0m in \u001b[0;36mforward_impl\u001b[0;34m(self, inp)\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m \u001b[0mquant_bias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_quant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_scale\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_bit_width\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_inference_quant_bias\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cached_bias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_CachedIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquant_bias\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/conda/envs/torch_1.10/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1100\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1101\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1103\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1104\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/workspace/scratch/git/fork_brevitas/src/brevitas/proxy/parameter_quant.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, input_scale, input_bit_width)\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0mimpl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport_handler\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport_mode\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor_quant\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_input_scale\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minput_scale\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Input scale required\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_input_bit_width\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minput_bit_width\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Input bit-width required\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "Cell \u001b[0;32mIn[27], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m output_bias_quant_conv \u001b[38;5;241m=\u001b[39m QuantConv2d(\n\u001b[1;32m 2\u001b[0m in_channels\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, out_channels\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m, kernel_size\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m3\u001b[39m,\u001b[38;5;241m3\u001b[39m), bias\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 3\u001b[0m output_quant\u001b[38;5;241m=\u001b[39mInt8ActPerTensorFloat, bias_quant\u001b[38;5;241m=\u001b[39mInt8Bias, return_quant_tensor\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 4\u001b[0m output_bias_quant_conv(torch\u001b[38;5;241m.\u001b[39mrandn(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m5\u001b[39m, \u001b[38;5;241m5\u001b[39m))\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_conv.py:198\u001b[0m, in \u001b[0;36mQuantConv2d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Union[Tensor, QuantTensor]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[Tensor, QuantTensor]:\n\u001b[0;32m--> 198\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_impl(\u001b[38;5;28minput\u001b[39m)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_layer.py:326\u001b[0m, in \u001b[0;36mQuantWeightBiasInputOutputLayer.forward_impl\u001b[0;34m(self, inp)\u001b[0m\n\u001b[1;32m 323\u001b[0m output_signed \u001b[38;5;241m=\u001b[39m inp\u001b[38;5;241m.\u001b[39msigned \u001b[38;5;129;01mor\u001b[39;00m quant_weight\u001b[38;5;241m.\u001b[39msigned\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 326\u001b[0m quant_bias \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias_quant(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias, output_scale, output_bit_width)\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcache_inference_quant_bias:\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cached_bias \u001b[38;5;241m=\u001b[39m _CachedIO(quant_bias\u001b[38;5;241m.\u001b[39mdetach(), metadata_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/parameter_quant.py:206\u001b[0m, in \u001b[0;36mBiasQuantProxyFromInjector.forward\u001b[0;34m(self, x, input_scale, input_bit_width)\u001b[0m\n\u001b[1;32m 204\u001b[0m impl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexport_handler \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexport_mode \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtensor_quant\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_input_scale \u001b[38;5;129;01mand\u001b[39;00m input_scale \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInput scale required\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 207\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_input_bit_width \u001b[38;5;129;01mand\u001b[39;00m input_bit_width \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInput bit-width required\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", "\u001b[0;31mRuntimeError\u001b[0m: Input scale required" ] } @@ -1007,26 +1063,26 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([[[[ 0.2152, 0.8346, 0.0746],\n", - " [-0.0738, -0.5212, 0.1019],\n", - " [-0.6004, 0.1500, -0.1453]],\n", + "tensor([[[[-0.4360, -0.2674, -0.4194],\n", + " [-0.2412, -0.6360, -0.6838],\n", + " [-0.5227, -0.0199, -0.1445]],\n", "\n", - " [[-1.1551, -1.3458, -0.1312],\n", - " [ 0.2502, -0.5267, 0.2412],\n", - " [-0.3556, -0.3289, -0.2276]],\n", + " [[-0.3524, 0.8025, 0.2844],\n", + " [ 0.9945, -0.4782, 0.8064],\n", + " [ 0.5732, 0.1249, 0.3110]],\n", "\n", - " [[-0.4599, -0.6094, 0.4682],\n", - " [-0.5064, -0.6768, -0.6638],\n", - " [ 0.0066, -0.3581, 0.2359]]]], grad_fn=)" + " [[ 0.3223, 0.2530, 0.2753],\n", + " [ 0.5764, -0.2533, -0.0181],\n", + " [-0.4147, 0.2049, -0.9944]]]], grad_fn=)" ] }, - "execution_count": 124, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -1051,30 +1107,30 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[-0.6879, -0.6632, -0.2411],\n", - " [ 0.2064, -0.7371, 0.3910],\n", - " [ 0.9533, 0.2994, 0.6546]],\n", + "QuantTensor(value=tensor([[[[-0.6912, 0.0086, 0.1628],\n", + " [-0.4786, -0.8073, 0.5224],\n", + " [ 0.4157, 0.4686, 0.2560]],\n", "\n", - " [[-0.4684, -0.4495, -0.5021],\n", - " [ 0.5738, 0.4199, -0.3380],\n", - " [ 0.6218, -0.0408, -0.8483]],\n", + " [[ 0.3170, -0.5486, -0.5216],\n", + " [ 0.1832, 1.0217, -0.3637],\n", + " [-0.1115, 0.6974, -0.0452]],\n", "\n", - " [[-0.5625, 0.1837, -1.0575],\n", - " [-1.2816, -0.4993, -0.3409],\n", - " [ 0.4556, -1.4269, 0.5369]]]], grad_fn=), scale=tensor([[[[3.0975e-05]]]], grad_fn=), zero_point=tensor([[[[ 1276.0774]],\n", + " [[-0.6168, -0.5241, -0.6593],\n", + " [ 0.6408, 0.2674, 0.4537],\n", + " [-0.3744, -0.7771, -0.2848]]]], grad_fn=), scale=tensor([[[[3.0094e-05]]]], grad_fn=), zero_point=tensor([[[[ 339.3404]],\n", "\n", - " [[-3152.4585]],\n", + " [[-4597.1797]],\n", "\n", - " [[ 7320.2324]]]], grad_fn=), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" + " [[-3452.3711]]]], grad_fn=), bit_width=tensor(21.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 125, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -1089,22 +1145,19 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 30, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 126, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert out_tensor.is_valid" + "assert_with_message(out_tensor.is_valid)" ] }, { @@ -1116,26 +1169,26 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([[[[ 0.8357, 0.0733, 0.9527],\n", - " [ 0.1803, 0.2154, 0.7598],\n", - " [ 1.1121, -0.8728, 1.0039]],\n", + "tensor([[[[-0.2327, 0.9267, 0.6294],\n", + " [ 0.0901, 0.1027, -0.0727],\n", + " [-0.5614, 0.6182, 0.5394]],\n", "\n", - " [[ 0.7917, 1.0063, 0.6516],\n", - " [-0.1852, -0.7263, 0.0956],\n", - " [-0.1876, 0.2747, -0.1617]],\n", + " [[ 0.4179, -0.5184, -0.2016],\n", + " [ 0.1390, -0.3925, -0.6171],\n", + " [ 0.4782, 0.0814, 0.6124]],\n", "\n", - " [[ 0.8299, 0.9934, -0.3821],\n", - " [ 0.4865, 0.9309, -0.7924],\n", - " [-0.4201, 0.2343, 0.1532]]]], grad_fn=)" + " [[ 0.2896, -0.3779, 0.9408],\n", + " [-0.1334, 0.6186, 0.2167],\n", + " [-0.5926, 0.3690, -0.0284]]]], grad_fn=)" ] }, - "execution_count": 127, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -1157,7 +1210,7 @@ ], "metadata": { "kernelspec": { - "display_name": "torch_1.10", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1171,7 +1224,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.15" + "version": "3.11.5" }, "vscode": { "interpreter": { @@ -1180,5 +1233,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/notebooks/02_quant_activation_overview.ipynb b/notebooks/02_quant_activation_overview.ipynb index 4d2ac73d1..7bb97f46f 100644 --- a/notebooks/02_quant_activation_overview.ipynb +++ b/notebooks/02_quant_activation_overview.ipynb @@ -3,7 +3,10 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } }, "source": [ "# An Overview of Quantized Activations" @@ -21,19 +24,36 @@ { "cell_type": "code", "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# helpers\n", + "def assert_with_message(condition):\n", + " assert condition\n", + " print(condition)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": { "scrolled": true }, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/_tensor.py:1362: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/c10/core/TensorImpl.h:1900.)\n", + " return super().rename(names)\n" + ] } ], "source": [ @@ -41,11 +61,11 @@ "from brevitas.nn import QuantConv2d, QuantIdentity\n", "from brevitas.quant.scaled_int import Int8ActPerTensorFloat \n", "\n", - "torch.manual_seed(0)\n", + "torch.manual_seed(0) # set a seed to make sure the random weight init is reproducible\n", "output_quant_conv = QuantConv2d(\n", " in_channels=2, out_channels=3, kernel_size=(3,3), output_quant=Int8ActPerTensorFloat)\n", "\n", - "torch.manual_seed(0)\n", + "torch.manual_seed(0) # reproduce the same random weight init as above\n", "default_quant_conv = QuantConv2d(\n", " in_channels=2, out_channels=3, kernel_size=(3,3))\n", "output_identity_quant = QuantIdentity()\n", @@ -54,7 +74,7 @@ "out_tensor1 = output_quant_conv(inp)\n", "out_tensor2 = output_identity_quant(default_quant_conv(inp))\n", "\n", - "assert out_tensor1.isclose(out_tensor2).all().item()" + "assert_with_message(out_tensor1.isclose(out_tensor2).all().item())" ] }, { @@ -66,18 +86,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ @@ -96,7 +113,7 @@ "out_tensor1 = input_output_quant_conv(inp)\n", "out_tensor2 = output_identity_quant(default_quant_conv(input_identity_quant(inp)))\n", "\n", - "assert out_tensor1.isclose(out_tensor2).all().item()" + "assert_with_message(out_tensor1.isclose(out_tensor2).all().item())" ] }, { @@ -115,23 +132,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ "disabled_quant_identity = QuantIdentity(act_quant=None)\n", - "(inp == disabled_quant_identity(inp)).all().item()" + "assert_with_message((inp == disabled_quant_identity(inp)).all().item())" ] }, { @@ -143,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -163,7 +177,7 @@ " grad_fn=), scale=tensor(0.0190, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -176,22 +190,19 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert out_tensor.is_valid" + "assert_with_message(out_tensor.is_valid)" ] }, { @@ -203,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -220,10 +231,10 @@ " [-0.7990, -1.2936, -0.7419, -1.3127, -0.2283],\n", " [-2.4351, -0.0761, 0.2283, 0.7990, -0.1902],\n", " [-0.3615, -1.2175, -0.6278, -0.4566, 1.9214]]]],\n", - " grad_fn=)" + " grad_fn=)" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -235,7 +246,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -252,10 +263,10 @@ " [-0.7990, -1.2936, -0.7419, -1.3127, -0.2283],\n", " [-2.4351, -0.0761, 0.2283, 0.7990, -0.1902],\n", " [-0.3615, -1.2175, -0.6278, -0.4566, 1.9214]]]],\n", - " grad_fn=), scale=tensor(0.0190, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" + " grad_fn=), scale=tensor(0.0190, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -275,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -294,7 +305,7 @@ " [0.0000, 0.0000, 0.0000, 0.0000, 1.9230]]]], grad_fn=), scale=tensor(0.0093, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(False), training_t=tensor(True))" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -315,16 +326,27 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "tensor(True)\n", + "tensor(True)\n", + "tensor(True)\n" + ] + } + ], "source": [ "return_disabled_quant_relu = QuantReLU(act_quant=None, return_quant_tensor=True)\n", "relu_out_tensor = return_disabled_quant_relu(out_tensor)\n", - "assert relu_out_tensor.is_valid==True\n", - "assert relu_out_tensor.scale == out_tensor.scale\n", - "assert relu_out_tensor.zero_point == out_tensor.zero_point\n", - "assert relu_out_tensor.bit_width == out_tensor.bit_width" + "assert_with_message((relu_out_tensor.is_valid==True))\n", + "assert_with_message(relu_out_tensor.scale == out_tensor.scale)\n", + "assert_with_message(relu_out_tensor.zero_point == out_tensor.zero_point)\n", + "assert_with_message(relu_out_tensor.bit_width == out_tensor.bit_width)" ] }, { @@ -336,13 +358,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=(tensor([[[[0.3878, 0.3611, 0.3655, 0.6433, 0.8236],\n", + "QuantTensor(value=tensor([[[[0.3878, 0.3611, 0.3655, 0.6433, 0.8236],\n", " [0.6257, 0.3567, 0.3611, 0.5474, 0.4810],\n", " [0.3788, 0.1820, 0.4526, 0.6077, 0.7911],\n", " [0.1630, 0.8883, 0.8471, 0.9151, 0.2456],\n", @@ -353,10 +375,10 @@ " [0.3102, 0.2152, 0.3226, 0.2120, 0.4432],\n", " [0.0805, 0.4810, 0.5568, 0.6898, 0.4526],\n", " [0.4106, 0.2284, 0.3480, 0.3878, 0.8723]]]],\n", - " grad_fn=), None, None, None), scale=None, zero_point=None, bit_width=None, signed_t=None, training_t=tensor(True))" + " grad_fn=), scale=None, zero_point=None, bit_width=None, signed_t=None, training_t=tensor(True))" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -371,22 +393,19 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "assert not sigmoid_out_tensor.is_valid" + "assert_with_message(not sigmoid_out_tensor.is_valid)" ] }, { @@ -400,7 +419,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -416,10 +435,10 @@ " [0.6421, 0.0000, 0.0000, 1.1708, 0.4343],\n", " [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n", " [0.0000, 0.0000, 0.2266, 0.7931, 0.0000],\n", - " [0.0000, 0.0000, 0.0000, 0.0000, 1.9262]]]], grad_fn=), scale=tensor(0.0189, grad_fn=), zero_point=tensor(129., grad_fn=), bit_width=tensor(8.), signed_t=tensor(False), training_t=tensor(True))" + " [0.0000, 0.0000, 0.0000, 0.0000, 1.9262]]]], grad_fn=), scale=tensor(0.0189, grad_fn=), zero_point=tensor(129., grad_fn=), bit_width=tensor(8.), signed_t=tensor(False), training_t=tensor(True))" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -442,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -461,7 +480,7 @@ " [0.0000, 0.0000, 0.4907]]]], grad_fn=)" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -482,7 +501,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -501,7 +520,7 @@ " [0.0000, 0.0000, 0.4839]]]], grad_fn=)" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -535,7 +554,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -553,47 +572,41 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ "out1_train = quant_identity(inp1)\n", "out2_train = quant_identity(inp2)\n", - "assert not out1_train.scale.isclose(out2_train.scale).item()" + "assert_with_message(not out1_train.scale.isclose(out2_train.scale).item())" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ "quant_identity.eval()\n", "out1_eval = quant_identity(inp1)\n", "out2_eval = quant_identity(inp2)\n", - "assert out1_eval.scale.isclose(out2_eval.scale).item()" + "assert_with_message(out1_eval.scale.isclose(out2_eval.scale).item())" ] }, { @@ -605,7 +618,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": { "tags": [ "raises-exception" @@ -617,19 +630,19 @@ "evalue": "'Int8ActPerTensorFloatMinMaxInit' can not resolve attribute 'max_val' while building 'scaling_init_impl'", "output_type": "error", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mDependencyError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mbrevitas\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mQuantHardTanh\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mQuantHardTanh\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\quant_activation.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, act_quant, input_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[0;32m 117\u001b[0m \u001b[0mact_quant\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mact_quant\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 118\u001b[0m \u001b[0mreturn_quant_tensor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_quant_tensor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 119\u001b[1;33m **kwargs)\n\u001b[0m\u001b[0;32m 120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 121\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\quant_layer.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, act_impl, passthrough_act, input_quant, act_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[0;32m 77\u001b[0m \u001b[0mpassthrough_act\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[0mact_quant\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 79\u001b[1;33m **kwargs)\n\u001b[0m\u001b[0;32m 80\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 81\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\mixin\\act.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, act_impl, passthrough_act, act_quant, **kwargs)\u001b[0m\n\u001b[0;32m 157\u001b[0m \u001b[0mproxy_prefix\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'act_'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 158\u001b[0m \u001b[0mkwargs_prefix\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 159\u001b[1;33m **kwargs)\n\u001b[0m\u001b[0;32m 160\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 161\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\mixin\\base.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, quant, proxy_protocol, none_quant_injector, proxy_prefix, kwargs_prefix, **kwargs)\u001b[0m\n\u001b[0;32m 98\u001b[0m \u001b[0mquant_injector\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 99\u001b[0m \u001b[0mquant_injector\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mfilter_kwargs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwargs_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 100\u001b[1;33m \u001b[0mquant\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproxy_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 101\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 102\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mproxy_protocol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\runtime_quant.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, quant_layer, quant_injector)\u001b[0m\n\u001b[0;32m 108\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 109\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 110\u001b[1;33m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mActQuantProxyFromInjector\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 111\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_passthrough_act\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_is_passthrough_act\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 112\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\quant_proxy.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, quant_layer, quant_injector, export_mode, export_handler)\u001b[0m\n\u001b[0;32m 74\u001b[0m \u001b[1;31m# Use a normal list and not a ModuleList since this is a pointer to parent modules\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 75\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtracked_module_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_tracked_module\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_layer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 77\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_handler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexport_handler\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_mode\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexport_mode\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\quant_proxy.py\u001b[0m in \u001b[0;36madd_tracked_module\u001b[1;34m(self, module)\u001b[0m\n\u001b[0;32m 130\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtracked_module_list\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate_tracked_modules\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 132\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minit_tensor_quant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 133\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 134\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Trying to add None as a parent module.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\runtime_quant.py\u001b[0m in \u001b[0;36minit_tensor_quant\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 121\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0minit_tensor_quant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 122\u001b[1;33m \u001b[0mtensor_quant\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtensor_quant\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 123\u001b[0m \u001b[0mact_impl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mact_impl\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 124\u001b[0m \u001b[0mis_act_enabled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_is_act_enabled\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mact_impl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_quant\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", - "\u001b[1;31mDependencyError\u001b[0m: 'Int8ActPerTensorFloatMinMaxInit' can not resolve attribute 'max_val' while building 'scaling_init_impl'" + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mDependencyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[19], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbrevitas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m QuantHardTanh\n\u001b[0;32m----> 3\u001b[0m QuantHardTanh()\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_activation.py:96\u001b[0m, in \u001b[0;36mQuantHardTanh.__init__\u001b[0;34m(self, act_quant, input_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 92\u001b[0m act_quant: Optional[ActQuantType] \u001b[38;5;241m=\u001b[39m Int8ActPerTensorFloatMinMaxInit,\n\u001b[1;32m 93\u001b[0m input_quant: Optional[ActQuantType] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 94\u001b[0m return_quant_tensor: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 95\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 96\u001b[0m QuantNLAL\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 97\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 98\u001b[0m act_impl\u001b[38;5;241m=\u001b[39mnn\u001b[38;5;241m.\u001b[39mHardtanh,\n\u001b[1;32m 99\u001b[0m passthrough_act\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 100\u001b[0m input_quant\u001b[38;5;241m=\u001b[39minput_quant,\n\u001b[1;32m 101\u001b[0m act_quant\u001b[38;5;241m=\u001b[39mact_quant,\n\u001b[1;32m 102\u001b[0m return_quant_tensor\u001b[38;5;241m=\u001b[39mreturn_quant_tensor,\n\u001b[1;32m 103\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_layer.py:36\u001b[0m, in \u001b[0;36mQuantNonLinearActLayer.__init__\u001b[0;34m(self, act_impl, passthrough_act, input_quant, act_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[1;32m 34\u001b[0m QuantLayerMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, return_quant_tensor)\n\u001b[1;32m 35\u001b[0m QuantInputMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, input_quant, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 36\u001b[0m QuantNonLinearActMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, act_impl, passthrough_act, act_quant, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/mixin/act.py:118\u001b[0m, in \u001b[0;36mQuantNonLinearActMixin.__init__\u001b[0;34m(self, act_impl, passthrough_act, act_quant, act_proxy_prefix, act_kwargs_prefix, **kwargs)\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 109\u001b[0m act_impl: Optional[Type[Module]],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 113\u001b[0m act_kwargs_prefix\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 114\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 115\u001b[0m prefixed_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 116\u001b[0m act_kwargs_prefix \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mact_impl\u001b[39m\u001b[38;5;124m'\u001b[39m: act_impl,\n\u001b[1;32m 117\u001b[0m act_kwargs_prefix \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpassthrough_act\u001b[39m\u001b[38;5;124m'\u001b[39m: passthrough_act}\n\u001b[0;32m--> 118\u001b[0m QuantProxyMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 119\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 120\u001b[0m quant\u001b[38;5;241m=\u001b[39mact_quant,\n\u001b[1;32m 121\u001b[0m proxy_prefix\u001b[38;5;241m=\u001b[39mact_proxy_prefix,\n\u001b[1;32m 122\u001b[0m kwargs_prefix\u001b[38;5;241m=\u001b[39mact_kwargs_prefix,\n\u001b[1;32m 123\u001b[0m proxy_protocol\u001b[38;5;241m=\u001b[39mActQuantProxyProtocol,\n\u001b[1;32m 124\u001b[0m none_quant_injector\u001b[38;5;241m=\u001b[39mNoneActQuant,\n\u001b[1;32m 125\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mprefixed_kwargs,\n\u001b[1;32m 126\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:70\u001b[0m, in \u001b[0;36mQuantProxyMixin.__init__\u001b[0;34m(self, quant, proxy_protocol, none_quant_injector, proxy_prefix, kwargs_prefix, **kwargs)\u001b[0m\n\u001b[1;32m 68\u001b[0m quant_injector \u001b[38;5;241m=\u001b[39m quant\n\u001b[1;32m 69\u001b[0m quant_injector \u001b[38;5;241m=\u001b[39m quant_injector\u001b[38;5;241m.\u001b[39mlet(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mfilter_kwargs(kwargs_prefix, kwargs))\n\u001b[0;32m---> 70\u001b[0m quant \u001b[38;5;241m=\u001b[39m quant_injector\u001b[38;5;241m.\u001b[39mproxy_class(\u001b[38;5;28mself\u001b[39m, quant_injector)\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(quant, proxy_protocol):\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/runtime_quant.py:89\u001b[0m, in \u001b[0;36mActQuantProxyFromInjector.__init__\u001b[0;34m(self, quant_layer, quant_injector)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, quant_layer, quant_injector):\n\u001b[0;32m---> 89\u001b[0m QuantProxyFromInjector\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, quant_layer, quant_injector)\n\u001b[1;32m 90\u001b[0m ActQuantProxyProtocol\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_passthrough_act \u001b[38;5;241m=\u001b[39m _is_passthrough_act(quant_injector)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/quant_proxy.py:89\u001b[0m, in \u001b[0;36mQuantProxyFromInjector.__init__\u001b[0;34m(self, quant_layer, quant_injector)\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;66;03m# Use a normal list and not a ModuleList since this is a pointer to parent modules\u001b[39;00m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtracked_module_list \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 89\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madd_tracked_module(quant_layer)\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdisable_quant \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/quant_proxy.py:131\u001b[0m, in \u001b[0;36mQuantProxyFromInjector.add_tracked_module\u001b[0;34m(self, module)\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtracked_module_list\u001b[38;5;241m.\u001b[39mappend(module)\n\u001b[1;32m 130\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mupdate_tracked_modules()\n\u001b[0;32m--> 131\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minit_tensor_quant()\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTrying to add None as a parent module.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/runtime_quant.py:102\u001b[0m, in \u001b[0;36mActQuantProxyFromInjector.init_tensor_quant\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minit_tensor_quant\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 102\u001b[0m tensor_quant \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquant_injector\u001b[38;5;241m.\u001b[39mtensor_quant\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mact_impl\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquant_injector:\n\u001b[1;32m 104\u001b[0m act_impl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquant_injector\u001b[38;5;241m.\u001b[39mact_impl\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/inject/__init__.py:129\u001b[0m, in \u001b[0;36m_ExtendedInjectorType.__getattr__\u001b[0;34m(cls, attrname)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 127\u001b[0m message \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m can not resolve attribute \u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 128\u001b[0m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, current_attr)\n\u001b[0;32m--> 129\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m DependencyError(message)\n\u001b[1;32m 131\u001b[0m marker, attribute, args, have_defaults \u001b[38;5;241m=\u001b[39m spec\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mset\u001b[39m(args)\u001b[38;5;241m.\u001b[39missubset(cached):\n", + "\u001b[0;31mDependencyError\u001b[0m: 'Int8ActPerTensorFloatMinMaxInit' can not resolve attribute 'max_val' while building 'scaling_init_impl'" ] } ], @@ -648,7 +661,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -664,25 +677,22 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ "out1_train = quant_hard_tanh(inp1)\n", "quant_hard_tanh.eval()\n", "out2_eval = quant_hard_tanh(inp2)\n", - "assert out1_train.scale.isclose(out2_eval.scale).item()" + "assert_with_message(out1_train.scale.isclose(out2_eval.scale).item())" ] }, { @@ -697,7 +707,7 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -711,9 +721,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.11.5" } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/notebooks/03_anatomy_of_a_quantizer.ipynb b/notebooks/03_anatomy_of_a_quantizer.ipynb index 2055a1714..9ac14b70f 100644 --- a/notebooks/03_anatomy_of_a_quantizer.ipynb +++ b/notebooks/03_anatomy_of_a_quantizer.ipynb @@ -3,7 +3,10 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } }, "source": [ "# Anatomy of a Quantizer\n", @@ -135,6 +138,11 @@ "import inspect\n", "from IPython.display import Markdown, display\n", "\n", + "# helpers\n", + "def assert_with_message(condition):\n", + " assert condition\n", + " print(condition)\n", + "\n", "def pretty_print_source(source):\n", " display(Markdown('```python\\n' + source + '\\n```'))" ] @@ -247,10 +255,10 @@ { "data": { "text/plain": [ - "(tensor([[ 0.1000, 0.1000, 0.1000, 0.1000],\n", - " [-0.1000, -0.1000, 0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, 0.1000, -0.1000],\n", - " [ 0.1000, -0.1000, 0.1000, -0.1000]], grad_fn=),\n", + "(tensor([[-0.1000, -0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, 0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, -0.1000, 0.1000, 0.1000],\n", + " [ 0.1000, 0.1000, 0.1000, -0.1000]], grad_fn=),\n", " tensor(0.1000, grad_fn=),\n", " tensor(0.),\n", " tensor(1.))" @@ -264,6 +272,9 @@ "source": [ "import torch\n", "\n", + "# set seed for notebook\n", + "torch.manual_seed(0)\n", + "\n", "manual_tensor_quant = BinaryQuant(scaling_impl=ParameterScaling(scaling_init=0.1))\n", "manual_tensor_quant(torch.randn(4, 4))" ] @@ -292,10 +303,10 @@ { "data": { "text/plain": [ - "(tensor([[-0.1000, 0.1000, -0.1000, 0.1000],\n", - " [ 0.1000, 0.1000, -0.1000, -0.1000],\n", - " [-0.1000, 0.1000, -0.1000, 0.1000],\n", - " [-0.1000, 0.1000, 0.1000, 0.1000]], grad_fn=),\n", + "(tensor([[-0.1000, -0.1000, 0.1000, 0.1000],\n", + " [ 0.1000, -0.1000, -0.1000, 0.1000],\n", + " [ 0.1000, -0.1000, -0.1000, 0.1000],\n", + " [ 0.1000, 0.1000, 0.1000, -0.1000]], grad_fn=),\n", " tensor(0.1000, grad_fn=),\n", " tensor(0.),\n", " tensor(1.))" @@ -342,10 +353,10 @@ { "data": { "text/plain": [ - "(tensor([[ 1., -1., 1., 1.],\n", - " [ 1., 1., -1., 1.],\n", - " [ 1., 1., 1., -1.],\n", - " [-1., 1., -1., -1.]], grad_fn=),\n", + "(tensor([[-1., 1., -1., 1.],\n", + " [ 1., 1., 1., 1.],\n", + " [-1., 1., -1., 1.],\n", + " [ 1., 1., -1., -1.]], grad_fn=),\n", " tensor(1., grad_fn=),\n", " tensor(0.),\n", " tensor(1.))" @@ -379,10 +390,10 @@ { "data": { "text/plain": [ - "(tensor([[ 0.1000, -0.1000, -0.1000, -0.1000],\n", - " [-0.1000, 0.1000, -0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, 0.1000, 0.1000],\n", - " [-0.1000, 0.1000, -0.1000, 0.1000]], grad_fn=),\n", + "(tensor([[-0.1000, 0.1000, 0.1000, 0.1000],\n", + " [-0.1000, -0.1000, 0.1000, -0.1000],\n", + " [-0.1000, 0.1000, 0.1000, 0.1000],\n", + " [-0.1000, -0.1000, 0.1000, -0.1000]], grad_fn=),\n", " tensor(0.1000, grad_fn=),\n", " tensor(0.),\n", " tensor(1.))" @@ -448,30 +459,30 @@ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.1000, 0.1000, -0.1000],\n", - " [-0.1000, 0.1000, -0.1000],\n", - " [ 0.1000, -0.1000, -0.1000]],\n", + "QuantTensor(value=tensor([[[[ 0.1000, -0.1000, 0.1000],\n", + " [ 0.1000, 0.1000, 0.1000],\n", + " [-0.1000, -0.1000, 0.1000]],\n", "\n", - " [[-0.1000, 0.1000, -0.1000],\n", + " [[ 0.1000, 0.1000, 0.1000],\n", " [ 0.1000, -0.1000, 0.1000],\n", " [-0.1000, -0.1000, 0.1000]],\n", "\n", - " [[ 0.1000, -0.1000, -0.1000],\n", - " [ 0.1000, 0.1000, -0.1000],\n", - " [-0.1000, -0.1000, 0.1000]]],\n", + " [[ 0.1000, -0.1000, 0.1000],\n", + " [-0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, -0.1000, 0.1000]]],\n", "\n", "\n", - " [[[ 0.1000, -0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, -0.1000],\n", - " [ 0.1000, -0.1000, 0.1000]],\n", + " [[[ 0.1000, 0.1000, 0.1000],\n", + " [-0.1000, -0.1000, 0.1000],\n", + " [ 0.1000, -0.1000, -0.1000]],\n", "\n", - " [[-0.1000, 0.1000, -0.1000],\n", - " [ 0.1000, 0.1000, 0.1000],\n", + " [[ 0.1000, 0.1000, -0.1000],\n", + " [-0.1000, -0.1000, -0.1000],\n", " [-0.1000, -0.1000, -0.1000]],\n", "\n", - " [[ 0.1000, 0.1000, -0.1000],\n", - " [-0.1000, 0.1000, -0.1000],\n", - " [ 0.1000, 0.1000, 0.1000]]]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=None, training_t=tensor(True))" + " [[ 0.1000, -0.1000, 0.1000],\n", + " [-0.1000, 0.1000, 0.1000],\n", + " [-0.1000, -0.1000, 0.1000]]]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=None, training_t=tensor(True))" ] }, "execution_count": 11, @@ -498,9 +509,17 @@ "cell_type": "code", "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ - "assert not quant_weight.is_valid" + "assert_with_message(not quant_weight.is_valid)" ] }, { @@ -519,29 +538,29 @@ "data": { "text/plain": [ "QuantTensor(value=tensor([[[[ 0.1000, 0.1000, -0.1000],\n", + " [ 0.1000, -0.1000, -0.1000],\n", + " [-0.1000, -0.1000, -0.1000]],\n", + "\n", + " [[-0.1000, -0.1000, 0.1000],\n", " [-0.1000, -0.1000, 0.1000],\n", - " [ 0.1000, 0.1000, -0.1000]],\n", + " [-0.1000, 0.1000, -0.1000]],\n", "\n", " [[ 0.1000, 0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, -0.1000]],\n", - "\n", - " [[-0.1000, 0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, -0.1000],\n", - " [-0.1000, -0.1000, -0.1000]]],\n", + " [ 0.1000, 0.1000, -0.1000],\n", + " [-0.1000, -0.1000, 0.1000]]],\n", "\n", "\n", - " [[[ 0.1000, 0.1000, 0.1000],\n", - " [ 0.1000, 0.1000, -0.1000],\n", - " [-0.1000, -0.1000, 0.1000]],\n", + " [[[-0.1000, -0.1000, -0.1000],\n", + " [-0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, 0.1000, -0.1000]],\n", "\n", - " [[-0.1000, -0.1000, 0.1000],\n", - " [-0.1000, 0.1000, 0.1000],\n", + " [[ 0.1000, -0.1000, -0.1000],\n", + " [-0.1000, -0.1000, -0.1000],\n", " [-0.1000, -0.1000, -0.1000]],\n", "\n", - " [[-0.1000, 0.1000, -0.1000],\n", - " [-0.1000, 0.1000, -0.1000],\n", - " [-0.1000, 0.1000, -0.1000]]]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 0.1000, -0.1000, 0.1000],\n", + " [-0.1000, -0.1000, 0.1000],\n", + " [-0.1000, 0.1000, 0.1000]]]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, "execution_count": 13, @@ -560,11 +579,19 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ - "assert signed_quant_weight.is_valid == True" + "assert_with_message(signed_quant_weight.is_valid)" ] }, { @@ -578,39 +605,39 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "QuantTensor(value=tensor([[[[-0.1000, -0.1000, 0.1000],\n", - " [-0.1000, -0.1000, -0.1000],\n", - " [ 0.1000, -0.1000, 0.1000]],\n", + " [-0.1000, 0.1000, -0.1000],\n", + " [ 0.1000, 0.1000, 0.1000]],\n", "\n", - " [[-0.1000, 0.1000, -0.1000],\n", + " [[ 0.1000, -0.1000, -0.1000],\n", " [-0.1000, 0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, -0.1000]],\n", + " [ 0.1000, -0.1000, 0.1000]],\n", "\n", - " [[-0.1000, 0.1000, -0.1000],\n", - " [-0.1000, -0.1000, 0.1000],\n", - " [-0.1000, -0.1000, -0.1000]]],\n", + " [[-0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, 0.1000, -0.1000]]],\n", "\n", "\n", - " [[[-0.1000, -0.1000, -0.1000],\n", - " [-0.1000, -0.1000, -0.1000],\n", - " [ 0.1000, 0.1000, -0.1000]],\n", + " [[[-0.1000, -0.1000, 0.1000],\n", + " [-0.1000, 0.1000, 0.1000],\n", + " [ 0.1000, -0.1000, -0.1000]],\n", "\n", - " [[-0.1000, -0.1000, 0.1000],\n", - " [-0.1000, 0.1000, -0.1000],\n", - " [ 0.1000, -0.1000, 0.1000]],\n", + " [[ 0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, -0.1000, -0.1000],\n", + " [ 0.1000, -0.1000, -0.1000]],\n", "\n", - " [[ 0.1000, 0.1000, -0.1000],\n", - " [ 0.1000, 0.1000, 0.1000],\n", - " [ 0.1000, -0.1000, 0.1000]]]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 0.1000, 0.1000, 0.1000],\n", + " [-0.1000, 0.1000, -0.1000],\n", + " [-0.1000, -0.1000, -0.1000]]]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 16, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -640,19 +667,27 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/_tensor.py:1362: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/c10/core/TensorImpl.h:1900.)\n", + " return super().rename(names)\n" + ] + }, { "data": { "text/plain": [ - "QuantTensor(value=tensor([[-0.1000, 0.1000, -0.1000, 0.1000],\n", + "QuantTensor(value=tensor([[-0.1000, 0.1000, -0.1000, -0.1000],\n", " [ 0.1000, 0.1000, 0.1000, 0.1000],\n", - " [-0.1000, 0.1000, 0.1000, 0.1000],\n", - " [-0.1000, -0.1000, 0.1000, -0.1000]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [-0.1000, -0.1000, 0.1000, -0.1000],\n", + " [-0.1000, 0.1000, -0.1000, 0.1000]], grad_fn=), scale=tensor(0.1000, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 17, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -678,19 +713,19 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[ 0.0010, 0.0010, 0.0010, -0.0010],\n", - " [ 0.0010, -0.0010, 0.0010, -0.0010],\n", - " [-0.0010, -0.0010, -0.0010, -0.0010],\n", - " [ 0.0010, 0.0010, 0.0010, 0.0010]], grad_fn=), scale=tensor(0.0010, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + "QuantTensor(value=tensor([[ 0.0010, -0.0010, -0.0010, 0.0010],\n", + " [ 0.0010, 0.0010, 0.0010, 0.0010],\n", + " [ 0.0010, -0.0010, 0.0010, 0.0010],\n", + " [ 0.0010, -0.0010, -0.0010, -0.0010]], grad_fn=), scale=tensor(0.0010, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 18, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -716,7 +751,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -740,7 +775,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": { "scrolled": true }, @@ -748,33 +783,33 @@ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[-0.1876, -0.1876, -0.1876],\n", - " [ 0.1876, 0.1876, 0.1876],\n", - " [-0.1876, -0.1876, 0.1876]],\n", + "QuantTensor(value=tensor([[[[ 0.1820, -0.1820, -0.1820],\n", + " [ 0.1820, 0.1820, 0.1820],\n", + " [-0.1820, -0.1820, -0.1820]],\n", "\n", - " [[-0.1876, -0.1876, 0.1876],\n", - " [ 0.1876, 0.1876, -0.1876],\n", - " [-0.1876, 0.1876, 0.1876]],\n", + " [[ 0.1820, -0.1820, -0.1820],\n", + " [ 0.1820, -0.1820, -0.1820],\n", + " [ 0.1820, 0.1820, -0.1820]],\n", "\n", - " [[-0.1876, -0.1876, -0.1876],\n", - " [ 0.1876, 0.1876, 0.1876],\n", - " [-0.1876, 0.1876, -0.1876]]],\n", + " [[-0.1820, 0.1820, 0.1820],\n", + " [ 0.1820, -0.1820, 0.1820],\n", + " [-0.1820, -0.1820, -0.1820]]],\n", "\n", "\n", - " [[[-0.1876, -0.1876, -0.1876],\n", - " [ 0.1876, 0.1876, -0.1876],\n", - " [ 0.1876, -0.1876, -0.1876]],\n", + " [[[ 0.1820, 0.1820, -0.1820],\n", + " [-0.1820, -0.1820, 0.1820],\n", + " [-0.1820, 0.1820, -0.1820]],\n", "\n", - " [[-0.1876, 0.1876, -0.1876],\n", - " [ 0.1876, -0.1876, -0.1876],\n", - " [-0.1876, -0.1876, 0.1876]],\n", + " [[-0.1820, 0.1820, -0.1820],\n", + " [ 0.1820, 0.1820, -0.1820],\n", + " [ 0.1820, 0.1820, 0.1820]],\n", "\n", - " [[-0.1876, 0.1876, 0.1876],\n", - " [ 0.1876, -0.1876, 0.1876],\n", - " [-0.1876, -0.1876, -0.1876]]]], grad_fn=), scale=tensor(0.1876, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [[-0.1820, -0.1820, -0.1820],\n", + " [ 0.1820, -0.1820, 0.1820],\n", + " [ 0.1820, -0.1820, 0.1820]]]], grad_fn=), scale=tensor(0.1820, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 20, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -793,22 +828,19 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ - "(param_from_max_quant_conv.quant_weight_scale() == param_from_max_quant_conv.weight.abs().max()).item()" + "assert_with_message((param_from_max_quant_conv.quant_weight_scale() == param_from_max_quant_conv.weight.abs().max()).item())" ] }, { @@ -820,16 +852,16 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor(0.1897, grad_fn=)" + "tensor(0.1924, grad_fn=)" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -850,7 +882,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "metadata": { "tags": [ "raises-exception" @@ -862,11 +894,11 @@ "evalue": "Error(s) in loading state_dict for QuantConv2d:\n\tMissing key(s) in state_dict: \"weight_quant.tensor_quant.scaling_impl.value\". ", "output_type": "error", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mparam_from_max_quant_conv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfloat_conv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\pytorch\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[1;34m(self, state_dict, strict)\u001b[0m\n\u001b[0;32m 1405\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror_msgs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1406\u001b[0m raise RuntimeError('Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n\u001b[1;32m-> 1407\u001b[1;33m self.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n\u001b[0m\u001b[0;32m 1408\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0m_IncompatibleKeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmissing_keys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munexpected_keys\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1409\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for QuantConv2d:\n\tMissing key(s) in state_dict: \"weight_quant.tensor_quant.scaling_impl.value\". " + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[22], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m param_from_max_quant_conv\u001b[38;5;241m.\u001b[39mload_state_dict(float_conv\u001b[38;5;241m.\u001b[39mstate_dict())\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:2152\u001b[0m, in \u001b[0;36mModule.load_state_dict\u001b[0;34m(self, state_dict, strict, assign)\u001b[0m\n\u001b[1;32m 2147\u001b[0m error_msgs\u001b[38;5;241m.\u001b[39minsert(\n\u001b[1;32m 2148\u001b[0m \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMissing key(s) in state_dict: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 2149\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m missing_keys)))\n\u001b[1;32m 2151\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_msgs) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 2152\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mError(s) in loading state_dict for \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 2153\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(error_msgs)))\n\u001b[1;32m 2154\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _IncompatibleKeys(missing_keys, unexpected_keys)\n", + "\u001b[0;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for QuantConv2d:\n\tMissing key(s) in state_dict: \"weight_quant.tensor_quant.scaling_impl.value\". " ] } ], @@ -916,39 +948,39 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.1897, -0.1897, 0.1897],\n", - " [-0.1897, 0.1897, -0.1897],\n", - " [-0.1897, 0.1897, -0.1897]],\n", + "QuantTensor(value=tensor([[[[ 0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, 0.1924],\n", + " [ 0.1924, 0.1924, 0.1924]],\n", "\n", - " [[-0.1897, 0.1897, 0.1897],\n", - " [ 0.1897, -0.1897, -0.1897],\n", - " [ 0.1897, -0.1897, 0.1897]],\n", + " [[-0.1924, -0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, 0.1924]],\n", "\n", - " [[-0.1897, 0.1897, -0.1897],\n", - " [-0.1897, 0.1897, 0.1897],\n", - " [-0.1897, 0.1897, 0.1897]]],\n", + " [[ 0.1924, 0.1924, -0.1924],\n", + " [-0.1924, 0.1924, 0.1924],\n", + " [-0.1924, 0.1924, 0.1924]]],\n", "\n", "\n", - " [[[ 0.1897, 0.1897, 0.1897],\n", - " [-0.1897, 0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, -0.1897]],\n", + " [[[ 0.1924, -0.1924, 0.1924],\n", + " [-0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, 0.1924]],\n", "\n", - " [[ 0.1897, -0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, 0.1897]],\n", + " [[-0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, -0.1924, -0.1924],\n", + " [-0.1924, 0.1924, 0.1924]],\n", "\n", - " [[-0.1897, 0.1897, -0.1897],\n", - " [-0.1897, 0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, 0.1897]]]], grad_fn=), scale=tensor(0.1897, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [[ 0.1924, 0.1924, -0.1924],\n", + " [-0.1924, -0.1924, -0.1924],\n", + " [ 0.1924, -0.1924, -0.1924]]]], grad_fn=), scale=tensor(0.1924, grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 25, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -979,25 +1011,22 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ "quant_conv1 = QuantConv2d(3, 2, (3, 3), weight_quant=MySignedBinaryWeightQuantizer)\n", "quant_conv2 = QuantConv2d(3, 2, (3, 3), weight_quant=MySignedBinaryWeightQuantizer)\n", "\n", - "quant_conv1.weight_quant is quant_conv2.weight_quant" + "assert_with_message(quant_conv1.weight_quant is not quant_conv2.weight_quant)" ] }, { @@ -1015,21 +1044,18 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ "quant_conv1 = QuantConv2d(3, 2, (3, 3), weight_quant=MySignedBinaryWeightQuantizer)\n", "quant_conv2 = QuantConv2d(3, 2, (3, 3), weight_quant=quant_conv1.weight_quant)\n", "\n", - "assert quant_conv1.weight_quant is quant_conv2.weight_quant" + "assert_with_message(quant_conv1.weight_quant is quant_conv2.weight_quant)" ] }, { @@ -1038,19 +1064,15 @@ "metadata": {}, "outputs": [ { - "ename": "AssertionError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_58415/1066539094.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mquant_conv1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquant_weight_scale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mquant_conv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquant_weight_scale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mAssertionError\u001b[0m: " + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" ] } ], "source": [ - "assert (quant_conv1.quant_weight_scale() == quant_conv2.quant_weight_scale()).item()" + "assert_with_message((quant_conv1.quant_weight_scale() == quant_conv2.quant_weight_scale()).item())" ] }, { @@ -1067,14 +1089,11 @@ "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] } ], "source": [ @@ -1092,16 +1111,24 @@ "quant_conv2 = QuantConv2d(3, 2, (3, 3), weight_quant=quant_conv1.weight_quant)\n", "new_quant_conv1_scale = quant_conv1.quant_weight_scale()\n", "\n", - "assert not (old_quant_conv1_scale == new_quant_conv1_scale).item()" + "assert_with_message(not (old_quant_conv1_scale == new_quant_conv1_scale).item())" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ - "assert (new_quant_conv1_scale == quant_conv2.quant_weight_scale()).item()" + "assert_with_message((new_quant_conv1_scale == quant_conv2.quant_weight_scale()).item())" ] }, { @@ -1140,14 +1167,22 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ "quant_conv_w_init = QuantConv2d(3, 2, (3, 3), weight_quant=ParamFromMaxWeightQuantizer)\n", "torch.nn.init.uniform_(quant_conv_w_init.weight)\n", "\n", - "assert not (quant_conv_w_init.weight.abs().max() == quant_conv_w_init.quant_weight_scale()).item()" + "assert_with_message(not (quant_conv_w_init.weight.abs().max() == quant_conv_w_init.quant_weight_scale()).item())" ] }, { @@ -1159,13 +1194,21 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ "quant_conv_w_init.weight_quant.init_tensor_quant()\n", "\n", - "assert (quant_conv_w_init.weight.abs().max() == quant_conv_w_init.quant_weight_scale()).item()" + "assert_with_message((quant_conv_w_init.weight.abs().max() == quant_conv_w_init.quant_weight_scale()).item())" ] }, { @@ -1260,42 +1303,42 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[-0.1842, 0.1842, -0.1842],\n", - " [-0.1842, -0.1842, 0.1842],\n", - " [-0.1842, -0.1842, 0.1842]],\n", + "QuantTensor(value=tensor([[[[ 0.1612, -0.1612, -0.1612],\n", + " [-0.1612, -0.1612, -0.1612],\n", + " [ 0.1612, 0.1612, 0.1612]],\n", "\n", - " [[-0.1842, -0.1842, 0.1842],\n", - " [ 0.1842, -0.1842, 0.1842],\n", - " [ 0.1842, 0.1842, -0.1842]],\n", + " [[-0.1612, 0.1612, -0.1612],\n", + " [-0.1612, 0.1612, 0.1612],\n", + " [-0.1612, -0.1612, 0.1612]],\n", "\n", - " [[-0.1842, -0.1842, 0.1842],\n", - " [ 0.1842, 0.1842, 0.1842],\n", - " [-0.1842, 0.1842, -0.1842]]],\n", + " [[-0.1612, 0.1612, 0.1612],\n", + " [ 0.1612, 0.1612, -0.1612],\n", + " [ 0.1612, 0.1612, 0.1612]]],\n", "\n", "\n", - " [[[ 0.1838, 0.1838, 0.1838],\n", - " [-0.1838, -0.1838, -0.1838],\n", - " [ 0.1838, 0.1838, -0.1838]],\n", + " [[[ 0.1924, 0.1924, 0.1924],\n", + " [-0.1924, -0.1924, 0.1924],\n", + " [-0.1924, 0.1924, -0.1924]],\n", "\n", - " [[ 0.1838, -0.1838, 0.1838],\n", - " [ 0.1838, 0.1838, 0.1838],\n", - " [-0.1838, 0.1838, -0.1838]],\n", + " [[ 0.1924, -0.1924, 0.1924],\n", + " [ 0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, -0.1924, -0.1924]],\n", "\n", - " [[-0.1838, 0.1838, -0.1838],\n", - " [ 0.1838, -0.1838, -0.1838],\n", - " [ 0.1838, -0.1838, 0.1838]]]], grad_fn=), scale=tensor([[[[0.1842]]],\n", + " [[-0.1924, -0.1924, 0.1924],\n", + " [ 0.1924, -0.1924, -0.1924],\n", + " [ 0.1924, -0.1924, 0.1924]]]], grad_fn=), scale=tensor([[[[0.1612]]],\n", "\n", "\n", - " [[[0.1838]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [[[0.1924]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 35, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -1318,42 +1361,42 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[[[ 0.1875, -0.1875, 0.1875],\n", - " [-0.1875, 0.1875, -0.1875],\n", - " [-0.1875, 0.1875, -0.1875]],\n", + "QuantTensor(value=tensor([[[[ 0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, 0.1924],\n", + " [ 0.1924, 0.1924, 0.1924]],\n", "\n", - " [[-0.1875, 0.1875, 0.1875],\n", - " [ 0.1875, -0.1875, -0.1875],\n", - " [ 0.1875, -0.1875, 0.1875]],\n", + " [[-0.1924, -0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, -0.1924],\n", + " [ 0.1924, 0.1924, 0.1924]],\n", "\n", - " [[-0.1875, 0.1875, -0.1875],\n", - " [-0.1875, 0.1875, 0.1875],\n", - " [-0.1875, 0.1875, 0.1875]]],\n", + " [[ 0.1924, 0.1924, -0.1924],\n", + " [-0.1924, 0.1924, 0.1924],\n", + " [-0.1924, 0.1924, 0.1924]]],\n", "\n", "\n", - " [[[ 0.1897, 0.1897, 0.1897],\n", - " [-0.1897, 0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, -0.1897]],\n", + " [[[ 0.1899, -0.1899, 0.1899],\n", + " [-0.1899, 0.1899, -0.1899],\n", + " [ 0.1899, 0.1899, 0.1899]],\n", "\n", - " [[ 0.1897, -0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, 0.1897]],\n", + " [[-0.1899, 0.1899, -0.1899],\n", + " [ 0.1899, -0.1899, -0.1899],\n", + " [-0.1899, 0.1899, 0.1899]],\n", "\n", - " [[-0.1897, 0.1897, -0.1897],\n", - " [-0.1897, 0.1897, -0.1897],\n", - " [ 0.1897, 0.1897, 0.1897]]]], grad_fn=), scale=tensor([[[[0.1875]]],\n", + " [[ 0.1899, 0.1899, -0.1899],\n", + " [-0.1899, -0.1899, -0.1899],\n", + " [ 0.1899, -0.1899, -0.1899]]]], grad_fn=), scale=tensor([[[[0.1924]]],\n", "\n", "\n", - " [[[0.1897]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" + " [[[0.1899]]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 36, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1374,19 +1417,19 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "tensor([[-0.0100, -0.0100, 0.0100, -0.0100],\n", - " [-0.0100, -0.0100, -0.0100, 0.0100],\n", - " [-0.0100, 0.0100, 0.0100, 0.0100],\n", - " [-0.0100, 0.0100, 0.0100, 0.0100]], grad_fn=)" + "tensor([[ 0.0100, 0.0100, -0.0100, 0.0100],\n", + " [ 0.0100, 0.0100, -0.0100, 0.0100],\n", + " [-0.0100, 0.0100, -0.0100, -0.0100],\n", + " [ 0.0100, -0.0100, -0.0100, -0.0100]], grad_fn=)" ] }, - "execution_count": 37, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -1421,21 +1464,21 @@ "evalue": "'AdvancedActQuantizer' can not resolve attribute 'per_channel_broadcastable_shape'", "output_type": "error", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mDependencyError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m quant_identity = QuantIdentity(\n\u001b[1;32m----> 4\u001b[1;33m act_quant=AdvancedActQuantizer, is_clamped=True, scaling_per_output_channel=True)\n\u001b[0m", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\quant_activation.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, act_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[0;32m 134\u001b[0m \u001b[0mact_quant\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mact_quant\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 135\u001b[0m \u001b[0mreturn_quant_tensor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_quant_tensor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 136\u001b[1;33m **kwargs)\n\u001b[0m\u001b[0;32m 137\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\quant_layer.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, act_impl, passthrough_act, input_quant, act_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[0;32m 77\u001b[0m \u001b[0mpassthrough_act\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[0mact_quant\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 79\u001b[1;33m **kwargs)\n\u001b[0m\u001b[0;32m 80\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 81\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\mixin\\act.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, act_impl, passthrough_act, act_quant, **kwargs)\u001b[0m\n\u001b[0;32m 157\u001b[0m \u001b[0mproxy_prefix\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'act_'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 158\u001b[0m \u001b[0mkwargs_prefix\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 159\u001b[1;33m **kwargs)\n\u001b[0m\u001b[0;32m 160\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 161\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\nn\\mixin\\base.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, quant, proxy_protocol, none_quant_injector, proxy_prefix, kwargs_prefix, **kwargs)\u001b[0m\n\u001b[0;32m 98\u001b[0m \u001b[0mquant_injector\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 99\u001b[0m \u001b[0mquant_injector\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mfilter_kwargs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwargs_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 100\u001b[1;33m \u001b[0mquant\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproxy_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 101\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 102\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mproxy_protocol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\runtime_quant.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, quant_layer, quant_injector)\u001b[0m\n\u001b[0;32m 108\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 109\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 110\u001b[1;33m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mActQuantProxyFromInjector\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_layer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 111\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_passthrough_act\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_is_passthrough_act\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_injector\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 112\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\quant_proxy.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, quant_layer, quant_injector, export_mode, export_handler)\u001b[0m\n\u001b[0;32m 74\u001b[0m \u001b[1;31m# Use a normal list and not a ModuleList since this is a pointer to parent modules\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 75\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtracked_module_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_tracked_module\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_layer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 77\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_handler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexport_handler\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 78\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_mode\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mexport_mode\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\quant_proxy.py\u001b[0m in \u001b[0;36madd_tracked_module\u001b[1;34m(self, module)\u001b[0m\n\u001b[0;32m 130\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtracked_module_list\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate_tracked_modules\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 132\u001b[1;33m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minit_tensor_quant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 133\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 134\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Trying to add None as a parent module.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\brevitas_fx\\src\\brevitas\\proxy\\runtime_quant.py\u001b[0m in \u001b[0;36minit_tensor_quant\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 121\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0minit_tensor_quant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 122\u001b[1;33m \u001b[0mtensor_quant\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtensor_quant\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 123\u001b[0m \u001b[0mact_impl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquant_injector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mact_impl\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 124\u001b[0m \u001b[0mis_act_enabled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_is_act_enabled\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mact_impl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_quant\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", - "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\pytorch\\lib\\site-packages\\_dependencies\\this.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, __self__)\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mkind\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\".\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 50\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msymbol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 52\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mDependencyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 53\u001b[0m message = (\n", - " \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n", - "\u001b[1;31mDependencyError\u001b[0m: 'AdvancedActQuantizer' can not resolve attribute 'per_channel_broadcastable_shape'" + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mDependencyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[36], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mbrevitas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m QuantIdentity\n\u001b[0;32m----> 3\u001b[0m quant_identity \u001b[38;5;241m=\u001b[39m QuantIdentity(\n\u001b[1;32m 4\u001b[0m act_quant\u001b[38;5;241m=\u001b[39mAdvancedActQuantizer, is_clamped\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, scaling_per_output_channel\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_activation.py:113\u001b[0m, in \u001b[0;36mQuantIdentity.__init__\u001b[0;34m(self, act_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 110\u001b[0m act_quant: Optional[ActQuantType] \u001b[38;5;241m=\u001b[39m Int8ActPerTensorFloat,\n\u001b[1;32m 111\u001b[0m return_quant_tensor: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 112\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 113\u001b[0m QuantNLAL\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 115\u001b[0m input_quant\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 116\u001b[0m act_impl\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 117\u001b[0m passthrough_act\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 118\u001b[0m act_quant\u001b[38;5;241m=\u001b[39mact_quant,\n\u001b[1;32m 119\u001b[0m return_quant_tensor\u001b[38;5;241m=\u001b[39mreturn_quant_tensor,\n\u001b[1;32m 120\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_layer.py:36\u001b[0m, in \u001b[0;36mQuantNonLinearActLayer.__init__\u001b[0;34m(self, act_impl, passthrough_act, input_quant, act_quant, return_quant_tensor, **kwargs)\u001b[0m\n\u001b[1;32m 34\u001b[0m QuantLayerMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, return_quant_tensor)\n\u001b[1;32m 35\u001b[0m QuantInputMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, input_quant, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 36\u001b[0m QuantNonLinearActMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, act_impl, passthrough_act, act_quant, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/mixin/act.py:118\u001b[0m, in \u001b[0;36mQuantNonLinearActMixin.__init__\u001b[0;34m(self, act_impl, passthrough_act, act_quant, act_proxy_prefix, act_kwargs_prefix, **kwargs)\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 108\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 109\u001b[0m act_impl: Optional[Type[Module]],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 113\u001b[0m act_kwargs_prefix\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m 114\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 115\u001b[0m prefixed_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 116\u001b[0m act_kwargs_prefix \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mact_impl\u001b[39m\u001b[38;5;124m'\u001b[39m: act_impl,\n\u001b[1;32m 117\u001b[0m act_kwargs_prefix \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpassthrough_act\u001b[39m\u001b[38;5;124m'\u001b[39m: passthrough_act}\n\u001b[0;32m--> 118\u001b[0m QuantProxyMixin\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 119\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 120\u001b[0m quant\u001b[38;5;241m=\u001b[39mact_quant,\n\u001b[1;32m 121\u001b[0m proxy_prefix\u001b[38;5;241m=\u001b[39mact_proxy_prefix,\n\u001b[1;32m 122\u001b[0m kwargs_prefix\u001b[38;5;241m=\u001b[39mact_kwargs_prefix,\n\u001b[1;32m 123\u001b[0m proxy_protocol\u001b[38;5;241m=\u001b[39mActQuantProxyProtocol,\n\u001b[1;32m 124\u001b[0m none_quant_injector\u001b[38;5;241m=\u001b[39mNoneActQuant,\n\u001b[1;32m 125\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mprefixed_kwargs,\n\u001b[1;32m 126\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:70\u001b[0m, in \u001b[0;36mQuantProxyMixin.__init__\u001b[0;34m(self, quant, proxy_protocol, none_quant_injector, proxy_prefix, kwargs_prefix, **kwargs)\u001b[0m\n\u001b[1;32m 68\u001b[0m quant_injector \u001b[38;5;241m=\u001b[39m quant\n\u001b[1;32m 69\u001b[0m quant_injector \u001b[38;5;241m=\u001b[39m quant_injector\u001b[38;5;241m.\u001b[39mlet(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mfilter_kwargs(kwargs_prefix, kwargs))\n\u001b[0;32m---> 70\u001b[0m quant \u001b[38;5;241m=\u001b[39m quant_injector\u001b[38;5;241m.\u001b[39mproxy_class(\u001b[38;5;28mself\u001b[39m, quant_injector)\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(quant, proxy_protocol):\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/runtime_quant.py:89\u001b[0m, in \u001b[0;36mActQuantProxyFromInjector.__init__\u001b[0;34m(self, quant_layer, quant_injector)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, quant_layer, quant_injector):\n\u001b[0;32m---> 89\u001b[0m QuantProxyFromInjector\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, quant_layer, quant_injector)\n\u001b[1;32m 90\u001b[0m ActQuantProxyProtocol\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 91\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_passthrough_act \u001b[38;5;241m=\u001b[39m _is_passthrough_act(quant_injector)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/quant_proxy.py:89\u001b[0m, in \u001b[0;36mQuantProxyFromInjector.__init__\u001b[0;34m(self, quant_layer, quant_injector)\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;66;03m# Use a normal list and not a ModuleList since this is a pointer to parent modules\u001b[39;00m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtracked_module_list \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 89\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madd_tracked_module(quant_layer)\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdisable_quant \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/quant_proxy.py:131\u001b[0m, in \u001b[0;36mQuantProxyFromInjector.add_tracked_module\u001b[0;34m(self, module)\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtracked_module_list\u001b[38;5;241m.\u001b[39mappend(module)\n\u001b[1;32m 130\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mupdate_tracked_modules()\n\u001b[0;32m--> 131\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minit_tensor_quant()\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTrying to add None as a parent module.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/runtime_quant.py:102\u001b[0m, in \u001b[0;36mActQuantProxyFromInjector.init_tensor_quant\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minit_tensor_quant\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 102\u001b[0m tensor_quant \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquant_injector\u001b[38;5;241m.\u001b[39mtensor_quant\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mact_impl\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquant_injector:\n\u001b[1;32m 104\u001b[0m act_impl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquant_injector\u001b[38;5;241m.\u001b[39mact_impl\n", + " \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/_dependencies/this.py:51\u001b[0m, in \u001b[0;36m_ThisSpec.__call__\u001b[0;34m(self, __self__)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m kind \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 51\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(result, symbol)\n\u001b[1;32m 52\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m DependencyError:\n\u001b[1;32m 53\u001b[0m message \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 54\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou tried to shift this more times than Injector has levels\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 55\u001b[0m )\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/inject/__init__.py:129\u001b[0m, in \u001b[0;36m_ExtendedInjectorType.__getattr__\u001b[0;34m(cls, attrname)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 127\u001b[0m message \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m can not resolve attribute \u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 128\u001b[0m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, current_attr)\n\u001b[0;32m--> 129\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m DependencyError(message)\n\u001b[1;32m 131\u001b[0m marker, attribute, args, have_defaults \u001b[38;5;241m=\u001b[39m spec\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mset\u001b[39m(args)\u001b[38;5;241m.\u001b[39missubset(cached):\n", + "\u001b[0;31mDependencyError\u001b[0m: 'AdvancedActQuantizer' can not resolve attribute 'per_channel_broadcastable_shape'" ] } ], @@ -1455,22 +1498,22 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 37, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "QuantTensor(value=tensor([[-0.0100, 0.0100, -0.0100, -0.0100],\n", - " [-0.0100, 0.0100, -0.0100, -0.0100],\n", - " [ 0.0100, -0.0100, 0.0100, -0.0100],\n", - " [ 0.0100, -0.0100, -0.0100, -0.0100]], grad_fn=), scale=tensor([[0.0100],\n", + "QuantTensor(value=tensor([[ 0.0100, 0.0100, 0.0100, -0.0100],\n", + " [ 0.0100, -0.0100, -0.0100, 0.0100],\n", + " [-0.0100, -0.0100, -0.0100, 0.0100],\n", + " [ 0.0100, 0.0100, 0.0100, 0.0100]], grad_fn=), scale=tensor([[0.0100],\n", " [0.0100],\n", " [0.0100],\n", " [0.0100]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))" ] }, - "execution_count": 39, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -1492,7 +1535,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1506,9 +1549,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.11.5" } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/notebooks/Brevitas_TVMCon2021.ipynb b/notebooks/Brevitas_TVMCon2021.ipynb index efd9421f0..998d4b228 100644 --- a/notebooks/Brevitas_TVMCon2021.ipynb +++ b/notebooks/Brevitas_TVMCon2021.ipynb @@ -39,14 +39,16 @@ " self,\n", " in_features: int,\n", " out_features: int,\n", - " bias: bool,\n", + " bias: Optional[bool] = True,\n", " weight_quant: Optional[WeightQuantType] = Int8WeightPerTensorFloat,\n", " bias_quant: Optional[BiasQuantType] = None,\n", " input_quant: Optional[ActQuantType] = None,\n", " output_quant: Optional[ActQuantType] = None,\n", " return_quant_tensor: bool = False,\n", + " device: Optional[torch.device] = None,\n", + " dtype: Optional[torch.dtype] = None,\n", " **kwargs) -> None:\n", - " Linear.__init__(self, in_features, out_features, bias)\n", + " Linear.__init__(self, in_features, out_features, bias, device=device, dtype=dtype)\n", " QuantWBIOL.__init__(\n", " self,\n", " weight_quant=weight_quant,\n", @@ -161,7 +163,7 @@ " tensor([[ -1, 83],\n", " [-127, -114],\n", " [ -59, 41],\n", - " [ -3, 122]], dtype=torch.int32)\n" + " [ -3, 122]], dtype=torch.int8)\n" ] } ], @@ -194,7 +196,15 @@ "Float output:\n", " tensor([[-0.9036, -0.4586, 0.3096, -0.6472],\n", " [ 1.2058, 0.6525, -0.3723, 0.8677],\n", - " [ 1.3873, 0.2801, -0.9009, 0.9507]], grad_fn=)\n" + " [ 1.3873, 0.2801, -0.9009, 0.9507]], grad_fn=)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/_tensor.py:1362: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/c10/core/TensorImpl.h:1900.)\n", + " return super().rename(names)\n" ] } ], @@ -372,7 +382,7 @@ "Quant output:\n", " tensor([[-0.9109, -0.4609, 0.3135, -0.6523],\n", " [ 1.2089, 0.6524, -0.3752, 0.8697],\n", - " [ 1.3893, 0.2816, -0.9011, 0.9521]], grad_fn=)\n" + " [ 1.3893, 0.2816, -0.9011, 0.9521]], grad_fn=)\n" ] } ], @@ -409,7 +419,7 @@ "Quant output:\n", " QuantTensor(value=tensor([[-0.9109, -0.4609, 0.3135, -0.6523],\n", " [ 1.2089, 0.6524, -0.3752, 0.8697],\n", - " [ 1.3893, 0.2816, -0.9011, 0.9521]], grad_fn=), scale=tensor([[9.0542e-05]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))\n" + " [ 1.3893, 0.2816, -0.9011, 0.9521]], grad_fn=), scale=tensor([[9.0542e-05]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))\n" ] } ], @@ -457,7 +467,7 @@ "Quant output:\n", " QuantTensor(value=tensor([[-0.9109, -0.4609, 0.3135, -0.6523],\n", " [ 1.2089, 0.6524, -0.3752, 0.8697],\n", - " [ 1.3893, 0.2816, -0.9011, 0.9521]], grad_fn=), scale=tensor([[9.0542e-05]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))\n" + " [ 1.3893, 0.2816, -0.9011, 0.9521]], grad_fn=), scale=tensor([[9.0542e-05]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))\n" ] } ], @@ -614,15 +624,17 @@ "evalue": "Input scale required", "output_type": "error", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32mC:\\Users\\ALESSA~1\\AppData\\Local\\Temp/ipykernel_18920/2660651517.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 6\u001b[0m \u001b[0mquant_linear\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mQuantLinear\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbias_quant\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mInt16Bias\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_quant_tensor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mquant_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mquant_linear\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfloat_input\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m~\\miniconda3\\envs\\pt190\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1051\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1052\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\users\\alessandro\\documenti\\brevitas_tvmcon\\src\\brevitas\\nn\\quant_linear.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m 96\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 97\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQuantTensor\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mUnion\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mQuantTensor\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 98\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 99\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 100\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0minner_forward_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_weight\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquant_bias\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\users\\alessandro\\documenti\\brevitas_tvmcon\\src\\brevitas\\nn\\quant_layer.py\u001b[0m in \u001b[0;36mforward_impl\u001b[1;34m(self, inp)\u001b[0m\n\u001b[0;32m 355\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 356\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbias\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 357\u001b[1;33m \u001b[0mquant_bias\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbias_quant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_scale\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_bit_width\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 358\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcache_inference_quant_bias\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 359\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cached_bias\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_CachedIO\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mquant_bias\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetadata_only\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\miniconda3\\envs\\pt190\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[0;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[1;32m-> 1051\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 1052\u001b[0m \u001b[1;31m# Do not call functions when jit is used\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32mc:\\users\\alessandro\\documenti\\brevitas_tvmcon\\src\\brevitas\\proxy\\parameter_quant.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x, input_scale, input_bit_width)\u001b[0m\n\u001b[0;32m 194\u001b[0m \u001b[0mimpl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_handler\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexport_mode\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtensor_quant\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 195\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequires_input_scale\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0minput_scale\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 196\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Input scale required\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 197\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequires_input_bit_width\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0minput_bit_width\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 198\u001b[0m \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Input bit-width required\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mRuntimeError\u001b[0m: Input scale required" + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[13], line 8\u001b[0m\n\u001b[1;32m 5\u001b[0m float_input \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mrandn(\u001b[38;5;241m3\u001b[39m, \u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m 6\u001b[0m quant_linear \u001b[38;5;241m=\u001b[39m QuantLinear(\u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m4\u001b[39m, bias\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, bias_quant\u001b[38;5;241m=\u001b[39mInt16Bias, return_quant_tensor\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 8\u001b[0m quant_output \u001b[38;5;241m=\u001b[39m quant_linear(float_input)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_linear.py:66\u001b[0m, in \u001b[0;36mQuantLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Union[Tensor, QuantTensor]) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[Tensor, QuantTensor]:\n\u001b[0;32m---> 66\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_impl(\u001b[38;5;28minput\u001b[39m)\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/nn/quant_layer.py:326\u001b[0m, in \u001b[0;36mQuantWeightBiasInputOutputLayer.forward_impl\u001b[0;34m(self, inp)\u001b[0m\n\u001b[1;32m 323\u001b[0m output_signed \u001b[38;5;241m=\u001b[39m inp\u001b[38;5;241m.\u001b[39msigned \u001b[38;5;129;01mor\u001b[39;00m quant_weight\u001b[38;5;241m.\u001b[39msigned\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 326\u001b[0m quant_bias \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias_quant(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias, output_scale, output_bit_width)\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcache_inference_quant_bias:\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cached_bias \u001b[38;5;241m=\u001b[39m _CachedIO(quant_bias\u001b[38;5;241m.\u001b[39mdetach(), metadata_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/scratch/fabian/brevitas/src/brevitas/proxy/parameter_quant.py:206\u001b[0m, in \u001b[0;36mBiasQuantProxyFromInjector.forward\u001b[0;34m(self, x, input_scale, input_bit_width)\u001b[0m\n\u001b[1;32m 204\u001b[0m impl \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexport_handler \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexport_mode \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtensor_quant\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_input_scale \u001b[38;5;129;01mand\u001b[39;00m input_scale \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInput scale required\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 207\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequires_input_bit_width \u001b[38;5;129;01mand\u001b[39;00m input_bit_width \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInput bit-width required\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mRuntimeError\u001b[0m: Input scale required" ] } ], @@ -654,7 +666,7 @@ "text/plain": [ "QuantTensor(value=tensor([[-0.6541, 0.1263, 0.1680, -0.1231],\n", " [ 1.4658, 1.2395, -0.5207, 1.3989],\n", - " [ 1.6461, 0.8687, -1.0466, 1.4813]], grad_fn=), scale=tensor([[9.0542e-05]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(18.), signed_t=tensor(True), training_t=tensor(True))" + " [ 1.6461, 0.8687, -1.0466, 1.4813]], grad_fn=), scale=tensor([[9.0542e-05]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(18.), signed_t=tensor(True), training_t=tensor(True))" ] }, "execution_count": 14, @@ -796,14 +808,6 @@ " [[ 1.2666, 2.0084],\n", " [ 0.6152, -0.8323]]], grad_fn=), scale=tensor(0.0181, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "C:\\Users\\Alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages\\torch\\nn\\functional.py:652: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at ..\\c10/core/TensorImpl.h:1156.)\n", - " return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode)\n" - ] } ], "source": [ @@ -855,7 +859,15 @@ " [ 0.1614, 0.7006, -0.1438, -0.1081]],\n", "\n", " [[ 0.7272, 0.8529, 0.9646, 0.0542],\n", - " [ 0.5478, -0.3937, -0.6817, -0.9807]]], grad_fn=)\n" + " [ 0.5478, -0.3937, -0.6817, -0.9807]]], grad_fn=)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_61612/661358273.py:7: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/torch/csrc/utils/python_arg_parser.cpp:368.)\n", + " quant_output = torch.tanh(quant_input)\n" ] } ], @@ -902,6 +914,16 @@ " [-2.0447, 0.5751, -0.7188, -0.3994],\n", " [-1.0863, -1.4057, -0.5910, 0.1757]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(False))\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_61612/3932472163.py:8: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/torch/csrc/utils/python_arg_parser.cpp:368.)\n", + " train_mode_cat = torch.cat([quant_identity(float_inp1), quant_identity(float_inp2)], dim=1)\n", + "/tmp/ipykernel_61612/3932472163.py:14: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/torch/csrc/utils/python_arg_parser.cpp:368.)\n", + " eval_mode_cat = torch.cat([eval_quant_inp1, eval_quant_inp2], dim=1)\n" + ] } ], "source": [ @@ -1100,7 +1122,7 @@ "\n", "Per-channel quant output:\n", " QuantTensor(value=tensor([[[ 0.8616, -0.7012, 0.4503],\n", - " [-1.1285, -0.4937, -0.1901]]], grad_fn=), scale=tensor([[[0.0021],\n", + " [-1.1285, -0.4937, -0.1901]]], grad_fn=), scale=tensor([[[0.0021],\n", " [0.0013]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))\n" ] } @@ -1158,7 +1180,7 @@ "\n", "Per-channel quant output:\n", " QuantTensor(value=tensor([[[ 0.8616, -0.7012, 0.4503],\n", - " [-1.1285, -0.4937, -0.1901]]], grad_fn=), scale=tensor([[[0.0021],\n", + " [-1.1285, -0.4937, -0.1901]]], grad_fn=), scale=tensor([[[0.0021],\n", " [0.0013]]], grad_fn=), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))\n" ] } @@ -1345,8 +1367,8 @@ "text/plain": [ "QuantTensor(value=tensor([[-0.9109, -0.4588, 0.3119, -0.6530],\n", " [ 1.2089, 0.6493, -0.3731, 0.8706],\n", - " [ 1.3893, 0.2823, -0.8979, 0.9543]], grad_fn=), scale=tensor([[9.0542e-05, 3.9068e-05, 5.6866e-05, 6.4251e-05]],\n", - " grad_fn=), zero_point=tensor(0.), bit_width=tensor(17., grad_fn=), signed_t=tensor(True), training_t=tensor(True))" + " [ 1.3893, 0.2823, -0.8979, 0.9543]], grad_fn=), scale=tensor([[9.0542e-05, 3.9068e-05, 5.6866e-05, 6.4251e-05]],\n", + " grad_fn=), zero_point=tensor(0.), bit_width=tensor(17., grad_fn=), signed_t=tensor(True), training_t=tensor(True))" ] }, "execution_count": 28, @@ -1406,11 +1428,11 @@ "evalue": "Error(s) in loading state_dict for QuantLinear:\n\tMissing key(s) in state_dict: \"input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value\", \"input_quant.fused_activation_quant_proxy.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset\", \"weight_quant.tensor_quant.scaling_impl.value\", \"weight_quant.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset\". ", "output_type": "error", "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32mC:\\Users\\ALESSA~1\\AppData\\Local\\Temp/ipykernel_18920/1653109852.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 10\u001b[0m return_quant_tensor=True, bias=False)\n\u001b[0;32m 11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m \u001b[0mquant_linear\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfloat_linear\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m~\\miniconda3\\envs\\pt190\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[1;34m(self, state_dict, strict)\u001b[0m\n\u001b[0;32m 1405\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror_msgs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1406\u001b[0m raise RuntimeError('Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n\u001b[1;32m-> 1407\u001b[1;33m self.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n\u001b[0m\u001b[0;32m 1408\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0m_IncompatibleKeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmissing_keys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munexpected_keys\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1409\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for QuantLinear:\n\tMissing key(s) in state_dict: \"input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value\", \"input_quant.fused_activation_quant_proxy.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset\", \"weight_quant.tensor_quant.scaling_impl.value\", \"weight_quant.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset\". " + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[29], line 12\u001b[0m\n\u001b[1;32m 5\u001b[0m float_linear \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mLinear(\u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m4\u001b[39m, bias\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 6\u001b[0m quant_linear \u001b[38;5;241m=\u001b[39m QuantLinear(\n\u001b[1;32m 7\u001b[0m \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m4\u001b[39m, \n\u001b[1;32m 8\u001b[0m input_quant\u001b[38;5;241m=\u001b[39mLearnedIntActPerTensorFloat,\n\u001b[1;32m 9\u001b[0m weight_quant\u001b[38;5;241m=\u001b[39mLearnedIntWeightPerChannelFloat, \n\u001b[1;32m 10\u001b[0m return_quant_tensor\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, bias\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m---> 12\u001b[0m quant_linear\u001b[38;5;241m.\u001b[39mload_state_dict(float_linear\u001b[38;5;241m.\u001b[39mstate_dict())\n", + "File \u001b[0;32m/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/nn/modules/module.py:2152\u001b[0m, in \u001b[0;36mModule.load_state_dict\u001b[0;34m(self, state_dict, strict, assign)\u001b[0m\n\u001b[1;32m 2147\u001b[0m error_msgs\u001b[38;5;241m.\u001b[39minsert(\n\u001b[1;32m 2148\u001b[0m \u001b[38;5;241m0\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mMissing key(s) in state_dict: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 2149\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m missing_keys)))\n\u001b[1;32m 2151\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_msgs) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m-> 2152\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mError(s) in loading state_dict for \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 2153\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(error_msgs)))\n\u001b[1;32m 2154\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _IncompatibleKeys(missing_keys, unexpected_keys)\n", + "\u001b[0;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for QuantLinear:\n\tMissing key(s) in state_dict: \"input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value\", \"input_quant.fused_activation_quant_proxy.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset\", \"weight_quant.tensor_quant.scaling_impl.value\", \"weight_quant.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset\". " ] } ], @@ -1575,10 +1597,12 @@ " (stats): _Stats(\n", " (stats_impl): AbsPercentile()\n", " )\n", - " (restrict_clamp_scaling): _RestrictClampValue(\n", - " (clamp_min_ste): Identity()\n", + " (restrict_scaling): _RestrictValue(\n", " (restrict_value_impl): FloatRestrictValue()\n", " )\n", + " (clamp_scaling): _ClampValue(\n", + " (clamp_min_ste): ScalarClampMinSte()\n", + " )\n", " (restrict_inplace_preprocess): Identity()\n", " (restrict_preprocess): Identity()\n", " )\n", @@ -1852,13 +1876,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: netron in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (5.3.9)\n", - "Requirement already satisfied: onnx in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (1.10.2)\n", - "Requirement already satisfied: onnxoptimizer in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (0.2.6)\n", - "Requirement already satisfied: numpy>=1.16.6 in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (from onnx) (1.21.2)\n", - "Requirement already satisfied: typing-extensions>=3.6.2.1 in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (from onnx) (3.10.0.2)\n", - "Requirement already satisfied: protobuf in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (from onnx) (3.19.1)\n", - "Requirement already satisfied: six in c:\\users\\alessandro\\miniconda3\\envs\\pt190\\lib\\site-packages (from onnx) (1.16.0)\n" + "Requirement already satisfied: netron in /scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages (7.4.5)\n", + "Requirement already satisfied: onnx in /scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages (1.15.0)\n", + "Requirement already satisfied: onnxoptimizer in /scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages (0.3.13)\n", + "Requirement already satisfied: numpy in /scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages (from onnx) (1.26.0)\n", + "Requirement already satisfied: protobuf>=3.20.2 in /scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages (from onnx) (3.20.3)\n" ] } ], @@ -1894,9 +1916,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/brevitas/src/brevitas/export/onnx/standard/manager.py:26: UserWarning: ONNX opset version set to 13, override with opset_version=\n", + " warnings.warn(f\"ONNX opset version set to {DEFAULT_OPSET}, override with {ka}=\")\n" + ] + }, + { + "data": { + "text/plain": [ + "ir_version: 7\n", + "producer_name: \"pytorch\"\n", + "producer_version: \"2.1.1\"\n", + "graph {\n", + " node {\n", + " output: \"/export_handler/Constant_output_0\"\n", + " name: \"/export_handler/Constant\"\n", + " op_type: \"Constant\"\n", + " attribute {\n", + " name: \"value\"\n", + " t {\n", + " data_type: 1\n", + " raw_data: \"\\000\\000\\000<\"\n", + " }\n", + " type: TENSOR\n", + " }\n", + " }\n", + " node {\n", + " output: \"/export_handler/Constant_1_output_0\"\n", + " name: \"/export_handler/Constant_1\"\n", + " op_type: \"Constant\"\n", + " attribute {\n", + " name: \"value\"\n", + " t {\n", + " data_type: 3\n", + " raw_data: \"\\000\"\n", + " }\n", + " type: TENSOR\n", + " }\n", + " }\n", + " node {\n", + " input: \"inp.1\"\n", + " input: \"/export_handler/Constant_output_0\"\n", + " input: \"/export_handler/Constant_1_output_0\"\n", + " output: \"/export_handler/QuantizeLinear_output_0\"\n", + " name: \"/export_handler/QuantizeLinear\"\n", + " op_type: \"QuantizeLinear\"\n", + " }\n", + " node {\n", + " output: \"/export_handler/Constant_2_output_0\"\n", + " name: \"/export_handler/Constant_2\"\n", + " op_type: \"Constant\"\n", + " attribute {\n", + " name: \"value\"\n", + " t {\n", + " dims: 4\n", + " dims: 2\n", + " dims: 3\n", + " data_type: 3\n", + " raw_data: \"\\003\\006\\376\\006\\377\\001\\007\\371\\373\\376\\375\\006\\373\\375\\373\\371\\374\\006\\003\\004\\000\\374\\001\\371\"\n", + " }\n", + " type: TENSOR\n", + " }\n", + " }\n", + " node {\n", + " output: \"/export_handler/Constant_3_output_0\"\n", + " name: \"/export_handler/Constant_3\"\n", + " op_type: \"Constant\"\n", + " attribute {\n", + " name: \"value\"\n", + " t {\n", + " data_type: 1\n", + " raw_data: \"\\242\\272_=\"\n", + " }\n", + " type: TENSOR\n", + " }\n", + " }\n", + " node {\n", + " output: \"/export_handler/Constant_4_output_0\"\n", + " name: \"/export_handler/Constant_4\"\n", + " op_type: \"Constant\"\n", + " attribute {\n", + " name: \"value\"\n", + " t {\n", + " dims: 4\n", + " data_type: 6\n", + " raw_data: \"M\\375\\377\\377\\023\\376\\377\\377\\\\\\002\\000\\0001\\002\\000\\000\"\n", + " }\n", + " type: TENSOR\n", + " }\n", + " }\n", + " node {\n", + " input: \"/export_handler/QuantizeLinear_output_0\"\n", + " input: \"/export_handler/Constant_output_0\"\n", + " input: \"/export_handler/Constant_1_output_0\"\n", + " input: \"/export_handler/Constant_2_output_0\"\n", + " input: \"/export_handler/Constant_3_output_0\"\n", + " input: \"/export_handler/Constant_1_output_0\"\n", + " input: \"/export_handler/Constant_output_0\"\n", + " input: \"/export_handler/Constant_1_output_0\"\n", + " input: \"/export_handler/Constant_4_output_0\"\n", + " output: \"/export_handler/QLinearConv_output_0\"\n", + " name: \"/export_handler/QLinearConv\"\n", + " op_type: \"QLinearConv\"\n", + " attribute {\n", + " name: \"dilations\"\n", + " ints: 1\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"group\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"kernel_shape\"\n", + " ints: 3\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"pads\"\n", + " ints: 0\n", + " ints: 0\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"strides\"\n", + " ints: 1\n", + " type: INTS\n", + " }\n", + " }\n", + " node {\n", + " input: \"/export_handler/QLinearConv_output_0\"\n", + " input: \"/export_handler/Constant_output_0\"\n", + " input: \"/export_handler/Constant_1_output_0\"\n", + " output: \"10\"\n", + " name: \"/export_handler/DequantizeLinear\"\n", + " op_type: \"DequantizeLinear\"\n", + " }\n", + " name: \"main_graph\"\n", + " input {\n", + " name: \"inp.1\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 2\n", + " }\n", + " dim {\n", + " dim_value: 5\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " output {\n", + " name: \"10\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " dim {\n", + " dim_value: 3\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}\n", + "opset_import {\n", + " domain: \"\"\n", + " version: 13\n", + "}" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "torch.manual_seed(0)\n", "\n", @@ -1918,7 +2131,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 40, "metadata": { "tags": [ "skip-execution" @@ -1947,10 +2160,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 39, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -1980,9 +2193,319 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "ir_version: 8\n", + "producer_name: \"pytorch\"\n", + "producer_version: \"2.1.1\"\n", + "graph {\n", + " node {\n", + " input: \"x.87\"\n", + " input: \"/input_quant/export_handler/Constant_1_output_0\"\n", + " input: \"/input_quant/export_handler/Constant_2_output_0\"\n", + " input: \"/input_quant/export_handler/Constant_output_0\"\n", + " output: \"/input_quant/export_handler/Quant_output_0\"\n", + " name: \"/input_quant/export_handler/Quant\"\n", + " op_type: \"Quant\"\n", + " attribute {\n", + " name: \"narrow\"\n", + " i: 0\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"rounding_mode\"\n", + " s: \"ROUND\"\n", + " type: STRING\n", + " }\n", + " attribute {\n", + " name: \"signed\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " domain: \"onnx.brevitas\"\n", + " }\n", + " node {\n", + " input: \"/weight_quant/export_handler/Constant_1_output_0\"\n", + " input: \"/weight_quant/export_handler/Constant_2_output_0\"\n", + " input: \"/input_quant/export_handler/Constant_2_output_0\"\n", + " input: \"/weight_quant/export_handler/Constant_output_0\"\n", + " output: \"/weight_quant/export_handler/Quant_output_0\"\n", + " name: \"/weight_quant/export_handler/Quant\"\n", + " op_type: \"Quant\"\n", + " attribute {\n", + " name: \"narrow\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"rounding_mode\"\n", + " s: \"ROUND\"\n", + " type: STRING\n", + " }\n", + " attribute {\n", + " name: \"signed\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " domain: \"onnx.brevitas\"\n", + " }\n", + " node {\n", + " input: \"bias\"\n", + " input: \"onnx.brevitas::Quant_11\"\n", + " input: \"/input_quant/export_handler/Constant_2_output_0\"\n", + " input: \"/bias_quant/export_handler/Constant_output_0\"\n", + " output: \"/bias_quant/export_handler/Quant_output_0\"\n", + " name: \"/bias_quant/export_handler/Quant\"\n", + " op_type: \"Quant\"\n", + " attribute {\n", + " name: \"narrow\"\n", + " i: 0\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"rounding_mode\"\n", + " s: \"ROUND\"\n", + " type: STRING\n", + " }\n", + " attribute {\n", + " name: \"signed\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " domain: \"onnx.brevitas\"\n", + " }\n", + " node {\n", + " input: \"/input_quant/export_handler/Quant_output_0\"\n", + " input: \"/weight_quant/export_handler/Quant_output_0\"\n", + " input: \"/bias_quant/export_handler/Quant_output_0\"\n", + " output: \"/Conv_output_0\"\n", + " name: \"/Conv\"\n", + " op_type: \"Conv\"\n", + " attribute {\n", + " name: \"dilations\"\n", + " ints: 1\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"group\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"kernel_shape\"\n", + " ints: 3\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"pads\"\n", + " ints: 0\n", + " ints: 0\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"strides\"\n", + " ints: 1\n", + " type: INTS\n", + " }\n", + " }\n", + " node {\n", + " input: \"/Conv_output_0\"\n", + " input: \"/input_quant/export_handler/Constant_1_output_0\"\n", + " input: \"/input_quant/export_handler/Constant_2_output_0\"\n", + " input: \"/input_quant/export_handler/Constant_output_0\"\n", + " output: \"15\"\n", + " name: \"/output_quant/export_handler/Quant\"\n", + " op_type: \"Quant\"\n", + " attribute {\n", + " name: \"narrow\"\n", + " i: 0\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"rounding_mode\"\n", + " s: \"ROUND\"\n", + " type: STRING\n", + " }\n", + " attribute {\n", + " name: \"signed\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " domain: \"onnx.brevitas\"\n", + " }\n", + " name: \"main_graph\"\n", + " initializer {\n", + " dims: 4\n", + " data_type: 1\n", + " name: \"bias\"\n", + " raw_data: \"w\\010\\227\\276\\360\\203W\\276q\\341\\203>\\002\\034u>\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/input_quant/export_handler/Constant_output_0\"\n", + " raw_data: \"\\000\\000\\000A\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/input_quant/export_handler/Constant_1_output_0\"\n", + " raw_data: \"\\000\\000\\000<\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/input_quant/export_handler/Constant_2_output_0\"\n", + " raw_data: \"\\000\\000\\000\\000\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_output_0\"\n", + " raw_data: \"\\000\\000\\200@\"\n", + " }\n", + " initializer {\n", + " dims: 4\n", + " dims: 2\n", + " dims: 3\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_1_output_0\"\n", + " raw_data: \"\\372\\313\\'>\\372\\313\\247>\\242\\272\\337\\275\\372\\313\\247>\\242\\272_\\275\\242\\272_=N\\303\\303>N\\303\\303\\276\\245\\324\\213\\276\\242\\272\\337\\275\\372\\313\\'\\276\\372\\313\\247>\\245\\324\\213\\276\\372\\313\\'\\276\\245\\324\\213\\276N\\303\\303\\276\\242\\272_\\276\\372\\313\\247>\\372\\313\\'>\\242\\272_>\\000\\000\\000\\000\\242\\272_\\276\\242\\272_=N\\303\\303\\276\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_2_output_0\"\n", + " raw_data: \"\\242\\272_=\"\n", + " }\n", + " initializer {\n", + " dims: 1\n", + " data_type: 1\n", + " name: \"onnx.brevitas::Quant_11\"\n", + " raw_data: \"\\242\\272\\3379\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/bias_quant/export_handler/Constant_output_0\"\n", + " raw_data: \"\\000\\000\\200A\"\n", + " }\n", + " input {\n", + " name: \"x.87\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 2\n", + " }\n", + " dim {\n", + " dim_value: 5\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " input {\n", + " name: \"bias\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " output {\n", + " name: \"15\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " dim {\n", + " dim_value: 3\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " value_info {\n", + " name: \"/input_quant/export_handler/Quant_output_0\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 2\n", + " }\n", + " dim {\n", + " dim_value: 5\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " value_info {\n", + " name: \"/weight_quant/export_handler/Quant_output_0\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " dim {\n", + " dim_value: 2\n", + " }\n", + " dim {\n", + " dim_value: 3\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " value_info {\n", + " name: \"/bias_quant/export_handler/Quant_output_0\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}\n", + "opset_import {\n", + " domain: \"\"\n", + " version: 17\n", + "}\n", + "opset_import {\n", + " domain: \"onnx.brevitas\"\n", + " version: 1\n", + "}" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "torch.manual_seed(0)\n", "\n", @@ -2003,7 +2526,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 42, "metadata": { "tags": [ "skip-execution" @@ -2032,10 +2555,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 40, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -2053,9 +2576,191 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "ir_version: 8\n", + "producer_name: \"pytorch\"\n", + "producer_version: \"2.1.1\"\n", + "graph {\n", + " node {\n", + " input: \"/weight_quant/export_handler/Constant_1_output_0\"\n", + " input: \"/weight_quant/export_handler/Constant_2_output_0\"\n", + " input: \"/weight_quant/export_handler/Constant_3_output_0\"\n", + " input: \"/weight_quant/export_handler/Constant_output_0\"\n", + " output: \"/weight_quant/export_handler/Quant_output_0\"\n", + " name: \"/weight_quant/export_handler/Quant\"\n", + " op_type: \"Quant\"\n", + " attribute {\n", + " name: \"narrow\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"rounding_mode\"\n", + " s: \"ROUND\"\n", + " type: STRING\n", + " }\n", + " attribute {\n", + " name: \"signed\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " domain: \"onnx.brevitas\"\n", + " }\n", + " node {\n", + " input: \"x.27\"\n", + " input: \"/weight_quant/export_handler/Quant_output_0\"\n", + " input: \"bias\"\n", + " output: \"8\"\n", + " name: \"/Conv\"\n", + " op_type: \"Conv\"\n", + " attribute {\n", + " name: \"dilations\"\n", + " ints: 1\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"group\"\n", + " i: 1\n", + " type: INT\n", + " }\n", + " attribute {\n", + " name: \"kernel_shape\"\n", + " ints: 3\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"pads\"\n", + " ints: 0\n", + " ints: 0\n", + " type: INTS\n", + " }\n", + " attribute {\n", + " name: \"strides\"\n", + " ints: 1\n", + " type: INTS\n", + " }\n", + " }\n", + " name: \"main_graph\"\n", + " initializer {\n", + " dims: 4\n", + " data_type: 1\n", + " name: \"bias\"\n", + " raw_data: \"\\243\\303\\206\\275\\325\\3600=\\366C\\275>\\222\\347\\301\\276\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_output_0\"\n", + " raw_data: \"\\000\\000\\200@\"\n", + " }\n", + " initializer {\n", + " dims: 4\n", + " dims: 2\n", + " dims: 3\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_1_output_0\"\n", + " raw_data: \"\\000\\000\\000\\200\\2227d>\\256)\\253\\276\\273\\242\\216\\276\\256)+\\276\\2227\\344=\\000\\000\\000\\200\\256)\\253>\\2227d\\275\\2227\\344=\\2227\\344\\275\\2227d\\275\\240\\260\\307\\276\\273\\242\\216\\276\\256)+\\276\\000\\000\\000\\000\\256)+>\\2227d>\\273\\242\\216\\276\\256)+\\276\\256)+>\\256)\\253>\\2227\\344\\275\\273\\242\\216>\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_2_output_0\"\n", + " raw_data: \"\\2227d=\"\n", + " }\n", + " initializer {\n", + " data_type: 1\n", + " name: \"/weight_quant/export_handler/Constant_3_output_0\"\n", + " raw_data: \"\\000\\000\\000\\000\"\n", + " }\n", + " input {\n", + " name: \"x.27\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 2\n", + " }\n", + " dim {\n", + " dim_value: 5\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " input {\n", + " name: \"bias\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " output {\n", + " name: \"8\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 1\n", + " }\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " dim {\n", + " dim_value: 3\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + " value_info {\n", + " name: \"/weight_quant/export_handler/Quant_output_0\"\n", + " type {\n", + " tensor_type {\n", + " elem_type: 1\n", + " shape {\n", + " dim {\n", + " dim_value: 4\n", + " }\n", + " dim {\n", + " dim_value: 2\n", + " }\n", + " dim {\n", + " dim_value: 3\n", + " }\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}\n", + "opset_import {\n", + " domain: \"\"\n", + " version: 17\n", + "}\n", + "opset_import {\n", + " domain: \"onnx.brevitas\"\n", + " version: 1\n", + "}" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "torch.manual_seed(0)\n", "\n", @@ -2067,7 +2772,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 44, "metadata": { "tags": [ "skip-execution" @@ -2096,10 +2801,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 41, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -2121,9 +2826,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/brevitas/src/brevitas/quant_tensor/__init__.py:68: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n", + " training = torch.tensor(training, dtype=torch.bool)\n" + ] + }, + { + "data": { + "text/plain": [ + "RecursiveScriptModule(original_name=_JitTraceExportWrapper)" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from brevitas.quant import ShiftedUint8ActPerTensorFloat\n", "from brevitas.export import export_torch_qop\n", @@ -2142,21 +2866,13 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 46, "metadata": { "tags": [ "skip-execution" ] }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\users\\alessandro\\documenti\\brevitas_tvmcon\\src\\brevitas\\quant_tensor\\__init__.py:74: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n", - " training = torch.tensor(training, dtype=torch.bool)\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -2179,10 +2895,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 42, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -2239,9 +2955,24 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 47, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/overrides.py:110: UserWarning: 'has_cuda' is deprecated, please use 'torch.backends.cuda.is_built()'\n", + " torch.has_cuda,\n", + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/overrides.py:111: UserWarning: 'has_cudnn' is deprecated, please use 'torch.backends.cudnn.is_available()'\n", + " torch.has_cudnn,\n", + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/overrides.py:117: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()'\n", + " torch.has_mps,\n", + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/overrides.py:118: UserWarning: 'has_mkldnn' is deprecated, please use 'torch.backends.mkldnn.is_available()'\n", + " torch.has_mkldnn,\n" + ] + } + ], "source": [ "from brevitas.graph.calibrate import bias_correction_mode\n", "from brevitas.graph.calibrate import calibration_mode\n", @@ -2266,7 +2997,7 @@ ], "metadata": { "kernelspec": { - "display_name": "pytorch_latest", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -2280,7 +3011,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.15 | packaged by conda-forge | (default, Nov 22 2022, 08:42:03) [MSC v.1929 64 bit (AMD64)]" + "version": "3.11.5" }, "vscode": { "interpreter": { diff --git a/notebooks/ONNX_export_tutorial.ipynb b/notebooks/ONNX_export_tutorial.ipynb index 304161fce..d9cb7d0a7 100644 --- a/notebooks/ONNX_export_tutorial.ipynb +++ b/notebooks/ONNX_export_tutorial.ipynb @@ -5,6 +5,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -22,9 +25,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: netron in /scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages (7.4.5)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install netron" ] @@ -34,6 +46,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -51,6 +66,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -77,6 +95,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -95,9 +116,12 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } @@ -108,6 +132,11 @@ "import time\n", "from IPython.display import IFrame\n", "\n", + "# helpers\n", + "def assert_with_message(condition):\n", + " assert condition\n", + " print(condition)\n", + "\n", "def show_netron(model_path, port):\n", " time.sleep(3.)\n", " netron.start(model_path, address=(\"localhost\", port), browse=False)\n", @@ -116,9 +145,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } @@ -133,6 +165,9 @@ "OUT_CH = 128\n", "BATCH_SIZE = 1\n", "\n", + "# set seed\n", + "torch.manual_seed(0)\n", + "\n", "linear = qnn.QuantLinear(IN_CH, OUT_CH, bias=True)\n", "inp = torch.randn(BATCH_SIZE, IN_CH)\n", "path = 'quant_linear_qcdq.onnx'\n", @@ -142,9 +177,12 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" }, @@ -175,10 +213,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -191,6 +229,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -207,6 +248,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -219,9 +263,12 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } @@ -248,9 +295,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" }, @@ -281,10 +331,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -297,6 +347,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -314,6 +367,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -334,9 +390,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } @@ -365,9 +424,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" }, @@ -398,10 +460,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -414,6 +476,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -428,6 +493,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -446,9 +514,12 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } @@ -458,7 +529,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\alessand\\documents\\brevitas\\src\\brevitas\\export\\onnx\\standard\\manager.py:23: UserWarning: ONNX opset version set to 13, override with opset_version=\n", + "/scratch/fabian/brevitas/src/brevitas/export/onnx/standard/manager.py:26: UserWarning: ONNX opset version set to 13, override with opset_version=\n", " warnings.warn(f\"ONNX opset version set to {DEFAULT_OPSET}, override with {ka}=\")\n" ] }, @@ -467,7 +538,7 @@ "text/plain": [ "ir_version: 7\n", "producer_name: \"pytorch\"\n", - "producer_version: \"1.13.1\"\n", + "producer_version: \"2.1.1\"\n", "graph {\n", " node {\n", " output: \"/input_quant/export_handler/Constant_output_0\"\n", @@ -496,7 +567,7 @@ " }\n", " }\n", " node {\n", - " input: \"inp.1\"\n", + " input: \"out.1\"\n", " input: \"/input_quant/export_handler/Constant_output_0\"\n", " input: \"/input_quant/export_handler/Constant_1_output_0\"\n", " output: \"/input_quant/export_handler/QuantizeLinear_output_0\"\n", @@ -515,7 +586,7 @@ " dims: 3\n", " dims: 3\n", " data_type: 3\n", - " raw_data: \"\\374\\372\\376\\374\\005\\000\\375\\374\\004\\375\\373\\373\\375\\007\\376\\374\\377\\000\\000\\373\\373\\004\\005\\371\\003\\375\\004\\373\\004\\374\\000\\006\\002\\003\\003\\005\\004\\377\\005\\000\\373\\376\\375\\376\\002\\376\\004\\377\\003\\005\\375\\371\\006\\373\\003\\007\\377\\374\\005\\375\\375\\006\\375\\377\\374\\001\\005\\371\\006\\005\\007\\376\\376\\372\\376\\004\\001\\374\\002\\373\\373\\376\\002\\376\\375\\377\\001\\376\\006\\371\\002\\000\\004\\005\\005\\000\\004\\373\\004\\002\\003\\000\\374\\376\\005\\000\\004\\372\\004\\000\\373\\000\\006\\377\\002\\005\\004\\005\\374\\000\\007\\377\\374\\371\\373\\007\\004\\376\\372\\001\\005\\001\\372\\377\\003\\001\\375\\006\\372\\377\\006\\003\\006\\004\\001\\004\\372\\005\\006\\003\\376\\373\\374\\375\\376\\005\\000\\004\\377\\372\\373\\000\\007\\377\\373\\003\\373\\376\\374\\374\\377\\375\\377\\003\\372\\005\\004\\007\\003\\375\\377\\001\\007\\377\\373\\374\\000\\377\\376\\374\\373\\377\\373\\375\\003\\004\\004\\376\\004\\377\\375\\003\\003\\377\\004\\000\\005\\004\\000\\372\\005\\007\\003\\004\\377\\373\\003\\371\\373\\002\\377\\006\\006\\007\\377\\376\\375\\002\\006\\005\\004\\374\\002\\000\\373\\004\\002\\002\\374\\371\\372\\371\\375\\001\\004\\000\\006\\376\\377\\002\\000\\372\\001\\001\\375\\007\\376\\005\\001\\373\\003\\374\\005\\003\\007\\005\\372\\004\\006\\375\\005\\003\\001\\373\\376\\374\\002\\376\\377\\376\\000\\006\\001\\375\\376\\377\\374\\000\\005\\002\\005\\006\\371\\375\\005\\375\\376\\374\\004\\001\\003\\001\\372\\005\\007\\371\\005\\000\\372\\001\\001\\371\\007\\374\\372\\373\\373\\372\\376\\004\\000\\002\\375\\376\\000\\004\\003\\003\\375\\003\\001\\376\\006\\001\\000\\372\\374\\376\\373\\002\\002\\004\\372\\377\\374\\005\\000\\001\\005\\005\\374\\007\\003\\377\\377\\000\\007\\002\\377\\377\\377\\374\\001\\001\\376\\000\\377\\373\\001\\004\\376\\003\\000\\007\\005\\000\\374\\372\\376\\005\\003\\003\\004\\372\\375\\372\\377\\006\\376\\374\\007\\373\\002\\374\\003\\377\\374\\002\\007\\373\\004\\376\\004\\004\\003\\005\\373\\003\\005\\376\\001\\000\\002\\371\\376\\000\\374\\377\\372\\375\\005\\373\\002\\373\\373\\377\\004\\375\\006\\377\\005\\005\\002\\375\\375\\003\\376\\376\\006\\002\\371\\000\\002\\373\\000\\006\\002\\372\\372\\006\\374\\372\\004\\006\\004\\000\\003\\001\\377\\371\\376\\003\\003\\373\\005\\000\\001\\003\\004\\001\\005\\001\\004\\373\\373\\372\\002\\371\\375\\372\\004\\377\\005\\375\\376\\374\\375\\003\\372\\001\\373\\372\\376\\005\\003\\372\\004\\373\\004\\374\\374\\376\\376\\377\\371\\375\\004\\375\\377\\376\\007\\004\\372\\000\\007\\372\\006\\002\\006\\001\\006\\372\\004\\004\\003\\002\\375\\006\\374\\002\\001\\001\\000\\376\\376\\006\\373\\374\\002\\372\\005\\374\\004\\004\\001\\374\\004\\377\\373\\002\\376\\001\\377\\003\\377\\007\\004\\372\\371\\002\\375\\377\\373\\002\\376\\375\\377\\006\\001\\001\\000\\374\\001\\006\\004\\371\\377\\375\\374\\377\\376\\003\\372\\373\\002\\005\\374\\000\\002\\004\\372\\004\\372\\003\\006\\375\\003\\377\\376\\000\\377\\374\\006\\377\\374\\375\\377\\373\\376\\372\\375\\006\\004\\371\\372\\374\\375\\004\\002\\372\\376\\001\\001\\002\\373\\000\\003\\000\\371\\001\\003\\377\\376\\371\\376\\004\\000\\003\\376\\002\\006\\004\\372\\007\\005\\004\\376\\000\\007\\372\\003\\002\\005\\005\\004\\372\\002\\377\\006\\002\\371\\375\\375\\372\\376\\005\\003\\000\\002\\371\\005\\372\\373\\377\\371\\376\\005\\374\\377\\007\\003\\001\\376\\006\\376\\001\\374\\374\\001\\373\\006\\376\\376\\001\\372\\377\\003\\006\\372\\373\\003\\377\\376\\000\\377\\373\\004\\372\\371\\376\\002\\004\\004\\006\\001\\372\\001\\376\\005\\001\\000\\000\\007\\002\\375\\002\\375\\375\\006\\007\\375\\375\\002\\006\\371\\375\\002\\377\\002\\377\\000\\373\\001\\372\\372\\001\\377\\372\\001\\002\\000\\375\\373\\377\\372\\001\\371\\372\\007\\372\\001\\377\\372\\004\\376\\376\\374\\375\\373\\373\\005\\371\\375\\006\\005\\007\\374\\373\\005\\372\\000\\001\\374\\005\\000\\002\\373\\004\\001\\004\\006\\002\\003\\373\\376\\372\\374\\003\\375\\005\\000\\005\\373\\001\\375\\374\\002\\002\\000\\373\\374\\003\\005\\376\\003\\374\\374\\373\\374\\000\\004\\371\\375\\372\\003\\375\\005\\005\\006\\007\\371\\003\\372\\003\\375\\004\\374\\001\\376\\373\\000\\004\\003\\001\\003\\372\\377\\003\\004\\374\\000\\376\\002\\377\\001\\374\\376\\002\\002\\001\\005\\375\\373\\001\\372\\000\\007\\004\\007\\006\\006\\000\\004\\004\\006\\000\\377\\375\\000\\002\\374\\376\\374\\006\\373\\377\\000\\374\\006\\373\\005\\001\\001\\006\\005\\373\\373\\001\\003\\371\\006\\372\\003\\005\\372\\003\\005\\006\\005\\006\\001\\001\\377\\372\\001\\003\\372\\005\\002\\376\\377\\373\\005\\376\\375\\373\\005\\004\\007\\001\\000\\002\\001\\374\\004\\003\\377\\004\\372\\373\\373\\007\\375\\002\\002\\377\\373\\007\\001\\004\\374\\007\\376\\000\\003\\376\\006\\371\\377\\003\\376\\003\\004\\375\\006\\376\\371\\373\\373\\004\\000\\005\\377\\372\\372\\377\\004\\002\\001\\000\\005\\372\\004\\377\\376\\375\\001\\005\\375\\375\\000\\003\\006\\374\\004\\377\\004\\006\\000\\374\\003\\000\\005\\376\\372\\371\\371\\000\\374\\372\\006\\004\\006\\376\\377\\001\\377\\376\\373\\374\\000\\003\\004\\372\\375\\000\\006\\002\\374\\377\\004\\372\\371\\373\\001\\006\\377\\003\\007\\377\\373\\000\\371\\002\\376\\003\\002\\377\\006\\006\\006\\371\\006\\373\\377\\006\\000\\374\\375\\376\\001\\376\\007\\003\\007\\376\\004\\001\\005\\003\\375\\372\\003\\004\\376\\374\\005\\372\\372\\000\\006\\377\\003\\000\\002\\001\\003\\375\\000\\004\\375\\372\\000\\001\\000\\000\\002\\000\\004\\005\\377\\005\\007\\376\\372\\001\\374\\006\\002\\376\\002\\005\\374\\372\\000\\375\\372\\372\\000\\001\\000\\377\\007\\376\\000\\374\\375\\000\\373\\003\\001\\006\\003\\376\\007\\374\\376\\374\\005\\371\\372\\001\\374\\374\\002\\375\\004\\001\\002\\002\\376\\003\\373\\000\\375\\375\\005\\373\\002\\376\\371\\006\\004\\001\\001\\371\\376\\005\\377\\375\\005\\003\\374\\375\\002\\373\\376\\001\\002\\001\\007\\002\\004\\376\\375\\377\\376\\004\\373\\000\\001\\375\\377\\372\\376\\002\\001\\375\\006\\005\\006\\004\\376\\004\\004\\001\\001\\377\\004\\006\\003\\001\\005\\006\\001\\377\\000\\000\\000\\372\\004\\375\\004\\377\\377\\006\\377\\373\\003\\375\\373\\004\\005\\377\\006\\376\\374\\374\\371\\376\\003\\376\\374\\001\\373\\001\\375\\001\\376\\376\\000\\376\\371\\376\\377\\372\\373\\374\\374\\375\\376\\003\\376\\002\\372\\375\\375\\007\\377\\373\\377\\006\\376\\377\\373\\002\\001\\000\\005\\004\\006\\376\\001\\373\\372\\371\\001\\371\\001\\373\\374\\001\\375\\373\\003\\375\\373\\005\\373\\004\\377\\002\\000\\002\\006\\001\\373\\375\\005\\376\\004\\000\\376\\003\\007\\000\\377\\003\\004\\005\\376\\004\\003\\004\\006\\006\\006\\371\\002\\374\\375\\003\\375\\000\\375\\377\\004\\003\\374\\373\\004\\005\\375\\003\\376\\001\\001\\374\\003\\377\\004\\006\\003\\377\\001\\003\\377\\377\\371\\000\\374\\003\\373\\374\\006\\372\\372\\006\\004\\375\\375\\373\\004\\005\\001\\373\\371\\377\\376\\004\\005\\373\\374\\005\\000\\376\\001\\002\\003\\006\\006\\374\\375\\374\\377\\001\\373\\003\\004\\372\\004\\375\\001\\371\\004\\002\\001\\376\\377\\005\\000\\376\\376\\372\\005\\000\\376\\004\\371\\000\\377\\377\\377\\373\\377\\001\\004\\002\\374\\373\\000\\374\\377\\373\\373\\374\\005\\006\\374\\003\\373\\000\\006\\001\\003\\371\\373\\006\\374\\005\\005\\006\\371\\002\\005\\373\\000\\377\\377\\003\\005\\003\\004\\376\\372\\000\\005\\004\\371\\372\\376\\371\\005\\375\\000\\001\\001\\000\\006\\005\\006\\002\\002\\000\\003\\006\\374\\005\\000\\373\\372\\376\\002\\372\\006\\003\\375\\373\\373\\375\\002\\004\\001\\007\\373\\377\\004\\005\\004\\375\\005\\376\\376\\004\\003\\000\\004\\376\\006\\001\\376\\003\\376\\007\\006\\002\\376\\001\\376\\006\\371\\006\\375\\375\\004\\003\\006\\377\\374\\004\\003\\375\\372\\374\\375\\006\\377\\000\\004\\373\\002\\006\\373\\377\\374\\372\\000\\000\\376\\006\\373\\372\\004\\001\\006\\003\\377\\006\\371\\006\\006\\004\\004\\005\\371\\376\\001\\003\\372\\005\\001\\002\\373\\001\\372\\375\\004\\372\\006\\373\\375\\001\\003\\375\\377\\003\\372\\374\\374\\373\\006\\005\\373\\002\\000\\004\\376\\377\\004\\374\\006\\374\\006\\373\\004\\375\\373\\006\\376\\006\\002\\002\\377\\372\\372\\005\\004\\375\\000\\002\\374\\002\\376\\007\\373\\376\\371\\377\\005\\376\\006\\002\\006\\376\\004\\372\\000\\005\\002\\002\\003\\006\\004\\377\\007\\374\\372\\372\\002\\375\\377\\001\\375\\005\\374\\377\\003\\007\\002\\005\\006\\006\\000\\001\\004\\000\\376\\371\\001\\000\\005\\004\\375\\372\\375\\004\\007\\371\\374\\002\\005\\000\\002\\002\\004\\004\\007\\005\\006\\373\\006\\004\\002\\005\\004\\376\\375\\000\\372\\004\\377\\003\\374\\003\\376\\006\\376\\006\\006\\005\\002\\006\\007\\002\\372\\372\\377\\373\\004\\373\\375\\004\\004\\003\\006\\002\\000\\002\\376\\000\\000\\005\\006\\005\\372\\003\\372\\006\\001\\007\\372\\002\\372\\004\\001\\005\\002\\005\\374\\372\\372\\002\\372\\001\\377\\002\\006\\005\\000\\005\\372\\375\\007\\377\\375\\004\\005\\003\\372\\004\\005\\376\\373\\001\\372\\003\\371\\371\\374\\005\\002\\005\\374\\377\\004\\002\\376\\004\\373\\377\\377\\377\\001\\005\\372\\003\\373\\375\\006\\374\\007\\376\\372\\006\\005\\371\\377\\005\\001\\003\\005\\002\\006\\003\\001\\377\\374\\004\\376\\374\\375\\376\\001\\001\\001\\004\\007\\007\\000\\005\\001\\376\\003\\376\\000\\000\\001\\001\\375\\371\\006\\002\\001\\373\\000\\377\\007\\004\\002\\374\\000\\001\\377\\003\\374\\003\\007\\373\\371\\373\\001\\005\\373\\372\\005\\373\\375\\005\\006\\372\\000\\005\\007\\003\\003\\377\\005\\006\\004\\374\\372\\375\\003\\004\\000\\005\\376\\374\\374\\375\\375\\377\\372\\000\\004\\002\\005\\002\\000\\374\\376\\373\\373\\376\\002\\374\\000\\376\\373\\000\\371\\373\\372\\006\\000\\376\\002\\375\\376\\005\\372\\004\\376\\375\\005\\006\\006\\004\\003\\002\\002\\002\\002\\375\\006\\377\\000\\004\\375\\004\\007\\004\\005\\372\\374\\004\\377\\003\\377\\000\\375\\372\\374\\372\\000\\004\\007\\002\\007\\372\\376\\004\\371\\375\\001\\001\\007\\003\\000\\004\\373\\001\\001\\376\\002\\377\\377\\006\\002\\003\\373\\373\\004\\372\\372\\376\\372\\002\\002\\002\\373\\001\\375\\374\\000\\004\\003\\376\\003\\376\\002\\373\\374\\003\\372\\371\\001\\375\\004\\371\\374\\004\\005\\002\\374\\371\\001\\373\\377\\374\\006\\373\\006\\000\\005\\005\\006\\006\\002\\375\\002\\001\\001\\005\\375\\000\\372\\371\\003\\004\\375\\376\\003\\377\\374\\005\\007\\007\\377\\374\\375\\374\\376\\373\\003\\002\\002\\374\\377\\373\\004\\375\\372\\374\\003\\374\\005\\376\\002\\373\\376\\006\\005\\374\\002\\371\\005\\004\\001\\373\\000\\377\\374\\003\\000\\001\\001\\003\\372\\005\\001\\371\\371\\000\\375\\001\\375\\372\\374\\003\\373\\376\\001\\371\\006\\005\\004\\377\\004\\376\\377\\377\\003\\373\\001\\372\\376\\006\\372\\372\\005\\374\\001\\374\\004\\001\\004\\375\\002\\002\\373\\006\\000\\001\\002\\377\\371\\005\\005\\374\\374\\006\\003\\001\\002\\001\\374\\377\\372\\000\\377\\374\\373\\371\\007\\003\\375\\373\\374\\373\\374\\005\\004\\005\\006\\002\\374\\000\\372\\001\\376\\002\\373\\371\\372\\374\\374\\377\\005\\375\\371\\002\\374\\374\\005\\377\\007\\004\\376\\007\\373\\372\\007\\007\\377\\004\\002\\002\\007\\377\\375\\002\\005\\006\\003\\002\\006\\376\\003\\004\\003\\000\\371\\002\\002\\374\\006\\373\\005\\003\\003\\002\\003\\376\\002\\004\\377\\377\\371\\007\\001\\373\\376\\003\\002\\007\\376\\002\\005\\004\\374\\003\\377\\374\\003\\007\\004\\377\\002\\001\\003\\005\\373\\377\\374\\002\\377\\004\\000\\000\\005\\007\\002\\003\\376\\371\\377\\006\\372\\372\\002\\372\\371\\375\\000\\376\\005\\372\\000\\373\\372\\007\\002\\001\\372\\374\\375\\005\\005\\004\\001\\002\\002\\006\\372\\001\\007\\373\\375\\000\\372\\005\\003\\000\\375\\377\\001\\003\\006\\000\\376\\374\\002\\375\\375\\003\\001\\007\\376\\377\\003\\000\\005\\376\\374\\005\\373\\004\\377\\000\\375\\002\\005\\001\\001\\000\\001\\375\\374\\001\\006\\372\\375\\376\\372\\371\\001\\372\\005\\004\\376\\373\\006\\005\\375\\006\\377\\001\\001\\000\\006\\000\\006\\007\\003\\372\\004\\375\\373\\372\\372\\000\\374\\001\\006\\007\\376\\374\\371\\373\\372\\375\\003\\377\\372\\377\\005\\002\\006\\372\\006\\004\\005\\000\\376\\007\\003\\372\\004\\377\\006\\001\\373\\375\\374\\373\\373\\004\\004\\375\\373\\005\\376\\000\\001\\375\\371\\372\\005\\375\\000\\002\\372\\003\\004\\372\\003\\374\\005\\002\\374\\377\\001\\005\\376\\377\\374\\376\\005\\376\\372\\003\\373\\372\\006\\372\\377\\373\\006\\372\\004\\006\\373\\005\\375\\375\\007\\374\\005\\002\\374\\374\\002\\002\\377\\375\\376\\372\\005\\375\\371\\003\\005\\003\\372\\377\\375\\372\\002\\005\\000\\006\\372\\005\\371\\376\\000\\001\\377\\004\\004\\006\\000\\377\\007\\002\\006\\000\\371\\375\\374\\374\\001\\373\\371\\002\\376\\002\\000\\374\\006\\001\\374\\006\\005\\001\\003\\376\\003\\374\\003\\374\\002\\007\\373\\002\\004\\007\\005\\374\\376\\372\\372\\001\\371\\002\\005\\373\\376\\006\\375\\372\\376\\004\\003\\001\\004\\376\\002\\373\\006\\006\\371\\372\\003\\004\\006\\375\\004\\007\\371\\000\\000\\001\\000\\374\\001\\006\\002\\006\\002\\000\\002\\373\\372\\372\\000\\372\\005\\006\\004\\000\\376\\372\\373\\006\\007\\373\\006\\373\\377\\003\\375\\373\\001\\377\\001\\002\\376\\003\\373\\002\\376\\007\\371\\371\\374\\006\\377\\001\\002\\005\\001\\376\\375\\000\\377\\371\\005\\372\\002\\377\\375\\375\\002\\375\\376\\003\\003\\373\\373\\005\\004\\004\\373\\000\\000\\007\\003\\372\\375\\004\\003\\376\\377\\373\\376\\004\\372\\004\\377\\376\\007\\002\\005\\003\\001\\006\\006\\002\\005\\373\\000\\004\\000\\004\\374\\372\\376\\007\\002\\003\\006\\002\\000\\372\\001\\374\\005\\376\\006\\007\\373\\001\\375\\004\\377\\374\\375\\377\\001\\377\\003\\375\\005\\000\\003\\376\\375\\003\\377\\372\\002\\006\\003\\007\\005\\374\\003\\006\\003\\000\\375\\000\\001\\000\\001\\002\\374\\377\\372\\004\\372\\377\\377\\003\\377\\007\\006\\371\\003\\005\\004\\007\\006\\371\\006\\001\\375\\001\\001\\376\\002\\374\\006\\375\\375\\376\\377\\002\\002\\007\\373\\373\\374\\373\\377\\001\\006\\375\\375\\001\\375\\373\\375\\373\\372\\376\\003\\371\\006\\376\\376\\375\\007\\377\\374\\376\\377\\006\\377\\001\\371\\377\\007\\375\\371\\005\\002\\373\\003\\005\\002\\371\\375\\003\\003\\003\\374\\000\\377\\375\\003\\002\\006\\006\\375\\006\\002\\000\\374\\373\\374\\002\\003\\373\\002\\375\\377\\004\\006\\003\\006\\000\\377\\372\\375\\375\\002\\002\\003\\006\\003\\003\\377\\373\\003\\003\\003\\003\\377\\004\\004\\372\\377\\000\\374\\375\\005\\004\\005\\003\\002\\375\\376\\001\\376\\003\\374\\002\\007\\002\\376\\377\\007\\006\\376\\372\\374\\004\\371\\004\\006\\006\\374\\374\\377\\374\\003\\006\\371\\377\\007\\372\\375\\006\\374\\374\\005\\372\\006\\372\\371\\001\\000\\375\\372\\374\\373\\374\\374\\374\\005\\004\\002\\375\\004\\007\\004\\006\\002\\005\\005\\372\\375\\000\\004\\000\\377\\004\\004\\001\\374\\377\\006\\003\\377\\374\\000\\376\\372\\376\\373\\377\\006\\377\\376\\002\\005\\005\\372\\004\\000\\001\\004\\005\\373\\005\\003\\371\\374\\373\\000\\375\\002\\375\\006\\003\\001\\004\\377\\374\\372\\005\\006\\005\\005\\005\\005\\007\\372\\006\\004\\006\\372\\372\\002\\373\\371\\001\\004\\006\\374\\005\\373\\004\\006\\001\\005\\006\\377\\006\\373\\001\\373\\373\\376\\375\\007\\372\\374\\372\\377\\004\\006\\004\\375\\374\\000\\007\\005\\000\\002\\377\\002\\372\\002\\001\\377\\372\\006\\002\\001\\000\\376\\375\\374\\003\\376\\371\\005\\001\\000\\002\\372\\373\\375\\004\\376\\371\\374\\376\\000\\004\\004\\376\\375\\007\\374\\377\\375\\377\\001\\003\\005\\372\\002\\376\\003\\003\\375\\001\\004\\001\\001\\000\\002\\004\\375\\375\\372\\003\\003\\372\\002\\375\\372\\377\\373\\000\\002\\371\\005\\003\\001\\001\\376\\372\\374\\001\\001\\376\\000\\001\\376\\001\\376\\005\\002\\374\\002\\004\\004\\000\\374\\007\\000\\000\\006\\003\\371\\376\\371\\006\\005\\006\\007\\002\\371\\373\\005\\372\\375\\006\\003\\373\\005\\375\\375\\373\\002\\000\\375\\005\\001\\372\\377\\377\\373\\375\\375\\374\\000\\376\\372\\000\\374\\001\\001\\372\\375\\373\\004\\374\\000\\006\\375\\004\\001\\006\\000\\373\\001\\375\\003\\372\\000\\373\\376\\003\\374\\005\\007\\377\\373\\007\\006\\002\\371\\373\\377\\004\\373\\001\\374\\000\\001\\004\\001\\005\\375\\372\\002\\376\\377\\371\\374\\375\\371\\373\\005\\376\\374\\001\\377\\376\\371\\375\\371\\000\\375\\373\\377\\006\\002\\003\\005\\372\\003\\004\\005\\005\\004\\000\\376\\372\\371\\006\\000\\377\\373\\003\\376\\005\\007\\006\\372\\004\\007\\374\\375\\376\\374\\000\\001\\001\\375\\003\\371\\001\\006\\374\\376\\006\\377\\000\\001\\375\\006\\004\\372\\371\\001\\377\\377\\377\\376\\006\\375\\372\\000\\371\\376\\002\\374\\372\\006\\372\\002\\006\\005\\001\\376\\004\\374\\002\\376\\000\\004\\376\\375\\000\\376\\004\\000\\006\\372\\005\\007\\006\\002\\004\\373\\373\\006\\003\\007\\001\\375\\007\\007\\372\\004\\005\\376\\005\\376\\007\\002\\376\\004\\373\\373\\376\\004\\372\\375\\373\\374\\001\\000\\375\\004\\375\\375\\377\\004\\001\\377\\002\\376\\004\\377\\001\\001\\374\\376\\374\\377\\377\\001\\000\\000\\377\\373\\374\\002\\006\\001\\375\\376\\000\\000\\374\\006\\004\\004\\004\\375\\001\\376\\001\\002\\373\\006\\006\\376\\002\\005\\005\\374\\373\\377\\376\\004\\005\\374\\000\\376\\002\\375\\376\\004\\373\\001\\377\\377\\002\\377\\373\\372\\371\\003\\003\\372\\006\\000\\002\\003\\005\\375\\371\\375\\004\\376\\374\\007\\375\\371\\002\\374\\000\\375\\005\\006\\374\\373\\004\\371\\000\\007\\376\\001\\375\\377\\372\\372\\373\\005\\005\\001\\372\\377\\371\\377\\375\"\n", + " raw_data: \"\\003\\377\\371\\374\\001\\002\\374\\006\\004\\374\\372\\007\\004\\006\\002\\005\\376\\375\\006\\374\\375\\372\\005\\373\\376\\000\\375\\374\\005\\372\\001\\001\\004\\004\\001\\005\\373\\000\\005\\001\\002\\002\\000\\004\\374\\000\\007\\000\\377\\372\\375\\006\\000\\000\\005\\001\\374\\374\\377\\003\\003\\377\\372\\007\\376\\001\\373\\005\\001\\003\\372\\375\\007\\374\\373\\005\\006\\371\\006\\377\\001\\001\\000\\376\\376\\000\\006\\377\\000\\373\\006\\003\\006\\004\\372\\004\\376\\004\\373\\004\\003\\372\\000\\000\\371\\003\\005\\376\\372\\372\\377\\007\\005\\375\\373\\371\\373\\005\\006\\373\\004\\373\\006\\376\\374\\371\\001\\375\\005\\000\\006\\377\\001\\373\\001\\373\\377\\373\\376\\003\\371\\371\\374\\002\\376\\376\\377\\004\\373\\001\\001\\377\\002\\375\\371\\376\\005\\006\\373\\377\\376\\000\\375\\373\\004\\375\\007\\376\\377\\001\\005\\375\\007\\000\\373\\004\\371\\002\\005\\004\\002\\376\\375\\003\\373\\376\\377\\377\\006\\375\\374\\377\\376\\372\\003\\376\\377\\006\\372\\372\\377\\001\\376\\006\\000\\005\\374\\372\\376\\375\\002\\005\\007\\376\\005\\373\\375\\003\\000\\377\\373\\374\\000\\004\\376\\374\\374\\002\\000\\372\\004\\372\\373\\005\\006\\373\\005\\004\\004\\004\\004\\001\\375\\005\\004\\377\\007\\003\\002\\375\\371\\004\\372\\001\\372\\005\\000\\372\\002\\000\\002\\376\\001\\376\\005\\006\\371\\002\\002\\004\\002\\003\\376\\373\\002\\377\\004\\376\\373\\003\\373\\371\\006\\007\\003\\374\\375\\372\\000\\003\\374\\373\\375\\373\\376\\374\\001\\001\\000\\372\\005\\000\\006\\375\\006\\376\\376\\371\\004\\372\\007\\007\\375\\002\\006\\002\\377\\004\\374\\375\\376\\005\\003\\005\\371\\003\\005\\003\\373\\006\\004\\374\\007\\000\\007\\373\\005\\004\\371\\004\\000\\375\\001\\375\\004\\374\\002\\374\\001\\375\\000\\005\\377\\373\\372\\004\\006\\376\\000\\000\\002\\004\\002\\004\\374\\002\\005\\007\\007\\003\\003\\373\\002\\376\\005\\005\\373\\376\\005\\003\\002\\375\\372\\002\\374\\377\\003\\376\\005\\005\\001\\373\\371\\005\\006\\005\\000\\375\\002\\004\\001\\006\\001\\006\\001\\000\\003\\377\\000\\006\\006\\006\\377\\006\\375\\375\\372\\377\\373\\376\\000\\000\\376\\005\\372\\371\\377\\371\\374\\376\\001\\000\\003\\006\\002\\375\\002\\005\\005\\002\\006\\006\\376\\373\\373\\000\\001\\376\\003\\003\\006\\376\\004\\376\\377\\375\\377\\006\\375\\007\\003\\375\\373\\005\\005\\000\\376\\001\\007\\001\\007\\372\\003\\373\\373\\372\\005\\377\\374\\377\\005\\005\\376\\007\\001\\376\\002\\000\\007\\377\\003\\373\\000\\003\\003\\003\\001\\373\\377\\373\\000\\000\\377\\000\\000\\004\\376\\003\\007\\374\\374\\375\\002\\001\\005\\006\\002\\371\\003\\005\\377\\372\\373\\376\\374\\377\\372\\371\\007\\002\\007\\374\\005\\001\\005\\375\\004\\371\\000\\004\\004\\376\\377\\376\\004\\376\\372\\372\\375\\001\\376\\372\\375\\000\\376\\375\\373\\004\\375\\373\\006\\372\\003\\375\\005\\006\\005\\001\\003\\372\\374\\005\\376\\003\\376\\375\\006\\377\\001\\371\\007\\376\\373\\377\\000\\377\\002\\005\\371\\372\\000\\006\\002\\005\\377\\372\\002\\377\\374\\375\\377\\004\\377\\000\\005\\375\\003\\371\\002\\376\\001\\377\\377\\377\\371\\377\\374\\375\\375\\000\\377\\002\\375\\376\\375\\001\\002\\005\\003\\006\\003\\006\\000\\005\\374\\000\\003\\005\\007\\006\\376\\002\\373\\377\\377\\003\\002\\005\\004\\004\\376\\001\\377\\006\\005\\001\\376\\003\\374\\006\\375\\377\\004\\007\\004\\000\\000\\376\\004\\377\\001\\004\\373\\372\\006\\375\\377\\374\\005\\006\\005\\004\\371\\372\\377\\000\\003\\374\\375\\372\\372\\004\\002\\005\\373\\372\\003\\371\\004\\005\\375\\373\\000\\005\\374\\001\\003\\004\\005\\001\\005\\005\\375\\371\\371\\375\\373\\004\\374\\000\\376\\001\\002\\371\\005\\001\\374\\000\\373\\002\\375\\004\\373\\000\\001\\003\\006\\002\\006\\371\\373\\001\\004\\004\\376\\375\\004\\376\\003\\006\\001\\375\\000\\006\\002\\001\\005\\000\\000\\005\\373\\004\\373\\374\\002\\001\\374\\003\\005\\000\\006\\376\\374\\371\\006\\003\\377\\376\\375\\376\\002\\374\\374\\374\\373\\375\\374\\004\\007\\376\\003\\003\\374\\374\\377\\004\\375\\002\\377\\372\\001\\006\\007\\005\\004\\375\\003\\377\\372\\001\\006\\004\\375\\373\\000\\377\\377\\003\\002\\004\\377\\373\\000\\372\\376\\000\\001\\003\\376\\372\\002\\005\\375\\002\\376\\000\\377\\003\\372\\004\\005\\373\\373\\001\\374\\377\\005\\003\\376\\372\\003\\002\\002\\001\\374\\003\\373\\375\\005\\003\\374\\004\\375\\375\\371\\377\\005\\001\\000\\004\\372\\374\\000\\003\\373\\377\\005\\373\\003\\006\\376\\373\\001\\374\\373\\000\\004\\002\\373\\373\\373\\002\\372\\002\\006\\001\\000\\002\\000\\004\\372\\376\\371\\373\\377\\372\\001\\000\\375\\000\\007\\004\\371\\372\\374\\003\\375\\003\\371\\000\\377\\004\\373\\373\\374\\375\\007\\001\\373\\376\\376\\375\\374\\376\\000\\006\\371\\001\\006\\002\\001\\372\\372\\006\\003\\007\\006\\004\\003\\373\\001\\376\\001\\002\\001\\005\\005\\002\\374\\372\\373\\371\\006\\002\\004\\376\\004\\376\\003\\003\\372\\374\\003\\377\\004\\371\\372\\376\\371\\000\\004\\001\\003\\376\\005\\376\\371\\373\\004\\006\\374\\002\\376\\004\\001\\371\\006\\003\\000\\007\\000\\002\\003\\375\\374\\006\\005\\374\\375\\372\\005\\372\\004\\374\\004\\006\\002\\003\\001\\000\\372\\000\\372\\376\\375\\006\\376\\000\\372\\375\\375\\003\\007\\377\\005\\006\\373\\375\\372\\371\\377\\003\\372\\375\\000\\000\\005\\373\\005\\375\\000\\373\\004\\005\\007\\006\\003\\375\\002\\375\\372\\377\\376\\375\\372\\006\\375\\372\\373\\003\\375\\376\\377\\006\\375\\006\\373\\005\\003\\377\\375\\377\\007\\372\\001\\373\\006\\001\\373\\004\\005\\373\\374\\006\\003\\003\\005\\001\\004\\377\\374\\373\\003\\007\\002\\002\\372\\377\\004\\377\\374\\000\\001\\376\\005\\376\\002\\372\\006\\373\\004\\374\\376\\007\\002\\007\\006\\373\\000\\375\\375\\000\\372\\005\\377\\374\\376\\377\\374\\006\\001\\376\\001\\377\\000\\373\\371\\376\\371\\003\\002\\004\\000\\004\\001\\007\\375\\003\\001\\372\\005\\000\\006\\002\\372\\006\\004\\375\\005\\372\\372\\000\\376\\000\\374\\376\\006\\374\\371\\005\\001\\003\\002\\005\\375\\375\\372\\375\\375\\372\\007\\375\\377\\002\\377\\373\\375\\377\\004\\374\\006\\372\\005\\374\\374\\001\\005\\004\\001\\006\\002\\003\\375\\373\\000\\004\\005\\000\\374\\003\\003\\006\\376\\374\\004\\003\\374\\000\\002\\375\\003\\377\\375\\377\\374\\006\\373\\005\\007\\372\\377\\371\\377\\003\\371\\377\\001\\376\\002\\006\\002\\375\\000\\003\\371\\002\\007\\376\\001\\373\\006\\002\\375\\004\\002\\003\\377\\372\\371\\001\\376\\001\\003\\002\\373\\004\\375\\373\\006\\374\\373\\377\\003\\374\\372\\375\\371\\004\\002\\373\\006\\000\\003\\001\\374\\002\\000\\004\\375\\374\\373\\377\\002\\000\\376\\372\\377\\375\\373\\371\\003\\372\\003\\374\\004\\003\\005\\375\\372\\004\\377\\004\\006\\007\\377\\006\\001\\375\\001\\005\\006\\372\\376\\005\\006\\002\\375\\006\\377\\377\\001\\000\\373\\006\\006\\373\\375\\003\\001\\006\\001\\374\\004\\377\\373\\377\\004\\372\\001\\003\\006\\001\\373\\000\\374\\376\\376\\372\\375\\373\\371\\371\\003\\371\\007\\375\\376\\004\\374\\377\\004\\377\\006\\004\\000\\004\\000\\000\\374\\375\\006\\377\\376\\005\\005\\003\\007\\001\\002\\374\\003\\000\\005\\006\\374\\376\\375\\375\\374\\007\\374\\000\\001\\000\\000\\375\\377\\377\\373\\007\\003\\005\\002\\004\\006\\373\\005\\000\\373\\001\\003\\374\\372\\371\\374\\372\\004\\375\\372\\373\\377\\004\\005\\002\\004\\373\\376\\005\\374\\002\\376\\004\\372\\001\\374\\373\\005\\375\\000\\001\\373\\004\\374\\375\\371\\377\\002\\002\\003\\375\\003\\376\\007\\374\\377\\000\\005\\375\\373\\002\\005\\372\\376\\376\\005\\001\\372\\006\\376\\372\\007\\377\\002\\000\\004\\375\\375\\374\\374\\374\\005\\373\\000\\000\\007\\376\\375\\002\\001\\002\\005\\373\\001\\377\\372\\377\\002\\004\\000\\373\\376\\004\\003\\373\\001\\377\\001\\005\\374\\377\\005\\377\\007\\000\\001\\377\\002\\376\\007\\006\\000\\374\\002\\377\\372\\001\\373\\376\\373\\005\\003\\374\\005\\373\\001\\372\\004\\374\\375\\002\\371\\004\\000\\373\\377\\374\\002\\003\\372\\005\\375\\006\\377\\002\\375\\000\\377\\376\\374\\000\\372\\374\\375\\000\\374\\375\\001\\000\\000\\374\\005\\376\\373\\004\\001\\000\\372\\374\\004\\376\\373\\004\\372\\004\\376\\003\\001\\007\\003\\000\\007\\372\\001\\002\\373\\374\\375\\374\\377\\374\\007\\000\\007\\002\\377\\001\\374\\373\\003\\377\\001\\377\\005\\371\\374\\374\\376\\006\\005\\373\\376\\006\\373\\001\\003\\003\\005\\005\\002\\371\\375\\377\\373\\374\\001\\375\\001\\003\\004\\373\\006\\003\\001\\376\\376\\000\\377\\371\\001\\373\\372\\004\\377\\376\\377\\375\\004\\000\\375\\007\\005\\006\\371\\001\\001\\006\\002\\002\\005\\003\\372\\371\\374\\377\\002\\005\\372\\377\\373\\006\\004\\004\\377\\374\\374\\000\\000\\000\\006\\374\\005\\003\\373\\375\\374\\000\\004\\004\\377\\006\\003\\001\\372\\372\\004\\004\\375\\373\\000\\372\\005\\004\\002\\373\\377\\375\\003\\007\\000\\376\\005\\376\\375\\372\\001\\006\\005\\003\\005\\372\\007\\376\\371\\003\\002\\376\\371\\000\\375\\374\\006\\003\\372\\375\\375\\372\\373\\007\\000\\006\\001\\007\\374\\002\\375\\002\\005\\005\\374\\000\\001\\000\\375\\002\\372\\376\\374\\375\\374\\004\\376\\003\\000\\000\\006\\004\\007\\372\\007\\375\\000\\001\\371\\003\\374\\006\\377\\373\\005\\000\\373\\001\\377\\371\\002\\375\\377\\006\\000\\373\\005\\373\\003\\002\\006\\372\\375\\002\\376\\373\\001\\003\\000\\006\\372\\005\\007\\375\\377\\374\\371\\001\\371\\007\\006\\376\\001\\374\\006\\000\\002\\376\\000\\373\\377\\003\\004\\372\\003\\003\\377\\003\\372\\375\\376\\375\\000\\375\\372\\374\\001\\001\\006\\373\\004\\005\\375\\376\\373\\377\\000\\372\\006\\371\\375\\001\\007\\000\\003\\000\\001\\375\\373\\004\\005\\006\\375\\001\\006\\005\\004\\000\\002\\006\\002\\376\\000\\373\\372\\375\\376\\005\\374\\372\\377\\003\\374\\371\\373\\377\\003\\007\\375\\002\\000\\373\\000\\373\\003\\007\\371\\371\\001\\005\\000\\372\\376\\372\\000\\005\\005\\374\\007\\002\\376\\005\\373\\376\\006\\001\\375\\373\\373\\375\\375\\377\\003\\005\\000\\005\\006\\003\\005\\376\\002\\003\\376\\371\\373\\374\\377\\375\\005\\374\\006\\005\\373\\374\\002\\001\\373\\000\\000\\377\\004\\002\\000\\004\\005\\375\\371\\373\\003\\001\\004\\376\\006\\376\\377\\373\\372\\002\\006\\371\\004\\372\\005\\373\\004\\374\\375\\376\\375\\006\\001\\001\\376\\005\\000\\372\\005\\373\\004\\376\\003\\001\\004\\006\\376\\002\\001\\372\\004\\001\\000\\377\\375\\375\\003\\007\\375\\376\\377\\372\\000\\372\\001\\005\\005\\000\\007\\003\\377\\004\\374\\000\\005\\001\\002\\372\\374\\372\\373\\002\\000\\000\\374\\006\\371\\000\\006\\001\\006\\375\\002\\003\\007\\000\\372\\371\\004\\374\\372\\376\\000\\004\\006\\001\\000\\000\\003\\001\\006\\376\\374\\001\\002\\001\\003\\375\\001\\375\\377\\374\\002\\371\\002\\376\\000\\372\\375\\004\\372\\005\\374\\000\\000\\000\\000\\000\\001\\000\\005\\377\\005\\003\\000\\373\\372\\006\\004\\371\\377\\004\\006\\000\\377\\006\\375\\006\\003\\003\\374\\376\\373\\005\\373\\003\\377\\377\\002\\376\\005\\377\\377\\372\\006\\373\\004\\004\\373\\005\\001\\001\\376\\000\\377\\373\\005\\007\\006\\004\\000\\006\\004\\372\\377\\372\\373\\373\\371\\004\\001\\375\\375\\003\\372\\377\\000\\374\\375\\007\\006\\000\\374\\006\\003\\005\\000\\001\\375\\372\\003\\000\\377\\006\\376\\376\\374\\002\\001\\374\\006\\005\\002\\000\\005\\002\\000\\374\\004\\002\\001\\377\\002\\005\\004\\006\\374\\374\\376\\003\\373\\375\\376\\005\\000\\375\\376\\373\\002\\376\\006\\375\\003\\002\\002\\003\\373\\002\\002\\377\\373\\374\\006\\377\\005\\005\\003\\000\\003\\372\\005\\374\\377\\376\\373\\002\\372\\006\\006\\004\\004\\377\\001\\000\\373\\377\\000\\373\\001\\007\\377\\001\\377\\001\\000\\375\\004\\372\\006\\373\\004\\376\\004\\006\\373\\005\\375\\001\\003\\005\\004\\377\\377\\000\\002\\004\\377\\377\\001\\372\\007\\000\\377\\373\\000\\006\\373\\003\\000\\376\\372\\002\\374\\003\\375\\372\\003\\004\\372\\001\\376\\372\\371\\006\\375\\005\\004\\376\\372\\373\\004\\005\\001\\002\\001\\003\\371\\003\\372\\003\\003\\000\\377\\001\\001\\372\\007\\376\\002\\374\\006\\371\\003\\372\\001\\373\\375\\376\\004\\374\\375\\377\\373\\007\\372\\003\\002\\004\\374\\376\\005\\375\\001\\003\\375\\002\\006\\376\\377\\006\\377\\003\\377\\372\\002\\004\\375\\373\\375\\006\\377\\004\\004\\000\\004\\001\\372\\003\\001\\005\\003\\377\\375\\001\\001\\000\\006\\373\\007\\377\\000\\000\\004\\005\\001\\375\\001\\376\\376\\000\\371\\002\\375\\006\\371\\004\\372\\372\\006\\372\\371\\377\\377\\004\\374\\000\\006\\000\\001\\375\\003\\375\\001\\377\\007\\377\\371\\377\\372\\375\\004\\006\\000\\375\\375\\001\\375\\002\\372\\005\\373\\004\\002\\374\\372\\002\\372\\376\\006\\001\\001\\372\\377\\003\\002\\005\\373\\006\\005\\007\\002\\004\\377\\375\\004\\002\\371\\007\\372\\005\\007\\007\\376\\375\\373\\376\\002\\002\\001\\373\\376\\005\\002\\377\\005\\374\\004\\003\\375\\004\\374\\003\\007\\372\\005\\372\\372\\375\\000\\000\\374\\006\\372\\002\\376\\376\\374\\004\\377\\372\\376\\007\\002\\377\\002\\372\\003\\374\\373\\004\\376\\007\\006\\000\\376\\000\\006\\003\\374\\003\\376\\375\\002\\374\\000\\371\\374\\005\\377\\006\\373\\000\\007\\375\\377\\373\\003\\374\\003\\375\\001\\007\\373\\371\\375\\372\\000\\376\\001\\376\\373\\002\\003\\001\\372\\373\\374\\375\\375\\372\\371\\003\\372\\374\\372\\377\\001\\373\\003\\372\\001\\000\\376\\372\\002\\376\\001\\373\\372\\000\\000\\377\\377\\375\\000\\372\\376\\002\\373\\375\\002\\002\\376\\371\\373\\002\\371\\373\\002\\377\\001\\005\\373\\377\\001\\374\\001\\004\\373\\000\\374\\377\\006\\006\\377\\007\\377\\003\\376\\372\\005\\006\\005\\372\\373\\003\\000\\377\\001\\002\\006\\007\\005\\372\\006\\374\\004\\003\\001\\377\\000\\006\\000\\006\\006\\374\\000\\000\\007\\005\\004\\372\\004\\003\\372\\376\\375\\002\\000\\374\\004\\001\\002\\001\\371\\001\\373\\002\\376\\005\\005\\001\\002\\003\\003\\006\\375\\005\\003\\372\\375\\375\\374\\372\\375\\376\\002\\002\\375\\001\\007\\372\\372\\002\\373\\376\\372\\371\\373\\375\\000\\005\\000\\002\\373\\373\\376\\003\\374\\374\\375\\375\\006\\006\\003\\373\\376\\002\\001\\007\\377\\006\\375\\001\\003\\007\\000\\001\\373\\372\\376\\001\\374\\371\\005\\372\\376\\000\\374\\005\\005\\004\\004\\005\\372\\375\\001\\374\\006\\376\\000\\375\\003\\000\\003\\006\\376\\006\\372\\372\\375\\005\\375\\003\\006\\001\\376\\001\\002\\000\\377\\002\\376\\375\\377\\002\\005\\005\\004\\371\\004\\377\\006\\002\\006\\001\\000\\005\\373\\001\\002\\001\\003\\372\\374\\005\\374\\377\\001\\001\\007\\005\\377\\002\\006\\001\\374\\001\\373\\374\\373\\375\\373\\375\\001\\002\\000\\004\\006\\003\\006\\002\\377\\373\\375\\376\\373\\004\\000\\006\\377\\004\\000\\375\\371\\372\\001\\373\\374\\006\\372\\002\\377\\000\\376\\004\\372\\375\\003\\377\\006\\000\\372\\376\\002\\000\\007\\375\\005\\003\\377\\000\\001\\002\\375\\005\\375\\375\\377\\006\\376\\374\\007\\000\\373\\004\\006\\002\\001\\373\\373\\374\\004\\376\\007\\005\\373\\006\\002\\373\\002\\000\\001\\005\\372\\375\\374\\375\\375\\002\\376\\004\\373\\002\\375\\002\\004\\375\\371\\007\\374\\003\\376\\005\\372\\005\\375\\371\\006\\000\\002\\002\\375\\376\\005\\004\\002\\373\\004\\376\\377\\003\\376\\376\\007\\371\\001\\006\\000\\004\\004\\002\\002\\374\\002\\374\\000\\372\\005\\376\\375\\374\\002\\005\\006\\003\\005\\005\\006\\005\\003\\007\\002\\005\\372\\003\\371\\373\\001\\371\\372\\003\\002\\376\\003\\007\\007\\376\\372\\373\\374\\376\\002\\005\\003\\374\\002\\000\\372\\373\\377\\000\\006\\375\\000\\004\\002\\372\\373\\007\\373\\372\\372\\003\\377\\375\\373\\002\\374\\005\\371\\003\\377\\004\\006\\003\\372\\002\\372\\376\\007\\001\\005\\372\\002\\373\\000\\371\\000\\003\\003\\006\\001\\374\\000\\372\\005\\372\\003\\005\\003\\004\\002\\004\\005\\004\\373\\374\\001\\003\\000\\001\\000\\376\\005\\005\\005\\001\\004\\374\\000\\375\\375\\002\\002\\000\\374\\004\\006\\377\\374\\372\\376\\376\\374\\000\\004\\002\\006\\373\\375\\371\\372\\001\\000\\003\\002\\377\\371\\002\\374\\372\\006\\000\\001\\373\\374\\003\\372\\371\\375\\373\\000\\374\\373\\002\\372\\000\\372\\004\\377\\006\\372\\372\\000\\374\\377\\374\\007\\004\\006\\376\\371\\373\\007\\006\\006\\372\\375\\376\\372\\375\\371\\377\\373\\376\\002\\006\\375\\371\\374\\373\\005\\377\\003\\005\\374\\377\\375\\004\\372\\376\\003\\376\\374\\006\\373\\374\\002\\007\\007\\374\\005\\375\\004\\002\\001\\373\\371\\375\\004\\007\\375\\005\\000\\373\\000\\005\\003\\373\\372\\006\\002\\371\\005\\372\\002\\006\\375\\003\\372\\376\\000\\376\\004\\000\\376\\005\\004\\001\\002\\371\\371\\003\\003\\000\\377\\001\\372\\004\\006\\007\\375\\375\\375\\003\\371\\371\\004\\005\\003\\372\\004\\005\\374\\003\\375\\376\\374\\007\\002\\002\\377\\377\\006\\006\\007\\371\\002\\374\\372\\375\\372\\375\\374\\001\\006\\003\\002\\372\\376\\005\\006\\006\\004\\376\\377\\000\\007\\002\\372\\376\\377\\004\\375\\374\\006\\376\\003\\004\\000\\003\\374\\006\\374\\375\\377\\377\\006\\371\\004\\005\\002\\004\\375\\374\\001\\002\\377\\374\\003\\377\\001\\374\\377\\373\\003\\003\\376\\002\\001\\006\\376\\375\\004\\376\\003\\377\\001\\007\\371\\376\\002\\003\\003\\006\\372\\001\\376\\372\\371\\004\\003\\002\\376\\375\\003\\000\\003\\001\\004\\374\\376\\376\\374\\372\\373\\001\\003\\372\\372\\003\\372\\004\\000\\006\\374\\000\\375\\374\\000\\375\\000\\373\\376\\375\\003\\006\\004\\004\\001\\005\\005\\375\\373\\005\\002\\004\\373\\006\\002\\001\\006\\002\\003\\374\\005\\007\\006\\001\\004\\007\\007\\006\\004\\006\\372\\373\\006\\002\\377\\377\\374\\004\\003\\006\\001\\007\\372\\005\\007\\000\\005\\373\\003\\376\\002\\374\\375\\001\\007\\373\\004\\000\"\n", " }\n", " type: TENSOR\n", " }\n", @@ -528,7 +599,7 @@ " name: \"value\"\n", " t {\n", " data_type: 1\n", - " raw_data: \"\\263-\\341<\"\n", + " raw_data: \"=3\\341<\"\n", " }\n", " type: TENSOR\n", " }\n", @@ -542,7 +613,7 @@ " t {\n", " dims: 128\n", " data_type: 6\n", - " raw_data: \"\\271\\377\\377\\377\\032\\003\\000\\0009\\001\\000\\000\\302\\002\\000\\000;\\375\\377\\377\\031\\000\\000\\000\\024\\003\\000\\000d\\003\\000\\000\\327\\374\\377\\377\\363\\377\\377\\377u\\003\\000\\000\\374\\000\\000\\000t\\000\\000\\000\\321\\002\\000\\000\\236\\377\\377\\377\\241\\377\\377\\377\\237\\375\\377\\377\\010\\000\\000\\000\\350\\002\\000\\000}\\376\\377\\377\\267\\377\\377\\377\\374\\000\\000\\000\\355\\001\\000\\000N\\375\\377\\377\\\\\\002\\000\\000\\346\\002\\000\\000\\317\\000\\000\\000\\207\\001\\000\\000?\\000\\000\\000\\302\\002\\000\\000Y\\377\\377\\377\\326\\376\\377\\377\\\\\\003\\000\\000\\374\\376\\377\\377\\334\\000\\000\\000\\200\\001\\000\\000\\362\\377\\377\\377+\\000\\000\\000\\304\\375\\377\\377u\\000\\000\\000\\340\\000\\000\\000\\275\\001\\000\\000\\324\\377\\377\\377\\332\\000\\000\\000\\026\\001\\000\\000\\333\\001\\000\\000\\371\\375\\377\\377\\363\\000\\000\\000|\\002\\000\\000\\335\\376\\377\\377\\226\\375\\377\\377\\335\\002\\000\\0002\\001\\000\\000F\\377\\377\\377\\006\\003\\000\\000\\310\\375\\377\\377\\344\\377\\377\\377\\177\\376\\377\\377>\\001\\000\\000\\033\\002\\000\\000I\\003\\000\\000\\006\\376\\377\\377\\315\\375\\377\\377\\033\\003\\000\\000\\236\\000\\000\\000@\\376\\377\\377\\031\\002\\000\\000\\321\\002\\000\\000;\\000\\000\\000\\035\\377\\377\\377\\354\\377\\377\\377Z\\001\\000\\000N\\375\\377\\377I\\001\\000\\000\\030\\001\\000\\000w\\377\\377\\377\\303\\002\\000\\000\\022\\000\\000\\000\\377\\001\\000\\000!\\000\\000\\000\\035\\001\\000\\000\\003\\375\\377\\377^\\377\\377\\377\\336\\374\\377\\377p\\377\\377\\377\\351\\002\\000\\000X\\376\\377\\377\\247\\000\\000\\000H\\376\\377\\377}\\000\\000\\000\\225\\374\\377\\3776\\001\\000\\000\\301\\001\\000\\000\\210\\001\\000\\000\\374\\376\\377\\377\\307\\377\\377\\377\\320\\374\\377\\377\\267\\377\\377\\377F\\375\\377\\377\\352\\377\\377\\377=\\377\\377\\3770\\376\\377\\377#\\000\\000\\000\\313\\376\\377\\377\\334\\000\\000\\000\\261\\001\\000\\000\\363\\001\\000\\000\\037\\001\\000\\000\\220\\377\\377\\377\\202\\000\\000\\000d\\377\\377\\377\\013\\002\\000\\000\\266\\002\\000\\000\\347\\374\\377\\377+\\001\\000\\000\\301\\376\\377\\377\\341\\377\\377\\377O\\003\\000\\000\\037\\375\\377\\377\\244\\375\\377\\377\\352\\000\\000\\000\\302\\001\\000\\000I\\002\\000\\000~\\377\\377\\377*\\376\\377\\377\\333\\000\\000\\000\\214\\000\\000\\000\\014\\002\\000\\000\"\n", + " raw_data: \"[\\002\\000\\0000\\002\\000\\000\\020\\002\\000\\000\\204\\002\\000\\000\\010\\002\\000\\000\\206\\377\\377\\377A\\003\\000\\000J\\003\\000\\000H\\377\\377\\377\\321\\001\\000\\000\\277\\376\\377\\377\\324\\000\\000\\000\\332\\002\\000\\000\\t\\002\\000\\000\\r\\376\\377\\377\\030\\003\\000\\000\\013\\001\\000\\000\\010\\002\\000\\000b\\001\\000\\000\\000\\002\\000\\000\\224\\000\\000\\000\\357\\377\\377\\377a\\375\\377\\377H\\003\\000\\000\\240\\377\\377\\377k\\000\\000\\000+\\375\\377\\377^\\002\\000\\000z\\003\\000\\000\\247\\374\\377\\377\\255\\374\\377\\377\\335\\001\\000\\000\\213\\375\\377\\377!\\002\\000\\000\\250\\001\\000\\000\\245\\376\\377\\377{\\003\\000\\000F\\375\\377\\377\\375\\377\\377\\377[\\003\\000\\000\\034\\375\\377\\377\\201\\002\\000\\000\\375\\002\\000\\000\\200\\000\\000\\000e\\002\\000\\000k\\001\\000\\000\\335\\000\\000\\0002\\377\\377\\377\\300\\375\\377\\377O\\377\\377\\377t\\003\\000\\000\\233\\002\\000\\000\\257\\001\\000\\000\\305\\000\\000\\000\\217\\374\\377\\377[\\377\\377\\377X\\377\\377\\377\\223\\377\\377\\377\\222\\000\\000\\000x\\376\\377\\377\\246\\374\\377\\3772\\003\\000\\000\\002\\377\\377\\377\\327\\374\\377\\377\\267\\002\\000\\000r\\376\\377\\377\\203\\002\\000\\000\\321\\002\\000\\000\\243\\002\\000\\000~\\377\\377\\377\\326\\377\\377\\377/\\001\\000\\000\\217\\000\\000\\000/\\375\\377\\377\\341\\001\\000\\000\\031\\375\\377\\377\\222\\377\\377\\377\\037\\000\\000\\000\\005\\001\\000\\000)\\003\\000\\000\\206\\000\\000\\000K\\376\\377\\377\\271\\374\\377\\377\\244\\377\\377\\377\\370\\002\\000\\000F\\375\\377\\377\\201\\002\\000\\000\\\"\\002\\000\\000\\211\\376\\377\\377?\\001\\000\\000\\337\\376\\377\\377p\\001\\000\\000f\\376\\377\\377C\\003\\000\\000;\\376\\377\\377\\224\\377\\377\\377Q\\376\\377\\377\\237\\000\\000\\000\\256\\374\\377\\377)\\003\\000\\000I\\002\\000\\000\\231\\001\\000\\000\\212\\376\\377\\377b\\003\\000\\000B\\377\\377\\377\\361\\376\\377\\377t\\375\\377\\377>\\377\\377\\377\\020\\376\\377\\377-\\000\\000\\000S\\000\\000\\000$\\376\\377\\377G\\377\\377\\377Z\\000\\000\\000g\\000\\000\\000\\036\\003\\000\\000=\\376\\377\\377\\341\\376\\377\\377_\\001\\000\\000A\\376\\377\\377\\033\\001\\000\\000\\202\\376\\377\\377\\306\\000\\000\\000\\022\\000\\000\\000\\341\\001\\000\\000\\354\\001\\000\\000\\346\\002\\000\\000l\\377\\377\\377\"\n", " }\n", " type: TENSOR\n", " }\n", @@ -600,9 +671,9 @@ " name: \"/linear/export_handler/DequantizeLinear\"\n", " op_type: \"DequantizeLinear\"\n", " }\n", - " name: \"torch_jit\"\n", + " name: \"main_graph\"\n", " input {\n", - " name: \"inp.1\"\n", + " name: \"out.1\"\n", " type {\n", " tensor_type {\n", " elem_type: 1\n", @@ -652,7 +723,7 @@ "}" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -661,6 +732,8 @@ "from brevitas.quant.scaled_int import Int8ActPerTensorFloat, Int32Bias\n", "from brevitas.export import export_onnx_qop\n", "\n", + "torch.manual_seed(1)\n", + "\n", "IN_CH = 3\n", "IMG_SIZE = 128\n", "OUT_CH = 128\n", @@ -691,9 +764,12 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" }, @@ -724,10 +800,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -740,6 +816,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -759,6 +838,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -778,6 +860,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -792,14 +877,32 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-15 08:44:36.021678676 [W:onnxruntime:, graph.cc:1283 Graph] Initializer linear.bias appears in graph inputs and will not be treated as constant value/weight. This may prevent some of the graph optimizations, like const folding. Move it out of graph inputs if there is no need to override it, by either re-generating the model with latest exporter/converter or with the tool onnxruntime/tools/python/remove_initializer_from_input.py.\n" + ] + } + ], "source": [ "import onnxruntime as ort\n", "\n", @@ -830,13 +933,16 @@ "out_brevitas = model(inp)\n", "out_ort = torch.tensor(pred_onx)\n", "\n", - "assert torch.allclose(out_brevitas, out_ort)" + "assert_with_message(torch.allclose(out_brevitas, out_ort))" ] }, { "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -862,9 +968,12 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } @@ -886,6 +995,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -898,6 +1010,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -913,16 +1028,26 @@ "execution_count": 13, "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%%\n" } }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\alessand\\documents\\brevitas\\src\\brevitas\\export\\onnx\\standard\\manager.py:23: UserWarning: ONNX opset version set to 13, override with opset_version=\n", + "/scratch/fabian/brevitas/src/brevitas/export/onnx/standard/manager.py:26: UserWarning: ONNX opset version set to 13, override with opset_version=\n", " warnings.warn(f\"ONNX opset version set to {DEFAULT_OPSET}, override with {ka}=\")\n" ] } @@ -962,7 +1087,7 @@ "out_brevitas = model(inp).int()\n", "out_ort = torch.tensor(pred_onx, dtype=torch.int8)\n", "\n", - "assert torch.allclose(out_brevitas, out_ort, atol=1)" + "assert_with_message(torch.allclose(out_brevitas, out_ort, atol=1))" ] }, { @@ -970,6 +1095,9 @@ "cell_type": "markdown", "metadata": { "collapsed": false, + "jupyter": { + "outputs_hidden": false + }, "pycharm": { "name": "#%% md\n" } @@ -983,7 +1111,7 @@ ], "metadata": { "kernelspec": { - "display_name": "pytorch_latest", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -997,9 +1125,8 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.15" + "version": "3.11.5" }, - "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "b6e150ee02c45d2c3f896173a651a21b25567e05411969bcc0f3a62fa15a0a0b" @@ -1007,5 +1134,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/notebooks/quantized_recurrent.ipynb b/notebooks/quantized_recurrent.ipynb index 766e82745..20cfc8fee 100644 --- a/notebooks/quantized_recurrent.ipynb +++ b/notebooks/quantized_recurrent.ipynb @@ -35,15 +35,17 @@ " hidden_size: int,\n", " num_layers: int = 1,\n", " nonlinearity: str = 'tanh',\n", - " bias: bool = True,\n", + " bias: Optional[bool] = True,\n", " batch_first: bool = False,\n", " bidirectional: bool = False,\n", - " weight_quant = Int8WeightPerTensorFloat,\n", - " bias_quant = Int32Bias,\n", - " io_quant = Int8ActPerTensorFloat,\n", - " gate_acc_quant = Int8ActPerTensorFloat,\n", - " shared_input_hidden_weights = False,\n", + " weight_quant=Int8WeightPerTensorFloat,\n", + " bias_quant=Int32Bias,\n", + " io_quant=Int8ActPerTensorFloat,\n", + " gate_acc_quant=Int8ActPerTensorFloat,\n", + " shared_input_hidden_weights=False,\n", " return_quant_tensor: bool = False,\n", + " dtype: Optional[torch.dtype] = None,\n", + " device: Optional[torch.device] = None,\n", " **kwargs):\n", " super(QuantRNN, self).__init__(\n", " layer_impl=_QuantRNNLayer,\n", @@ -60,6 +62,8 @@ " gate_acc_quant=gate_acc_quant,\n", " shared_input_hidden_weights=shared_input_hidden_weights,\n", " return_quant_tensor=return_quant_tensor,\n", + " dtype=dtype,\n", + " device=device,\n", " **kwargs)\n", "\n", "```" @@ -79,6 +83,11 @@ "import torch\n", "torch.manual_seed(0)\n", "\n", + "# helpers\n", + "def assert_with_message(condition):\n", + " assert condition\n", + " print(condition)\n", + "\n", "def pretty_print_source(source):\n", " display(Markdown('```python\\n' + source + '\\n```'))\n", " \n", @@ -107,7 +116,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\alessand\\documents\\brevitas\\src\\brevitas\\nn\\mixin\\base.py:112: UserWarning: Keyword arguments are being passed but they not being used.\n", + "/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", " warn('Keyword arguments are being passed but they not being used.')\n" ] } @@ -278,46 +287,46 @@ "Input-hidden weight bit-width: 4.0\n", "Hidden-hidden weight bit-width: 4.0\n", "I/O quant bit-width: 6.0\n", - "Input-hidden weight scale: tensor([[0.0316],\n", - " [0.0317],\n", - " [0.0319],\n", - " [0.0318],\n", - " [0.0314],\n", + "Input-hidden weight scale: tensor([[0.0297],\n", + " [0.0311],\n", " [0.0298],\n", + " [0.0295],\n", + " [0.0316],\n", + " [0.0311],\n", + " [0.0318],\n", + " [0.0309],\n", " [0.0317],\n", - " [0.0285],\n", - " [0.0306],\n", - " [0.0312],\n", + " [0.0309],\n", + " [0.0316],\n", + " [0.0319],\n", + " [0.0319],\n", " [0.0318],\n", " [0.0315],\n", - " [0.0298],\n", - " [0.0314],\n", - " [0.0293],\n", " [0.0310],\n", - " [0.0306],\n", - " [0.0310],\n", - " [0.0309],\n", - " [0.0317]], grad_fn=)\n", - "Hidden-hidden weight scale: tensor([[0.0316],\n", - " [0.0317],\n", + " [0.0319],\n", " [0.0319],\n", " [0.0318],\n", - " [0.0314],\n", + " [0.0312]], grad_fn=)\n", + "Hidden-hidden weight scale: tensor([[0.0297],\n", + " [0.0311],\n", " [0.0298],\n", + " [0.0295],\n", + " [0.0316],\n", + " [0.0311],\n", + " [0.0318],\n", + " [0.0309],\n", " [0.0317],\n", - " [0.0285],\n", - " [0.0306],\n", - " [0.0312],\n", + " [0.0309],\n", + " [0.0316],\n", + " [0.0319],\n", + " [0.0319],\n", " [0.0318],\n", " [0.0315],\n", - " [0.0298],\n", - " [0.0314],\n", - " [0.0293],\n", - " [0.0310],\n", - " [0.0306],\n", " [0.0310],\n", - " [0.0309],\n", - " [0.0317]], grad_fn=)\n" + " [0.0319],\n", + " [0.0319],\n", + " [0.0318],\n", + " [0.0312]], grad_fn=)\n" ] } ], @@ -387,52 +396,52 @@ "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\alessand\\documents\\brevitas\\src\\brevitas\\nn\\mixin\\base.py:343: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at C:\\cb\\pytorch_1000000000000\\work\\torch\\csrc\\utils\\python_arg_parser.cpp:354.)\n", + "/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:312: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/torch/csrc/utils/python_arg_parser.cpp:368.)\n", " return torch.cat(outputs, dim=seq_dim)\n" ] }, { "data": { "text/plain": [ - "(QuantTensor(value=tensor([[[-0.4458, -0.1651, -0.7045, -0.5889, -0.2532, -0.0330, -0.1651,\n", - " 0.1706, 0.1376, 0.4348, 0.5834, -0.3577, -0.2807, 0.1046,\n", - " 0.2532, 0.2807, 0.2532, -0.4293, 0.1376, -0.1486],\n", - " [-0.1569, 0.3530, -0.6995, -0.0458, -0.5295, -0.3007, -0.7257,\n", - " 0.2877, -0.1308, 0.6603, 0.0196, -0.8237, 0.0065, -0.4380,\n", - " -0.2615, 0.3138, -0.0850, 0.0065, 0.0458, -0.1961],\n", - " [ 0.1929, -0.5981, -0.2508, -0.2251, -0.5917, 0.2251, 0.0257,\n", - " 0.2508, -0.3023, 0.2830, 0.3344, -0.4309, -0.0836, 0.2701,\n", - " 0.3666, -0.1351, 0.1736, -0.0257, 0.1286, -0.6174],\n", - " [ 0.4682, -0.1804, 0.2780, 0.4974, 0.4389, -0.0585, -0.6242,\n", - " -0.0098, 0.2341, 0.3511, -0.2926, -0.4925, 0.1414, -0.4633,\n", - " -0.0683, 0.2633, 0.3804, 0.3024, 0.1951, 0.1707],\n", - " [-0.0852, 0.0965, -0.4656, -0.3180, -0.3464, -0.2782, -0.1931,\n", - " -0.6360, -0.3180, -0.3293, 0.7211, 0.4316, 0.4145, -0.3066,\n", - " -0.5224, -0.3066, -0.5849, -0.7211, 0.3293, 0.1420]],\n", + "(QuantTensor(value=tensor([[[-0.0062, -0.2872, 0.7931, 0.4309, 0.5495, -0.4558, 0.2373,\n", + " 0.6807, 0.4621, 0.6120, -0.1124, 0.3872, 0.3060, 0.7681,\n", + " -0.3684, 0.0437, -0.7369, -0.3247, 0.7743, 0.3372],\n", + " [ 0.5450, 0.2962, -0.3969, 0.3555, -0.5628, 0.2429, -0.4976,\n", + " 0.1777, -0.1244, 0.0296, -0.2607, 0.0948, 0.5036, -0.3673,\n", + " 0.5213, -0.2962, 0.7524, 0.0770, -0.0948, -0.0948],\n", + " [ 0.2691, -0.6624, -0.5434, 0.4968, -0.6624, 0.0983, 0.1345,\n", + " 0.1242, -0.0517, -0.3726, 0.3053, 0.1604, 0.3208, 0.0983,\n", + " 0.3105, 0.4243, 0.2794, 0.1604, 0.1035, -0.0724],\n", + " [ 0.1284, -0.3337, -0.5263, -0.0449, -0.5263, 0.3081, -0.1733,\n", + " 0.5648, 0.4942, -0.1412, 0.1733, 0.3337, 0.6225, 0.3401,\n", + " 0.5070, -0.1412, 0.0642, -0.3722, 0.2888, 0.1155],\n", + " [ 0.0579, -0.0058, -0.4054, -0.1564, -0.5560, -0.3301, 0.3533,\n", + " 0.0058, -0.1622, -0.3765, 0.1216, 0.0695, -0.4054, 0.0927,\n", + " 0.6139, -0.1390, 0.7066, 0.1274, 0.1622, -0.2896]],\n", " \n", - " [[ 0.5669, 0.2367, -0.3027, -0.3137, -0.3632, -0.1651, -0.5999,\n", - " 0.2036, 0.4293, 0.2201, -0.2862, -0.3908, -0.2091, -0.2532,\n", - " -0.2532, -0.5834, -0.2697, 0.0055, 0.2532, 0.1761],\n", - " [ 0.1242, 0.4184, -0.6472, -0.0196, -0.4707, -0.5034, -0.8368,\n", - " 0.3530, 0.1504, 0.0458, -0.0654, -0.7714, -0.1961, -0.4903,\n", - " -0.6015, -0.3596, -0.2484, -0.4380, -0.0458, 0.2942],\n", - " [ 0.3409, 0.8168, -0.7396, 0.2958, 0.2508, -0.1286, -0.1286,\n", - " 0.7782, -0.1994, 0.7846, -0.3087, -0.3666, 0.1029, 0.1479,\n", - " -0.3216, -0.1479, -0.2315, 0.4566, 0.5209, -0.3344],\n", - " [-0.0878, 0.0390, -0.1707, -0.1365, -0.2243, -0.2390, -0.3706,\n", - " 0.1609, -0.5511, -0.4096, 0.5121, -0.5901, 0.2633, -0.3609,\n", - " -0.5511, 0.3755, -0.4925, -0.0293, -0.0780, -0.2829],\n", - " [ 0.0965, -0.1987, 0.0057, 0.1306, 0.3861, 0.2839, -0.3861,\n", - " 0.5962, -0.1987, 0.3180, -0.1647, -0.3066, -0.0227, 0.4372,\n", - " 0.0852, 0.3748, 0.0852, -0.0057, -0.1703, -0.0738]]],\n", - " grad_fn=), scale=tensor(0.0058, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)),\n", - " QuantTensor(value=tensor([[[-0.0852, 0.0965, -0.4656, -0.3180, -0.3464, -0.2782, -0.1931,\n", - " -0.6360, -0.3180, -0.3293, 0.7211, 0.4316, 0.4145, -0.3066,\n", - " -0.5224, -0.3066, -0.5849, -0.7211, 0.3293, 0.1420],\n", - " [ 0.0965, -0.1987, 0.0057, 0.1306, 0.3861, 0.2839, -0.3861,\n", - " 0.5962, -0.1987, 0.3180, -0.1647, -0.3066, -0.0227, 0.4372,\n", - " 0.0852, 0.3748, 0.0852, -0.0057, -0.1703, -0.0738]]],\n", - " grad_fn=), scale=tensor(0.0057, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)))" + " [[ 0.1374, 0.5745, 0.0624, -0.2373, 0.3060, 0.3310, -0.5183,\n", + " 0.1186, 0.1124, 0.2997, 0.0375, 0.6369, -0.5308, 0.6307,\n", + " -0.5683, 0.7556, 0.2997, -0.4933, 0.3934, -0.4871],\n", + " [ 0.1066, -0.1244, -0.1718, 0.4266, 0.5569, 0.0178, 0.1185,\n", + " -0.3910, 0.2133, 0.0178, -0.1066, -0.2903, 0.1837, -0.2547,\n", + " -0.2903, 0.0770, 0.3495, 0.2547, 0.2311, -0.6161],\n", + " [-0.0880, -0.1966, 0.3001, -0.0569, 0.4140, -0.1552, -0.1345,\n", + " 0.4554, 0.5175, 0.1242, -0.2898, 0.1966, -0.0414, 0.3985,\n", + " -0.1708, -0.0621, -0.1708, 0.0828, 0.2225, 0.0517],\n", + " [ 0.2118, 0.5648, -0.2824, -0.0449, 0.5840, 0.3209, -0.5648,\n", + " 0.3530, 0.4043, -0.4942, -0.3786, 0.0257, 0.5327, -0.1990,\n", + " -0.1348, -0.8215, 0.3016, 0.5327, 0.5648, -0.1155],\n", + " [-0.0290, -0.1738, 0.0695, 0.3765, 0.1738, 0.0579, -0.4054,\n", + " -0.2664, 0.4923, 0.2143, -0.4170, 0.4112, 0.5502, 0.7066,\n", + " -0.6024, 0.7356, 0.0348, 0.1043, -0.1911, -0.4518]]],\n", + " grad_fn=), scale=tensor(0.0059, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)),\n", + " QuantTensor(value=tensor([[[ 0.0579, -0.0058, -0.4054, -0.1564, -0.5560, -0.3301, 0.3533,\n", + " 0.0058, -0.1622, -0.3765, 0.1216, 0.0695, -0.4054, 0.0927,\n", + " 0.6139, -0.1390, 0.7066, 0.1274, 0.1622, -0.2896],\n", + " [-0.0290, -0.1738, 0.0695, 0.3765, 0.1738, 0.0579, -0.4054,\n", + " -0.2664, 0.4923, 0.2143, -0.4170, 0.4112, 0.5502, 0.7066,\n", + " -0.6024, 0.7356, 0.0348, 0.1043, -0.1911, -0.4518]]],\n", + " grad_fn=), scale=tensor(0.0058, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)))" ] }, "execution_count": 10, @@ -461,48 +470,56 @@ "execution_count": 11, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/scratch/fabian/miniforge3/envs/torchgpu/lib/python3.11/site-packages/torch/_tensor.py:1362: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at /opt/conda/conda-bld/pytorch_1699449183005/work/c10/core/TensorImpl.h:1900.)\n", + " return super().rename(names)\n" + ] + }, { "data": { "text/plain": [ - "(QuantTensor(value=tensor([[[ 0.1760, 0.2670, -0.1214, -0.3702, 0.3884, 0.4127, 0.0243,\n", - " 0.0425, -0.2246, -0.0910, -0.2670, 0.4734, 0.0971, -0.3824,\n", - " 0.1396, 0.6858, 0.0061, 0.3702, 0.1275, 0.5037],\n", - " [ 0.2831, 0.0566, -0.2831, -0.2661, -0.0793, 0.3511, -0.4926,\n", - " 0.0510, -0.6455, 0.7191, -0.1812, -0.6172, 0.1529, 0.4077,\n", - " -0.7078, -0.0453, -0.0963, 0.4926, -0.4983, -0.4077],\n", - " [ 0.0000, -0.3977, 0.0947, 0.1894, -0.3725, -0.2589, -0.3914,\n", - " 0.3409, -0.0063, 0.2652, -0.5177, -0.4230, -0.0821, -0.0631,\n", - " 0.0505, -0.0189, 0.0253, -0.1578, -0.4988, 0.5556],\n", - " [ 0.4809, 0.8144, -0.6925, 0.4360, 0.0256, -0.4360, -0.5130,\n", - " 0.2501, -0.1347, 0.7631, -0.5386, -0.2437, 0.4296, -0.1988,\n", - " -0.7246, -0.1154, -0.2437, 0.3655, 0.0641, 0.3142],\n", - " [ 0.0706, -0.0192, -0.7185, -0.8211, -0.5709, 0.1155, 0.4683,\n", - " 0.3400, -0.3015, 0.3528, 0.3143, -0.1155, -0.3143, -0.0257,\n", - " 0.1411, -0.2309, 0.5132, 0.3721, 0.5196, -0.5453]],\n", + "(QuantTensor(value=tensor([[[ 0.2111, 0.1267, 0.0060, 0.6153, -0.7721, -0.3740, -0.5188,\n", + " 0.6273, 0.4162, 0.2051, 0.2292, 0.7239, 0.6032, 0.2533,\n", + " 0.5067, 0.6635, 0.1206, -0.5730, 0.0483, 0.3318],\n", + " [ 0.5742, 0.0194, -0.3807, -0.0710, -0.6000, 0.1807, 0.1355,\n", + " 0.4129, 0.3807, 0.3936, -0.0903, 0.1549, 0.1032, 0.0645,\n", + " 0.4775, -0.0645, 0.1161, -0.0065, 0.0194, -0.1097],\n", + " [ 0.0453, -0.4533, 0.1036, -0.0194, -0.2979, 0.3432, 0.0777,\n", + " 0.6346, -0.0842, 0.3302, 0.4727, 0.4856, -0.4144, 0.7382,\n", + " -0.0453, 0.5439, 0.2266, -0.4792, 0.4403, -0.1036],\n", + " [ 0.3198, 0.2741, -0.6395, 0.0971, -0.6052, -0.5196, 0.1770,\n", + " -0.5025, -0.1256, 0.2056, 0.2684, -0.6395, -0.0285, -0.7309,\n", + " 0.7194, -0.7194, 0.1542, -0.3426, -0.6509, 0.0343],\n", + " [ 0.0000, -0.4004, 0.3151, -0.0263, -0.5842, -0.1641, -0.3939,\n", + " 0.0263, -0.2429, 0.6499, -0.5186, 0.1247, -0.2101, 0.8337,\n", + " -0.1444, 0.6762, -0.1641, -0.5317, -0.1707, -0.0197]],\n", " \n", - " [[ 0.4066, -0.7768, 0.6008, 0.0546, 0.0182, 0.1821, 0.0971,\n", - " -0.3763, 0.3520, -0.5037, -0.0061, 0.2246, -0.0486, 0.2124,\n", - " 0.3641, -0.6433, 0.4248, 0.0789, 0.1275, -0.1214],\n", - " [ 0.2321, 0.1982, -0.1302, 0.1529, -0.0736, -0.3567, -0.4360,\n", - " -0.0283, 0.4869, 0.5379, -0.6964, -0.0340, -0.2944, -0.1529,\n", - " -0.2152, -0.4643, 0.3454, 0.3284, -0.3341, 0.5945],\n", - " [-0.2020, 0.0379, -0.8081, -0.7260, -0.0821, 0.0631, 0.4988,\n", - " 0.0694, 0.0253, 0.5430, 0.8018, 0.2273, -0.3472, -0.0505,\n", - " 0.4924, -0.4735, 0.5745, -0.5619, 0.6313, -0.1768],\n", - " [ 0.2501, -0.4360, 0.6541, 0.0385, 0.5835, -0.3078, -0.0449,\n", - " 0.3270, 0.7951, -0.3591, -0.4809, -0.2757, -0.3591, -0.7567,\n", - " 0.5194, 0.2757, 0.7438, 0.7695, 0.5451, 0.4296],\n", - " [ 0.2630, -0.4747, 0.1347, -0.0641, -0.2245, -0.3336, -0.4490,\n", - " -0.4619, -0.1796, -0.5517, 0.3913, 0.0257, -0.2053, -0.2823,\n", - " -0.6992, -0.6607, 0.1989, -0.6928, -0.5581, 0.5966]]],\n", + " [[ 0.2111, -0.2111, -0.3197, -0.0241, -0.5067, -0.0241, -0.2895,\n", + " 0.1749, -0.4283, 0.0000, -0.3680, 0.5308, -0.1267, 0.5248,\n", + " 0.1206, 0.2654, 0.6394, -0.1327, 0.2292, -0.3800],\n", + " [ 0.6775, -0.3355, -0.1807, 0.2774, -0.8259, -0.2000, -0.0065,\n", + " 0.5678, 0.4000, 0.2258, 0.4387, 0.2710, 0.5355, 0.1290,\n", + " 0.6710, -0.0645, -0.2710, -0.3613, 0.6388, 0.5226],\n", + " [-0.0065, -0.0777, -0.6475, -0.1684, -0.3820, 0.3885, 0.0065,\n", + " 0.1943, -0.3238, -0.2525, -0.1230, -0.0453, -0.0777, 0.3432,\n", + " 0.4921, -0.1101, 0.8224, 0.2396, 0.1554, -0.3885],\n", + " [-0.0514, -0.4111, -0.4625, -0.1713, -0.3369, 0.2512, -0.2969,\n", + " -0.4111, -0.2341, 0.3597, -0.1998, 0.0000, 0.2741, 0.7137,\n", + " -0.1256, 0.1370, -0.0742, -0.5938, -0.5424, -0.4168],\n", + " [ 0.3479, 0.5974, -0.3939, 0.1444, -0.6762, 0.1969, -0.6499,\n", + " 0.4136, 0.5383, -0.3085, 0.4070, 0.4070, 0.6630, -0.0263,\n", + " 0.2823, -0.1510, 0.1313, -0.5186, 0.4464, -0.0066]]],\n", " grad_fn=), scale=tensor(0.0062, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)),\n", - " QuantTensor(value=tensor([[[ 0.0706, -0.0192, -0.7185, -0.8211, -0.5709, 0.1155, 0.4683,\n", - " 0.3400, -0.3015, 0.3528, 0.3143, -0.1155, -0.3143, -0.0257,\n", - " 0.1411, -0.2309, 0.5132, 0.3721, 0.5196, -0.5453],\n", - " [ 0.2630, -0.4747, 0.1347, -0.0641, -0.2245, -0.3336, -0.4490,\n", - " -0.4619, -0.1796, -0.5517, 0.3913, 0.0257, -0.2053, -0.2823,\n", - " -0.6992, -0.6607, 0.1989, -0.6928, -0.5581, 0.5966]]],\n", - " grad_fn=), scale=tensor(0.0064, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)))" + " QuantTensor(value=tensor([[[ 0.0000, -0.4004, 0.3151, -0.0263, -0.5842, -0.1641, -0.3939,\n", + " 0.0263, -0.2429, 0.6499, -0.5186, 0.1247, -0.2101, 0.8337,\n", + " -0.1444, 0.6762, -0.1641, -0.5317, -0.1707, -0.0197],\n", + " [ 0.3479, 0.5974, -0.3939, 0.1444, -0.6762, 0.1969, -0.6499,\n", + " 0.4136, 0.5383, -0.3085, 0.4070, 0.4070, 0.6630, -0.0263,\n", + " 0.2823, -0.1510, 0.1313, -0.5186, 0.4464, -0.0066]]],\n", + " grad_fn=), scale=tensor(0.0066, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)))" ] }, "execution_count": 11, @@ -533,45 +550,45 @@ { "data": { "text/plain": [ - "(QuantTensor(value=tensor([[[-0.1984, 0.2499, -0.1102, 0.2499, -0.0955, -0.4630, -0.8672,\n", - " 0.1911, -0.4851, 0.8085, 0.6982, -0.5806, 0.0000, -0.4189,\n", - " -0.7423, -0.4851, -0.9260, -0.0147, 0.0514, -0.1984],\n", - " [-0.2167, 0.5092, -0.3846, 0.0650, 0.6717, -0.2492, -0.0867,\n", - " 0.3142, -0.3900, 0.3521, 0.4767, -0.1137, 0.6879, 0.1733,\n", - " -0.0596, 0.4279, -0.5471, -0.2762, 0.5904, -0.3737],\n", - " [-0.1335, -0.0140, -0.2810, -0.5339, -0.5339, 0.0562, 0.7236,\n", - " -0.1264, -0.0211, -0.3021, -0.1124, 0.4777, 0.3793, 0.2388,\n", - " -0.0702, 0.4847, -0.4988, 0.7236, 0.5901, -0.4847],\n", - " [ 0.3340, -0.5225, -0.1242, 0.1499, 0.3083, -0.1756, -0.1713,\n", - " 0.0000, 0.3512, -0.3041, 0.3126, -0.5482, 0.4882, 0.1028,\n", - " -0.4796, 0.1028, -0.2527, -0.3640, 0.1713, 0.0471],\n", - " [-0.4438, -0.2686, -0.3095, -0.2978, -0.0993, 0.0584, 0.4846,\n", - " -0.0526, 0.3737, -0.4496, 0.1109, 0.7416, -0.0526, 0.3445,\n", - " 0.4963, 0.2803, 0.1927, 0.0000, 0.6131, 0.1109]],\n", + "(QuantTensor(value=tensor([[[-0.3777, -0.2074, 0.7184, 0.9110, 0.0148, -0.1926, -0.7110,\n", + " 0.1926, -0.4222, -0.9480, 0.2592, 0.2222, -0.2370, -0.5407,\n", + " 0.5851, -0.2370, 0.3555, 0.1703, 0.4444, -0.2222],\n", + " [ 0.4814, -0.7355, -0.1605, 0.3878, -0.5282, 0.2073, 0.0000,\n", + " 0.3677, 0.1805, -0.1204, -0.4614, 0.2474, 0.7021, 0.0401,\n", + " 0.4346, 0.4480, -0.3143, 0.0401, 0.6887, 0.6753],\n", + " [ 0.5038, -0.3650, -0.6936, 0.0146, -0.9345, 0.0000, 0.1679,\n", + " -0.3066, 0.1825, 0.4089, 0.0949, -0.2555, 0.3870, -0.2482,\n", + " 0.5914, -0.0803, 0.1314, -0.4235, -0.3797, 0.1168],\n", + " [ 0.1795, 0.1795, 0.0449, 0.0449, 0.2308, 0.0898, -0.1282,\n", + " 0.5579, 0.1731, -0.1795, 0.1603, 0.3142, 0.1090, 0.5835,\n", + " -0.1475, 0.0449, 0.1795, -0.0256, 0.8143, -0.2437],\n", + " [-0.0066, 0.4804, 0.0066, -0.1184, 0.6843, -0.0197, 0.1448,\n", + " 0.1842, 0.6383, -0.1908, -0.0066, -0.1053, -0.1316, 0.0461,\n", + " -0.0066, -0.2764, 0.3751, 0.3619, 0.5001, -0.1316]],\n", " \n", - " [[ 0.1102, -0.8085, 0.5806, -0.0661, 0.3013, 0.2646, 0.2499,\n", - " -0.6321, 0.4557, 0.4777, 0.6321, 0.0294, -0.2646, -0.9407,\n", - " 0.7350, -0.6027, 0.6174, -0.4116, 0.6835, 0.0514],\n", - " [ 0.1787, 0.0271, 0.1354, -0.3033, 0.6229, -0.3250, -0.3846,\n", - " 0.0812, 0.5633, 0.6879, -0.0325, -0.2383, -0.3521, -0.5850,\n", - " 0.3033, -0.3900, 0.6771, 0.3196, 0.5633, 0.2383],\n", - " [-0.1264, 0.5901, -0.3934, 0.3231, 0.0492, -0.5128, -0.8149,\n", - " 0.1124, -0.7517, 0.8711, 0.4004, -0.8992, 0.0702, -0.2178,\n", - " -0.8851, -0.5760, -0.1054, -0.0702, -0.3512, -0.5198],\n", - " [ 0.2612, 0.2570, 0.1542, -0.1071, -0.0300, 0.0257, -0.3854,\n", - " -0.0685, -0.2570, 0.0728, -0.4240, -0.3083, 0.1627, -0.3383,\n", - " -0.0428, 0.0300, -0.1199, 0.3683, 0.3298, -0.3340],\n", - " [ 0.4204, -0.2452, -0.0934, 0.2336, 0.1285, -0.1285, 0.2044,\n", - " -0.0701, 0.0058, 0.3971, 0.0175, -0.3270, 0.2803, 0.1810,\n", - " -0.4963, -0.5547, 0.0467, 0.0175, 0.1927, -0.2452]]],\n", - " grad_fn=), scale=tensor(0.0060, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)),\n", - " QuantTensor(value=tensor([[[-0.4438, -0.2686, -0.3095, -0.2978, -0.0993, 0.0584, 0.4846,\n", - " -0.0526, 0.3737, -0.4496, 0.1109, 0.7416, -0.0526, 0.3445,\n", - " 0.4963, 0.2803, 0.1927, 0.0000, 0.6131, 0.1109],\n", - " [ 0.4204, -0.2452, -0.0934, 0.2336, 0.1285, -0.1285, 0.2044,\n", - " -0.0701, 0.0058, 0.3971, 0.0175, -0.3270, 0.2803, 0.1810,\n", - " -0.4963, -0.5547, 0.0467, 0.0175, 0.1927, -0.2452]]],\n", - " grad_fn=), scale=tensor(0.0058, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)))" + " [[ 0.5110, -0.3555, 0.6443, -0.8221, 0.4888, -0.2074, 0.0444,\n", + " 0.4888, 0.5999, 0.4370, 0.0000, 0.5036, -0.7628, 0.9332,\n", + " -0.6147, 0.7332, 0.3629, 0.9184, 0.7702, -0.8887],\n", + " [ 0.8492, -0.3410, -0.3878, 0.1404, -0.3410, 0.3143, -0.1204,\n", + " 0.5817, 0.4413, 0.5550, 0.6486, -0.1070, 0.6285, -0.4948,\n", + " 0.2006, 0.1605, 0.0535, -0.4079, 0.3811, 0.4948],\n", + " [ 0.6060, 0.7666, -0.8688, -0.6863, -0.5111, -0.0803, -0.6425,\n", + " -0.0146, -0.3577, 0.3431, -0.6571, 0.5622, 0.0000, 0.7374,\n", + " -0.1314, -0.3650, 0.7520, 0.2336, -0.2847, -0.8250],\n", + " [ 0.3014, 0.2950, -0.0898, -0.3142, 0.4040, 0.4681, -0.0705,\n", + " -0.2052, 0.8143, -0.1603, 0.3334, -0.6733, 0.0834, 0.0898,\n", + " -0.4937, 0.1924, 0.0064, 0.4104, 0.6348, -0.3527],\n", + " [-0.6449, 0.5856, -0.0263, -0.0197, 0.8357, -0.5856, 0.0395,\n", + " -0.3422, 0.8028, 0.0855, -0.7238, -0.6317, 0.2764, -0.0461,\n", + " -0.4211, -0.5988, 0.2632, 0.4014, -0.7501, -0.5659]]],\n", + " grad_fn=), scale=tensor(0.0069, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)),\n", + " QuantTensor(value=tensor([[[-0.0066, 0.4804, 0.0066, -0.1184, 0.6843, -0.0197, 0.1448,\n", + " 0.1842, 0.6383, -0.1908, -0.0066, -0.1053, -0.1316, 0.0461,\n", + " -0.0066, -0.2764, 0.3751, 0.3619, 0.5001, -0.1316],\n", + " [-0.6449, 0.5856, -0.0263, -0.0197, 0.8357, -0.5856, 0.0395,\n", + " -0.3422, 0.8028, 0.0855, -0.7238, -0.6317, 0.2764, -0.0461,\n", + " -0.4211, -0.5988, 0.2632, 0.4014, -0.7501, -0.5659]]],\n", + " grad_fn=), scale=tensor(0.0066, grad_fn=), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True)))" ] }, "execution_count": 12, @@ -631,7 +648,16 @@ "cell_type": "code", "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "True\n" + ] + } + ], "source": [ "from torch.nn import RNN\n", "from brevitas.nn import QuantRNN\n", @@ -650,9 +676,9 @@ "# Generate random input\n", "inp = torch.randn(5, 2, 10)\n", "# Check outputs are the same\n", - "assert torch.allclose(quant_rnn(inp)[0], float_rnn(inp)[0], atol=ATOL)\n", + "assert_with_message(torch.allclose(quant_rnn(inp)[0], float_rnn(inp)[0], atol=ATOL))\n", "# Check hidden states are the same\n", - "assert torch.allclose(quant_rnn(inp)[1], float_rnn(inp)[1], atol=ATOL)" + "assert_with_message(torch.allclose(quant_rnn(inp)[1], float_rnn(inp)[1], atol=ATOL))" ] }, { @@ -751,23 +777,25 @@ " input_size: int,\n", " hidden_size: int,\n", " num_layers: int = 1,\n", - " bias: bool = True,\n", + " bias: Optional[bool] = True,\n", " batch_first: bool = False,\n", " bidirectional: bool = False,\n", - " weight_quant = Int8WeightPerTensorFloat,\n", - " bias_quant = Int32Bias,\n", - " io_quant = Int8ActPerTensorFloat,\n", - " gate_acc_quant = Int8ActPerTensorFloat,\n", - " sigmoid_quant = Uint8ActPerTensorFloat,\n", - " tanh_quant = Int8ActPerTensorFloat,\n", - " cell_state_quant = Int8ActPerTensorFloat,\n", + " weight_quant=Int8WeightPerTensorFloat,\n", + " bias_quant=Int32Bias,\n", + " io_quant=Int8ActPerTensorFloat,\n", + " gate_acc_quant=Int8ActPerTensorFloat,\n", + " sigmoid_quant=Uint8ActPerTensorFloat,\n", + " tanh_quant=Int8ActPerTensorFloat,\n", + " cell_state_quant=Int8ActPerTensorFloat,\n", " coupled_input_forget_gates: bool = False,\n", - " cat_output_cell_states = True,\n", - " shared_input_hidden_weights = False,\n", - " shared_intra_layer_weight_quant = False,\n", - " shared_intra_layer_gate_acc_quant = False,\n", - " shared_cell_state_quant = True,\n", + " cat_output_cell_states=True,\n", + " shared_input_hidden_weights=False,\n", + " shared_intra_layer_weight_quant=False,\n", + " shared_intra_layer_gate_acc_quant=False,\n", + " shared_cell_state_quant=True,\n", " return_quant_tensor: bool = False,\n", + " device: Optional[torch.device] = None,\n", + " dtype: Optional[torch.dtype] = None,\n", " **kwargs):\n", " super(QuantLSTM, self).__init__(\n", " layer_impl=_QuantLSTMLayer,\n", @@ -790,6 +818,8 @@ " shared_intra_layer_gate_acc_quant=shared_intra_layer_gate_acc_quant,\n", " shared_cell_state_quant=shared_cell_state_quant,\n", " return_quant_tensor=return_quant_tensor,\n", + " dtype=dtype,\n", + " device=device,\n", " **kwargs)\n", " if cat_output_cell_states and cell_state_quant is not None and not shared_cell_state_quant:\n", " raise RuntimeError(\"Concatenating cell states requires shared cell quantizers.\")\n", @@ -936,7 +966,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -958,9 +988,17 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-15 10:20:15.222259670 [W:onnxruntime:, graph.cc:1283 Graph] Initializer onnx::LSTM_93 appears in graph inputs and will not be treated as constant value/weight. This may prevent some of the graph optimizations, like const folding. Move it out of graph inputs if there is no need to override it, by either re-generating the model with latest exporter/converter or with the tool onnxruntime/tools/python/remove_initializer_from_input.py.\n" + ] + } + ], "source": [ "import onnxruntime as ort\n", "import numpy as np\n", @@ -1027,7 +1065,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -1049,9 +1087,17 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-15 10:20:22.930760716 [W:onnxruntime:, graph.cc:1283 Graph] Initializer onnx::LSTM_87 appears in graph inputs and will not be treated as constant value/weight. This may prevent some of the graph optimizations, like const folding. Move it out of graph inputs if there is no need to override it, by either re-generating the model with latest exporter/converter or with the tool onnxruntime/tools/python/remove_initializer_from_input.py.\n" + ] + } + ], "source": [ "import onnxruntime as ort\n", "import numpy as np\n", @@ -1079,7 +1125,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/giuseppe/Documents/git/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", + "/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", " warn('Keyword arguments are being passed but they not being used.')\n" ] } @@ -1127,7 +1173,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 25, @@ -1155,7 +1201,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/giuseppe/Documents/git/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", + "/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", " warn('Keyword arguments are being passed but they not being used.')\n" ] } @@ -1203,7 +1249,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 27, @@ -1225,17 +1271,39 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\alessand\\documents\\brevitas\\src\\brevitas\\nn\\mixin\\base.py:112: UserWarning: Keyword arguments are being passed but they not being used.\n", + "/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", " warn('Keyword arguments are being passed but they not being used.')\n" ] - }, + } + ], + "source": [ + "import torch\n", + "from brevitas.nn import QuantLSTM\n", + "from brevitas.export import export_onnx_qcdq\n", + "\n", + "quant_lstm_weight_only_bidirectional_2_layers = QuantLSTM(\n", + " input_size=10, hidden_size=20, bidirectional=True, num_layers=2, weight_bit_width=4, shared_intra_layer_weight_quant=True,\n", + " io_quant=None, bias_quant=None, gate_acc_quant=None, sigmoid_quant=None, tanh_quant=None, cell_state_quant=None)\n", + "export_path = 'quant_lstm_weight_only_bidirectional_2_layers_shared_q.onnx'\n", + "exported_model = export_onnx_qcdq(quant_lstm_weight_only_bidirectional_2_layers, (torch.randn(5, 1, 10)), opset_version=14, export_path=export_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "tags": [ + "skip-execution" + ] + }, + "outputs": [ { "name": "stdout", "output_type": "stream", @@ -1258,35 +1326,14 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 24, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], - "source": [ - "import torch\n", - "from brevitas.nn import QuantLSTM\n", - "from brevitas.export import export_onnx_qcdq\n", - "\n", - "quant_lstm_weight_only_bidirectional_2_layers = QuantLSTM(\n", - " input_size=10, hidden_size=20, bidirectional=True, num_layers=2, weight_bit_width=4, shared_intra_layer_weight_quant=True,\n", - " io_quant=None, bias_quant=None, gate_acc_quant=None, sigmoid_quant=None, tanh_quant=None, cell_state_quant=None)\n", - "export_path = 'quant_lstm_weight_only_bidirectional_2_layers_shared_q.onnx'\n", - "exported_model = export_onnx_qcdq(quant_lstm_weight_only_bidirectional_2_layers, (torch.randn(5, 1, 10)), opset_version=14, export_path=export_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "skip-execution" - ] - }, - "outputs": [], "source": [ "show_netron(export_path, 8086)" ] @@ -1301,17 +1348,40 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\alessand\\documents\\brevitas\\src\\brevitas\\nn\\mixin\\base.py:112: UserWarning: Keyword arguments are being passed but they not being used.\n", + "/scratch/fabian/brevitas/src/brevitas/nn/mixin/base.py:77: UserWarning: Keyword arguments are being passed but they not being used.\n", " warn('Keyword arguments are being passed but they not being used.')\n" ] - }, + } + ], + "source": [ + "import torch\n", + "from brevitas.nn import QuantLSTM\n", + "from brevitas.export import export_onnx_qcdq\n", + "\n", + "quant_lstm_weight_only_bidirectional_2_layers = QuantLSTM(\n", + " input_size=10, hidden_size=20, bidirectional=True, num_layers=2, weight_bit_width=4, \n", + " shared_input_hidden_weights=True, shared_intra_layer_weight_quant=True,\n", + " io_quant=None, bias_quant=None, gate_acc_quant=None, sigmoid_quant=None, tanh_quant=None, cell_state_quant=None)\n", + "export_path = 'quant_lstm_weight_only_bidirectional_2_layers_shared_q_ih.onnx'\n", + "exported_model = export_onnx_qcdq(quant_lstm_weight_only_bidirectional_2_layers, (torch.randn(5, 1, 10)), opset_version=14, export_path=export_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "tags": [ + "skip-execution" + ] + }, + "outputs": [ { "name": "stdout", "output_type": "stream", @@ -1334,36 +1404,14 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 25, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], - "source": [ - "import torch\n", - "from brevitas.nn import QuantLSTM\n", - "from brevitas.export import export_onnx_qcdq\n", - "\n", - "quant_lstm_weight_only_bidirectional_2_layers = QuantLSTM(\n", - " input_size=10, hidden_size=20, bidirectional=True, num_layers=2, weight_bit_width=4, \n", - " shared_input_hidden_weights=True, shared_intra_layer_weight_quant=True,\n", - " io_quant=None, bias_quant=None, gate_acc_quant=None, sigmoid_quant=None, tanh_quant=None, cell_state_quant=None)\n", - "export_path = 'quant_lstm_weight_only_bidirectional_2_layers_shared_q_ih.onnx'\n", - "exported_model = export_onnx_qcdq(quant_lstm_weight_only_bidirectional_2_layers, (torch.randn(5, 1, 10)), opset_version=14, export_path=export_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "skip-execution" - ] - }, - "outputs": [], "source": [ "show_netron(export_path, 8087)" ] @@ -1380,8 +1428,37 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 32, "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[W shape_type_inference.cpp:1974] Warning: The shape inference of onnx.brevitas::QuantLSTMCell type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (function UpdateReliable)\n", + "[W shape_type_inference.cpp:1974] Warning: The shape inference of onnx.brevitas::QuantLSTMCell type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (function UpdateReliable)\n", + "[W shape_type_inference.cpp:1974] Warning: The shape inference of onnx.brevitas::QuantLSTMCell type is missing, so it may result in wrong shape inference for the exported graph. Please consider adding it in symbolic function. (function UpdateReliable)\n" + ] + } + ], + "source": [ + "import torch\n", + "from brevitas.nn import QuantLSTM\n", + "from brevitas.export import export_qonnx\n", + "\n", + "quant_lstm = QuantLSTM(input_size=10, hidden_size=20)\n", + "export_path = 'quant_lstm.onnx'\n", + "exported_model = export_qonnx(quant_lstm, (torch.randn(5, 1, 10)), export_path=export_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "tags": [ + "skip-execution" + ] + }, "outputs": [ { "name": "stdout", @@ -1405,33 +1482,14 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 26, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], - "source": [ - "import torch\n", - "from brevitas.nn import QuantLSTM\n", - "from brevitas.export import export_qonnx\n", - "\n", - "quant_lstm = QuantLSTM(input_size=10, hidden_size=20)\n", - "export_path = 'quant_lstm.onnx'\n", - "exported_model = export_qonnx(quant_lstm, (torch.randn(5, 1, 10)), export_path=export_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "skip-execution" - ] - }, - "outputs": [], "source": [ "show_netron(export_path, 8088)" ] @@ -1504,7 +1562,7 @@ ], "metadata": { "kernelspec": { - "display_name": "pytorch_latest", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1518,9 +1576,8 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.5 (default, Oct 25 2019, 15:51:11) \n[GCC 7.3.0]" + "version": "3.11.5" }, - "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "c90b4b2913cf255302969887b74954c34035fee81e7901b217735b8a389ceb78" @@ -1528,5 +1585,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }