llama-2-0-inference-with-pretrained-weights.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-05T12:29:45.559671Z","iopub.execute_input":"2024-08-05T12:29:45.560723Z","iopub.status.idle":"2024-08-05T12:29:45.569769Z","shell.execute_reply.started":"2024-08-05T12:29:45.560681Z","shell.execute_reply":"2024-08-05T12:29:45.568447Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"%cd ..\n%rm -r \"Llama-2.0-architecture-and-inference-from-scratch-with-PyTorch/\"","metadata":{"execution":{"iopub.status.busy":"2024-08-05T14:19:38.355566Z","iopub.execute_input":"2024-08-05T14:19:38.356697Z","iopub.status.idle":"2024-08-05T14:19:39.503081Z","shell.execute_reply.started":"2024-08-05T14:19:38.356653Z","shell.execute_reply":"2024-08-05T14:19:39.501734Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"/kaggle/working\n","output_type":"stream"}]},{"cell_type":"code","source":"!git clone \"https://github.com/kaledai069/Llama-2.0-architecture-and-inference-from-scratch-with-PyTorch\"\n%cd \"Llama-2.0-architecture-and-inference-from-scratch-with-PyTorch/\"","metadata":{"execution":{"iopub.status.busy":"2024-08-05T14:19:39.505376Z","iopub.execute_input":"2024-08-05T14:19:39.505769Z","iopub.status.idle":"2024-08-05T14:19:41.032126Z","shell.execute_reply.started":"2024-08-05T14:19:39.505735Z","shell.execute_reply":"2024-08-05T14:19:41.030492Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Cloning into 'Llama-2.0-architecture-and-inference-from-scratch-with-PyTorch'...\nremote: Enumerating objects: 34, done.\u001b[K\nremote: Counting objects: 100% (34/34), done.\u001b[K\nremote: Compressing objects: 100% (23/23), done.\u001b[K\nremote: Total 34 (delta 17), reused 25 (delta 8), pack-reused 0\u001b[K\nUnpacking objects: 100% (34/34), 11.65 KiB | 917.00 KiB/s, done.\n/kaggle/working/Llama-2.0-architecture-and-inference-from-scratch-with-PyTorch\n","output_type":"stream"}]},{"cell_type":"code","source":"!bash download.sh","metadata":{"execution":{"iopub.status.busy":"2024-08-05T14:19:43.451885Z","iopub.execute_input":"2024-08-05T14:19:43.452643Z","iopub.status.idle":"2024-08-05T14:26:37.289358Z","shell.execute_reply.started":"2024-08-05T14:19:43.452604Z","shell.execute_reply":"2024-08-05T14:26:37.286745Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Downloading LICENSE and Acceptable Usage Policy\n--2024-08-05 14:19:44--  https://download.llamameta.net/LICENSE?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.57, 18.164.52.8, 18.164.52.96, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.57|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 7020 (6.9K) [binary/octet-stream]\nSaving to: './LICENSE'\n\n./LICENSE           100%[===================>]   6.86K  --.-KB/s    in 0s      \n\n2024-08-05 14:19:44 (131 MB/s) - './LICENSE' saved [7020/7020]\n\n--2024-08-05 14:19:44--  https://download.llamameta.net/USE_POLICY.md?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.48, 18.164.52.57, 18.164.52.8, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.48|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 4766 (4.7K) [binary/octet-stream]\nSaving to: './USE_POLICY.md'\n\n./USE_POLICY.md     100%[===================>]   4.65K  --.-KB/s    in 0s      \n\n2024-08-05 14:19:44 (108 MB/s) - './USE_POLICY.md' saved [4766/4766]\n\nDownloading tokenizer\n--2024-08-05 14:19:44--  https://download.llamameta.net/tokenizer.model?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.57, 18.164.52.8, 18.164.52.96, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.57|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 499723 (488K) [binary/octet-stream]\nSaving to: './tokenizer.model'\n\n./tokenizer.model   100%[===================>] 488.01K  --.-KB/s    in 0.02s   \n\n2024-08-05 14:19:44 (22.0 MB/s) - './tokenizer.model' saved [499723/499723]\n\n--2024-08-05 14:19:44--  https://download.llamameta.net/tokenizer_checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.48, 18.164.52.57, 18.164.52.8, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.48|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 50 [binary/octet-stream]\nSaving to: './tokenizer_checklist.chk'\n\n./tokenizer_checkli 100%[===================>]      50  --.-KB/s    in 0s      \n\n2024-08-05 14:19:44 (1.80 MB/s) - './tokenizer_checklist.chk' saved [50/50]\n\ntokenizer.model: OK\nDownloading llama-2-7b\n--2024-08-05 14:19:44--  https://download.llamameta.net/llama-2-7b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.57, 18.164.52.8, 18.164.52.96, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.57|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 13476925163 (13G) [binary/octet-stream]\nSaving to: './llama-2-7b/consolidated.00.pth'\n\n./llama-2-7b/consol 100%[===================>]  12.55G  31.6MB/s    in 6m 11s  \n\n2024-08-05 14:25:57 (34.6 MB/s) - './llama-2-7b/consolidated.00.pth' saved [13476925163/13476925163]\n\n--2024-08-05 14:25:57--  https://download.llamameta.net/llama-2-7b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.57, 18.164.52.96, 18.164.52.48, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.57|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 102 [application/json]\nSaving to: './llama-2-7b/params.json'\n\n./llama-2-7b/params 100%[===================>]     102  --.-KB/s    in 0s      \n\n2024-08-05 14:25:58 (4.32 MB/s) - './llama-2-7b/params.json' saved [102/102]\n\n--2024-08-05 14:25:58--  https://download.llamameta.net/llama-2-7b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoiZTNuZHZrb3p2ZjY2Ym93NXdrcjBtM2g5IiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyMjk1Mzk0MX19fV19&Signature=D~mZbI-xMHmJ53ruEnnMPuF6ie5G3LBNdjSy~kO6PHgLJhBbCM2mZEp2kDRHUFuEmsZ2V9PDRo-4CI9ZeEZzaiD~9ymkGHkFwBTi0aJxIZEzxLM5Q2dH7hpP-5YFong4CZ6scjfAZb4LNcdrZsJ2VmmsLC2Ypyw04Ao1AbdbhqYpEOQKJ8Sdq3k1mNEZ0APRcXD8RkhFCby9cw4rx44H4hXId3YOzy0pHEeyDyQ~wACozChgpGvzM5gGNNevkK3b82io6JFmwUE4irq9ZFik2K9xs-m6lXoZ9VT6SGl5jGERsxh8z7SgWiUa8Y~tSuaQmUvyUlbhjPoSVN3jPMNoZg__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=7519134394858148\nResolving download.llamameta.net (download.llamameta.net)... 18.164.52.8, 18.164.52.57, 18.164.52.48, ...\nConnecting to download.llamameta.net (download.llamameta.net)|18.164.52.8|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 100 [binary/octet-stream]\nSaving to: './llama-2-7b/checklist.chk'\n\n./llama-2-7b/checkl 100%[===================>]     100  --.-KB/s    in 0s      \n\n2024-08-05 14:25:59 (3.45 MB/s) - './llama-2-7b/checklist.chk' saved [100/100]\n\nChecking checksums\nconsolidated.00.pth: OK\nparams.json: OK\n","output_type":"stream"}]},{"cell_type":"code","source":"!python inference.py","metadata":{"execution":{"iopub.status.busy":"2024-08-05T14:26:37.294662Z","iopub.execute_input":"2024-08-05T14:26:37.295268Z","iopub.status.idle":"2024-08-05T14:56:36.452785Z","shell.execute_reply.started":"2024-08-05T14:26:37.295209Z","shell.execute_reply":"2024-08-05T14:56:36.451117Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"Loading Checkpoint from llama-2-7b/consolidated.00.pth !!!\nLoaded checkpoint in 45.84s\n/opt/conda/lib/python3.10/site-packages/torch/__init__.py:614: UserWarning: torch.set_default_tensor_type() is deprecated as of PyTorch 2.1, please use torch.set_default_dtype() and torch.set_default_device() as alternatives. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:451.)\n  _C._set_default_tensor_type(t)\nLoaded state dictionary from checkpoint into the model in 1.53\nGenerating tokens: 100%|██████████████████████| 142/142 [27:41<00:00, 11.70s/it]\nExplain the theory of relativity for a 5 year old. (I am a 5 year old)\nI am a 5 year old and my dad is trying to explain the theory of relativity to me. He says that it is all about time and space and the speed of light and how everything is relative and how I can't understand it. I am trying to understand it. I think I understand it but I need help. Please help me.\nI know that time is relative to speed and that I am moving at the speed of light. I know that I am not moving at the speed of light because I am not light. I know that the speed of light\n--------------------------------------------------\nKathmandu is the capital city of  Nepal. regulated by the 13th amendment to the constitution of Nepal. The city has a rich culture,  history and  architecture. It was the royal capital  and the residence of the Nepalese monarchy, until the 1st of  B.S. 2007. The city has a population of 545,404 and the  metropolitan area, also known as Greater Kathmandu, has a population of 5,508,042 . Kathmandu Valley is part of three districts (Kathmandu, L\n--------------------------------------------------\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}