chapter_2_3_and_4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3c00b0e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertConfig, BertModel\n",
    "\n",
    "# Building the config\n",
    "config = BertConfig()\n",
    "\n",
    "# Building the model from the config\n",
    "model = BertModel(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "56a3151d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertConfig {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.33.0\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "973c6109",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "134fd2980b254313a7247ec18be9548e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2eca8ac44b6b42629659725b314849ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/436M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import BertModel\n",
    "\n",
    "model = BertModel.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "53acf67c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"directory_on_my_computer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3c071b40",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = [\"Hello!\", \"Cool.\", \"Nice!\"]\n",
    "encoded_sequences = [\n",
    "    [101, 7592, 999, 102],\n",
    "    [101, 4658, 1012, 102],\n",
    "    [101, 3835, 999, 102],\n",
    "]\n",
    "import torch\n",
    "\n",
    "model_inputs = torch.tensor(encoded_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6f9ef198",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = model(model_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7acaa714",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 4, 768])\n"
     ]
    }
   ],
   "source": [
    "print(output.last_hidden_state.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "936339e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Jim', 'Henson', 'was', 'a', 'puppeteer']\n"
     ]
    }
   ],
   "source": [
    "# tokenized_text = \"\".join(\"Jim Henson was a puppeteer\".split())\n",
    "tokenized_text = \"Jim Henson was a puppeteer\".split()\n",
    "print(tokenized_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9dfa8a31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa41d284b0bf407eb6184513c3744310",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91e2b495b434479384677058e2c46c85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6475ce4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9a7cd14e887443c8e24fcc56707a618",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cef048b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 7993, 170, 13809, 23763, 2443, 1110, 3014, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(\"Using a Transformer network is simple\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "016cf82c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('directory_on_my_computer\\\\tokenizer_config.json',\n",
       " 'directory_on_my_computer\\\\special_tokens_map.json',\n",
       " 'directory_on_my_computer\\\\vocab.txt',\n",
       " 'directory_on_my_computer\\\\added_tokens.json',\n",
       " 'directory_on_my_computer\\\\tokenizer.json')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"directory_on_my_computer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6024dbcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Using', 'a', 'Trans', '##former', 'network', 'is', 'simple']\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
    "\n",
    "sequence = \"Using a Transformer network is simple\"\n",
    "tokens = tokenizer.tokenize(sequence)\n",
    "\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4d5f5057",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7993, 170, 13809, 23763, 2443, 1110, 3014]\n"
     ]
    }
   ],
   "source": [
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "30c84435",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', '’', 've', 'been', 'waiting', 'for', 'a', 'Hu', '##gging', '##F', '##ace', 'course', 'my', 'whole', 'life', '.', 'I', 'hate', 'this', 'so', 'much', '!']\n",
      "[146, 787, 1396, 1151, 2613, 1111, 170, 20164, 10932, 2271, 7954, 1736, 1139, 2006, 1297, 119, 146, 4819, 1142, 1177, 1277, 106]\n"
     ]
    }
   ],
   "source": [
    "test_seq = [\"I’ve been waiting for a HuggingFace course my whole life.\", \"I hate this so much!\"]\n",
    "test_tokens = tokenizer.tokenize(test_seq)\n",
    "\n",
    "print(test_tokens)\n",
    "\n",
    "test_ids = tokenizer.convert_tokens_to_ids(test_tokens)\n",
    "\n",
    "print(test_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f2e7bd6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using a transformer network is simple\n"
     ]
    }
   ],
   "source": [
    "decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])\n",
    "print(decoded_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f01d6b25",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}