chapter_2_fin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "18e45417",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'TensorFlow'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-6226d4aed6b5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mTensorFlow\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mcheckpoint\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"distilbert-base-uncased-finetuned-sst-2-english\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'TensorFlow'"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "import TensorFlow\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "\n",
    "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
    "\n",
    "model_inputs = tokenizer(sequence)\n",
    "print(model_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23ee0cb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': [[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102], [101, 2061, 2031, 1045, 999, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1]]}\n"
     ]
    }
   ],
   "source": [
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "model_inputs = tokenizer(sequences)\n",
    "print(model_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b0a878d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Will pad the sequences up to the maximum sequence length\n",
    "model_inputs = tokenizer(sequences, padding=\"longest\")\n",
    "\n",
    "# Will pad the sequences up to the model max length\n",
    "# (512 for BERT or DistilBERT)\n",
    "model_inputs = tokenizer(sequences, padding=\"max_length\")\n",
    "\n",
    "# Will pad the sequences up to the specified max length\n",
    "model_inputs = tokenizer(sequences, padding=\"max_length\", max_length=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d770fbf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "# Will truncate the sequences that are longer than the model max length\n",
    "# (512 for BERT or DistilBERT)\n",
    "model_inputs = tokenizer(sequences, truncation=True)\n",
    "\n",
    "# Will truncate the sequences that are longer than the specified max length\n",
    "model_inputs = tokenizer(sequences, max_length=8, truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2a9d6e53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'transformers.tokenization_utils_base.BatchEncoding'>\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "Unable to convert output to TensorFlow tensors format, TensorFlow is not installed.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-9f21d60f3434>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;31m# Returns TensorFlow tensors\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0mmodel_inputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msequences\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpadding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_tensors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"tf\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;31m# Returns NumPy arrays\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[0;32m   2600\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_target_context_manager\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2601\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_switch_to_input_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2602\u001b[1;33m             \u001b[0mencodings\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_one\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtext_pair\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mall_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2603\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtext_target\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2604\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_switch_to_target_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36m_call_one\u001b[1;34m(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[0;32m   2686\u001b[0m                 )\n\u001b[0;32m   2687\u001b[0m             \u001b[0mbatch_text_or_text_pairs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext_pair\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mtext_pair\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2688\u001b[1;33m             return self.batch_encode_plus(\n\u001b[0m\u001b[0;32m   2689\u001b[0m                 \u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2690\u001b[0m                 \u001b[0madd_special_tokens\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0madd_special_tokens\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36mbatch_encode_plus\u001b[1;34m(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[0;32m   2877\u001b[0m         )\n\u001b[0;32m   2878\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2879\u001b[1;33m         return self._batch_encode_plus(\n\u001b[0m\u001b[0;32m   2880\u001b[0m             \u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_text_or_text_pairs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2881\u001b[0m             \u001b[0madd_special_tokens\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0madd_special_tokens\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_fast.py\u001b[0m in \u001b[0;36m_batch_encode_plus\u001b[1;34m(self, batch_text_or_text_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose)\u001b[0m\n\u001b[0;32m    498\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0minput_ids\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msanitized_tokens\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"input_ids\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    499\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_eventual_warn_about_too_long_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 500\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mBatchEncoding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msanitized_tokens\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msanitized_encodings\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mreturn_tensors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    501\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    502\u001b[0m     def _encode_plus(\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, encoding, tensor_type, prepend_batch_axis, n_sequences)\u001b[0m\n\u001b[0;32m    209\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_n_sequences\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_sequences\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    210\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 211\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconvert_to_tensors\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensor_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtensor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprepend_batch_axis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mprepend_batch_axis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    212\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    213\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\transformers\\tokenization_utils_base.py\u001b[0m in \u001b[0;36mconvert_to_tensors\u001b[1;34m(self, tensor_type, prepend_batch_axis)\u001b[0m\n\u001b[0;32m    689\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtensor_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mTensorType\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTENSORFLOW\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    690\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_tf_available\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 691\u001b[1;33m                 raise ImportError(\n\u001b[0m\u001b[0;32m    692\u001b[0m                     \u001b[1;34m\"Unable to convert output to TensorFlow tensors format, TensorFlow is not installed.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    693\u001b[0m                 )\n",
      "\u001b[1;31mImportError\u001b[0m: Unable to convert output to TensorFlow tensors format, TensorFlow is not installed."
     ]
    }
   ],
   "source": [
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "# Returns PyTorch tensors\n",
    "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"pt\")\n",
    "print(type(model_inputs))\n",
    "# Returns TensorFlow tensors\n",
    "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"tf\")\n",
    "\n",
    "# Returns NumPy arrays\n",
    "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"np\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3d45b49e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]\n",
      "[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]\n"
     ]
    }
   ],
   "source": [
    "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
    "\n",
    "model_inputs = tokenizer(sequence)\n",
    "print(model_inputs[\"input_ids\"])\n",
    "\n",
    "tokens = tokenizer.tokenize(sequence)\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9da17cef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] i've been waiting for a huggingface course my whole life. [SEP]\n",
      "i've been waiting for a huggingface course my whole life.\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(model_inputs[\"input_ids\"]))\n",
    "print(tokenizer.decode(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "983af8e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SequenceClassifierOutput(loss=None, logits=tensor([[-1.5607,  1.6123],\n",
      "        [-3.6183,  3.9137]], grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "output = model(**tokens)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29bc500b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}