tashkeel_inference_tf2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "arabic tashkeel tf2",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNEZFtKc37Obgglzhm+iW7a",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ojuba-org/arabic-ml-data/blob/master/tashkeel_inference_tf2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K7Nzhoae6oyN"
      },
      "source": [
        "Tensorflow documentation\n",
        "\n",
        "* [v2](https://www.tensorflow.org/tutorials/text/text_generation)\n",
        "* [v1](https://github.com/tensorflow/docs/blob/master/site/en/r1/tutorials/sequences/text_generation.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6bB-XTwj9io8"
      },
      "source": [
        "%tensorflow_version 2.x"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jOeUIdNu_AU-"
      },
      "source": [
        "import sys\n",
        "import os\n",
        "import time\n",
        "import random"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "60OfCkAv-mMZ"
      },
      "source": [
        "import re\n",
        "import unicodedata"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApcMwBdi9sgg"
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "33-0GY_z7RIr"
      },
      "source": [
        "import tensorflow as tf"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "K = tf.keras\n",
        "KL = K.layers"
      ],
      "metadata": {
        "id": "HN-KMNSTTSRI"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vFID0M6u9xbf",
        "outputId": "cdcadf80-0647-4fe6-ee17-47b6731cc474"
      },
      "source": [
        "print(\"tf version = {} and py version = {}\".format(tf.__version__, sys.version))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf version = 2.7.0 and py version = 3.7.12 (default, Jan 15 2022, 18:48:18) \n",
            "[GCC 7.5.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XJ6ovEI-AED"
      },
      "source": [
        "assert sys.version_info.major == 3, 'please use python 3'\n",
        "assert tf.test.gpu_device_name()!='', 'no GPU, please enable GPU'"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! mkdir pretrained || : ; curl -sSL -o pretrained/tashkeel-lstm-weights.h5 https://github.com/ojuba-org/arabic-ml-data/raw/master/pretrained/tashkeel-lstm-weights.h5"
      ],
      "metadata": {
        "id": "jHtjhx-heQFH",
        "outputId": "874f51b7-6dd4-445c-ff2c-f9fdec6e6476",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mkdir: cannot create directory ‘pretrained’: File exists\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tashkeel_set = {'ً', 'َ', 'ِ', 'ّ', 'ٌ', 'ُ', 'ٍ', 'ْ'}"
      ],
      "metadata": {
        "id": "Fe7mhKqCf3_H"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7yfmBqVTOiD3"
      },
      "source": [
        "def tokenize_tashkeel(line):\n",
        "  offs=[ ix for ix, ch in enumerate(line) if ch not in tashkeel_set ]\n",
        "  offs.append(len(line))\n",
        "  return [ line[off: offs[ix+1]] for ix, off in enumerate(offs[:-1]) ]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9kbSBAjSGXc"
      },
      "source": [
        "(\n",
        "  VOC_PAD,\n",
        "  VOC_START,\n",
        "  VOC_UNKNOWN,\n",
        "  VOC_UNKNOWN_TASHKEEL, VOC_UNKNOWN_SHADDA,\n",
        "  VOC_UNKNOWN_LETTER, VOC_UNKNOWN_HAMZA, VOC_UNKNOWN_TEH_OR_HAA, VOC_UNKNOWN_MAQSORA,\n",
        "  VOC_FATHATAN, VOC_DAMMATAN, VOC_KASRATAN, \n",
        "  VOC_FATHA, VOC_DAMMA, VOC_KASRA,\n",
        "  VOC_SHADDA_FATHA, VOC_SHADDA_DAMMA, VOC_SHADDA_KASRA,\n",
        "  VOC_SUKUN,\n",
        "  VOC_STOP,\n",
        ") = range(20)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loDjt3YCWf82"
      },
      "source": [
        "#alpha=sorted(s)\n",
        "#tashkeel_set = { ch for ch in alpha if unicodedata.category(ch)=='Mn' }\n",
        "#chars = [ ch for ch in alpha if unicodedata.category(ch)!='Mn']\n",
        "chars = [' ', 'ء', 'آ', 'أ', 'ؤ', 'إ', 'ئ', 'ا', 'ب', 'ة', 'ت', 'ث', 'ج', 'ح', 'خ', 'د',\n",
        " 'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط', 'ظ', 'ع', 'غ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه', 'و', 'ى', 'ي']\n",
        "chars.sort()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tKHNejyiW-lu"
      },
      "source": [
        "ch2id = { ch: ix+VOC_STOP+1 for ix, ch in enumerate(chars)}"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArhRFFZrXnCR"
      },
      "source": [
        "ch2id.update({\n",
        "  \"\": VOC_START,\n",
        "  \"\\n\": VOC_STOP,\n",
        "  \"\\u064B\": VOC_FATHATAN, \"\\u064C\": VOC_DAMMATAN, \"\\u064D\": VOC_KASRATAN,\n",
        "  \"\\u064E\": VOC_FATHA, \"\\u064F\": VOC_DAMMA, \"\\u0650\": VOC_KASRA,\n",
        "  \"\\u0651\\u064E\": VOC_SHADDA_FATHA, \"\\u0651\\u064F\": VOC_SHADDA_DAMMA, \"\\u0651\\u0650\": VOC_SHADDA_KASRA,\n",
        "  \"\\u0651\": VOC_SUKUN,\n",
        "  \"\\u061f\": VOC_UNKNOWN,\n",
        "  \"\\u0640\\u061f\\u0640\": VOC_UNKNOWN_TASHKEEL,\n",
        "  \"\\u0651\\u061f\": VOC_UNKNOWN_SHADDA,\n",
        "})"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cmo6gshSYkii"
      },
      "source": [
        "id2ch = { v: k for k, v in ch2id.items() }"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id2ch2 = dict(id2ch)\n",
        "id2ch2.update({\n",
        "    VOC_START: \"\", \"\\n\": VOC_STOP, VOC_UNKNOWN: \"\", VOC_UNKNOWN_TASHKEEL: \"\", VOC_UNKNOWN_SHADDA: \"ّ\",\n",
        "})"
      ],
      "metadata": {
        "id": "JaVrtHZuZrEa"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-inoKgGpbszJ"
      },
      "source": [
        "voc_size = max(ch2id.values())+1"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "--8Fnu5VanXN"
      },
      "source": [
        "id2ch.update({ ix:'\\u061f' for ix in range(voc_size) if ix not in id2ch})"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id_repr = { globals()[i]: i for i in dir() if i.startswith('VOC_')}\n",
        "id_repr.update({i: unicodedata.name(id2ch[i]) for i in range(VOC_STOP+1, voc_size)})"
      ],
      "metadata": {
        "id": "vtTMHnZNsrw2"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_one_id(sub):\n",
        "  return (ch2id.get(sub[0], VOC_UNKNOWN),\n",
        "          ch2id.get(sub[1:] or \"\\u0640\\u061f\\u0640\", VOC_UNKNOWN_TASHKEEL),\n",
        "  )\n",
        "\n",
        "def tokenize_tashkeel_id(line):\n",
        "  return [(VOC_START, VOC_UNKNOWN_TASHKEEL)]+[tokenize_one_id(sub) for sub in tokenize_tashkeel(line.strip()+\"\\n\")]\n",
        "\n",
        "def tr_all_unknown(seq):\n",
        "  return [ (con_id, VOC_UNKNOWN_TASHKEEL) for con_id, vo_id in seq ]\n",
        "\n",
        "def pad_one(tuples, size):\n",
        "  l = len(tuples)\n",
        "  return (tuples + [(VOC_PAD, VOC_PAD)]*(size-l)) if l<size else tuples\n"
      ],
      "metadata": {
        "id": "1BwKPnqvQ5Zt"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "max_con = 200\n",
        "max_len = 2*max_con\n",
        "max_w_pad = max_len+1\n",
        "\n",
        "latent_dim = voc_size\n",
        "num_encoder_tokens = voc_size\n",
        "num_decoder_tokens = voc_size"
      ],
      "metadata": {
        "id": "QZxI_uiYTOBZ"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define an input sequence and process it.\n",
        "encoder_inputs = KL.Input(shape=(None, num_encoder_tokens))\n",
        "encoder = KL.LSTM(latent_dim, return_state=True)\n",
        "encoder_outputs, state_h, state_c = encoder(encoder_inputs)\n",
        "# We discard `encoder_outputs` and only keep the states.\n",
        "encoder_states = [state_h, state_c]\n",
        "\n",
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = KL.Input(shape=(None, num_decoder_tokens))\n",
        "# We set up our decoder to return full output sequences,\n",
        "# and to return internal states as well. We don't use the\n",
        "# return states in the training model, but we will use them in inference.\n",
        "decoder_lstm = KL.LSTM(latent_dim, return_sequences=True, return_state=True)\n",
        "decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
        "                                     initial_state=encoder_states)\n",
        "decoder_dense = KL.Dense(num_decoder_tokens, activation='softmax')\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "\n",
        "# Define the model that will turn\n",
        "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
        "model = K.Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9jSY-T-sTYKl",
        "outputId": "013a2072-a8be-473a-ff25-b4f5877b713e"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_1 (InputLayer)           [(None, None, 57)]   0           []                               \n",
            "                                                                                                  \n",
            " input_2 (InputLayer)           [(None, None, 57)]   0           []                               \n",
            "                                                                                                  \n",
            " lstm (LSTM)                    [(None, 57),         26220       ['input_1[0][0]']                \n",
            "                                 (None, 57),                                                      \n",
            "                                 (None, 57)]                                                      \n",
            "                                                                                                  \n",
            " lstm_1 (LSTM)                  [(None, None, 57),   26220       ['input_2[0][0]',                \n",
            "                                 (None, 57),                      'lstm[0][1]',                   \n",
            "                                 (None, 57)]                      'lstm[0][2]']                   \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, None, 57)     3306        ['lstm_1[0][0]']                 \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 55,746\n",
            "Trainable params: 55,746\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')"
      ],
      "metadata": {
        "id": "NIiq2r4LULkz"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weights_fn = 'pretrained/tashkeel-lstm-weights.h5'\n",
        "if os.path.exists(weights_fn):\n",
        "    model.load_weights(weights_fn)"
      ],
      "metadata": {
        "id": "jHPP5dweURJ_"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inference model"
      ],
      "metadata": {
        "id": "euxQrgNYU5Gu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define sampling models\n",
        "encoder_model = K.Model(encoder_inputs, encoder_states)\n",
        "\n",
        "decoder_state_input_h = KL.Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = KL.Input(shape=(latent_dim,))\n",
        "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
        "decoder_outputs, state_h, state_c = decoder_lstm(\n",
        "    decoder_inputs, initial_state=decoder_states_inputs)\n",
        "decoder_states = [state_h, state_c]\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "decoder_model = K.Model(\n",
        "    [decoder_inputs] + decoder_states_inputs,\n",
        "    [decoder_outputs] + decoder_states)\n"
      ],
      "metadata": {
        "id": "x1O7BbCrUpDq"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoder_model.summary()\n",
        "decoder_model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_RM_H2oFWBRJ",
        "outputId": "94b64174-e630-4b49-edf1-ab9bdd68c792"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_1 (InputLayer)        [(None, None, 57)]        0         \n",
            "                                                                 \n",
            " lstm (LSTM)                 [(None, 57),              26220     \n",
            "                              (None, 57),                        \n",
            "                              (None, 57)]                        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 26,220\n",
            "Trainable params: 26,220\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, None, 57)]   0           []                               \n",
            "                                                                                                  \n",
            " input_3 (InputLayer)           [(None, 57)]         0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, 57)]         0           []                               \n",
            "                                                                                                  \n",
            " lstm_1 (LSTM)                  [(None, None, 57),   26220       ['input_2[0][0]',                \n",
            "                                 (None, 57),                      'input_3[0][0]',                \n",
            "                                 (None, 57)]                      'input_4[0][0]']                \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, None, 57)     3306        ['lstm_1[1][0]']                 \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 29,526\n",
            "Trainable params: 29,526\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "I = np.identity(voc_size)\n",
        "\n",
        "def my_to_categorical(v):\n",
        "  return np.array([ I[ix] for ix in v])\n"
      ],
      "metadata": {
        "id": "2fnpdmIBV2et"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tashkeel(line, echo=False):\n",
        "  my_input_seq = np.array(tokenize_tashkeel_id(line)).reshape(-1)\n",
        "  max_len = len(my_input_seq)\n",
        "  \n",
        "  my_input = my_to_categorical(my_input_seq)\n",
        "  my_input = np.expand_dims(my_input, 0)\n",
        "  \n",
        "  sampled = []\n",
        "  sampled_ix = VOC_START\n",
        "  states_value = encoder_model.predict(my_input)\n",
        "  for i, ix in enumerate(my_input_seq):\n",
        "    target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
        "    if ix>VOC_STOP: sampled_ix = ix\n",
        "    if echo: print(id_repr[sampled_ix])\n",
        "    sampled.append(sampled_ix)\n",
        "    target_seq[0, 0, sampled_ix] = 1.\n",
        "    output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
        "    states_value = [h, c]\n",
        "    # Sample a token\n",
        "    sampled_ix = np.argmax(output_tokens[0, -1, :])\n",
        "    if sampled_ix == VOC_STOP or len(sampled) >= max_len: break\n",
        "  return \"\".join([ id2ch2.get(ix, \"\") for ix in sampled ])"
      ],
      "metadata": {
        "id": "i97gRtUxbK_A"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "line = \"إنك لعلى خلق عظيم\"\n",
        "print(tashkeel(line))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9xV3lboAXZ_V",
        "outputId": "05287deb-4629-4b62-dc3e-021abd9b6d25"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "إِنَّكَ لَعَلَى خَلَق عَظِيمٌ\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "line=input(\"enter arabic line: \")\n",
        "line=line.strip()\n",
        "print(tashkeel(line))"
      ],
      "metadata": {
        "id": "rg8MRs2eg4OC",
        "outputId": "71cfbb91-18ac-4ae4-d1f6-f393b67f1cea",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "enter arabic line: سبحان الذي أسرى\n",
            "سَبَحاًنَ الذَي أَسرَى \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "rSORAJFEhSB3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}